<s> return BuildMI ( MBB , MBBI , DL , TII -> get ( TargetOpcode :: CFI_INSTRUCTION ) ) . addCFIIndex ( MF . addFrameInst ( CFIInst ) )

. setMIFlag ( MachineInstr :: FrameSetup ) ;


<s> MachineInstr * <ISA_LIT>FrameLowering :: buildCFI ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator MBBI , const DebugLoc & DL , const MCCFIInstruction & CFIInst ) const { MachineFunction & MF = * MBB . getParent ( ) ; const <ISA_LIT>InstrInfo * TII = MF . getSubtarget < GCNSubtarget > ( ) . getInstrInfo ( ) ; return BuildMI ( MBB , MBBI , DL , TII -> get ( TargetOpcode :: CFI_INSTRUCTION ) ) . addCFIIndex ( MF . addFrameInst ( CFIInst )

) . setMIFlag ( MachineInstr :: FrameSetup ) ;


<s> MachineInstr * <ISA_LIT>FrameLowering :: buildCFIForRegToRegSpill ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator MBBI , const DebugLoc

& DL , const Register Reg , const Register RegCopy ) const {


<s> return buildCFI ( MBB , MBBI , DL , MCCFIInstruction :: createRegister ( nullptr , MCRI . getDwarfRegNum

( Reg , false ) , MCRI . getDwarfRegNum ( RegCopy , false ) ) ) ;


<s> raw_svector_ostream OSCFIInst ( CFIInst ) ; SmallString < <NUM_LIT> > Block ; raw_svector_ostream OSBlock ( Block ) ; OSCFIInst << uint8_t ( dwarf :: DW_CFA_expression ) ; encodeULEB128 ( DwarfSGPR , OSCFIInst ) ; for ( <ISA_LIT>MachineFunctionInfo :: SpilledReg Spill : VGPRSpills ) { encodeDwarfRegisterLocation ( MCRI . getDwarfRegNum ( Spill . VGPR , false ) , OSBlock ) ; OSBlock << uint8_t ( dwarf :: DW_OP_bit_piece ) ; const unsigned SGPRBitSize = <NUM_LIT> ; encodeULEB128 ( SGPRBitSize , OSBlock ) ; encodeULEB128 ( SGPRBitSize * Spill . Lane , OSBlock ) ; } encodeULEB128 ( Block . size ( ) , OSCFIInst


<s> int DwarfVGPR = MCRI . getDwarfRegNum ( VGPR , false ) ; SmallString < <NUM_LIT> > CFIInst ; raw_svector_ostream OSCFIInst ( CFIInst ) ; SmallString < <NUM_LIT> > Block ; raw_svector_ostream OSBlock ( Block ) ; OSCFIInst << uint8_t ( dwarf :: DW_CFA_expression ) ; encodeULEB128 ( DwarfVGPR , OSCFIInst ) ; encodeDwarfRegisterLocation ( DwarfVGPR , OSBlock ) ; OSBlock << uint8_t ( dwarf :: DW_OP_swap ) ; OSBlock << uint8_t ( dwarf :: DW_OP_LLVM_offset_uconst ) ; encodeULEB128 ( Offset , OSBlock ) ; OSBlock << uint8_t ( dwarf :: DW_OP_LLVM_call_frame_entry_reg ) ; encodeULEB128 ( MCRI . getDwarfRegNum ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> , false ) , OSBlock ) ; OSBlock << uint8_t ( dwarf :: DW_OP_deref_size ) ; OSBlock << uint8_t ( ST . getWavefrontSize ( ) / CHAR_BIT ) ; OSBlock << uint8_t ( dwarf ::


<s> MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( MF , FI ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOStore , FrameInfo . getObjectSize

( FI ) , FrameInfo . getObjectAlign ( FI ) ) ;


<s> static Register buildScratchExecCopy ( LivePhysRegs & LiveRegs , MachineFunction & MF , MachineBasicBlock & MBB , MachineBasicBlock :: iterator MBBI , bool IsProlog ) { Register ScratchExecCopy ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; DebugLoc DL ; initLiveRegs ( LiveRegs , TRI , FuncInfo , MF , MBB , MBBI , IsProlog ) ; ScratchExecCopy = findScratchNonCalleeSaveRegister ( MRI , LiveRegs , * TRI . getWaveMaskRegClass ( ) ) ; if ( ! ScratchExecCopy ) report_fatal_error ( <STR_LIT> ) ; LiveRegs . addReg (


<s> const DebugLoc & DL = I -> getDebugLoc ( ) ; unsigned Opc = I -> getOpcode ( ) ; bool IsDestroy = Opc == TII -> getCallFrameDestroyOpcode ( ) ; uint64_t CalleePopAmount = IsDestroy ? I -> getOperand ( <NUM_LIT> ) . getImm ( ) : <NUM_LIT> ; if ( ! hasReservedCallFrame ( MF ) ) { Amount = alignTo ( Amount , getStackAlign ( ) ) ; assert ( isUInt < <NUM_LIT> > ( Amount ) && <STR_LIT> ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register SPReg = MFI ->


<s> MachineBasicBlock :: iterator <ISA_LIT>FrameLowering :: eliminateCallFramePseudoInstr ( MachineFunction & MF , MachineBasicBlock & MBB , MachineBasicBlock :: iterator I ) const { int64_t Amount = I -> getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( Amount == <NUM_LIT> ) return MBB . erase ( I ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const DebugLoc & DL = I -> getDebugLoc ( ) ; unsigned Opc = I -> getOpcode ( ) ; bool IsDestroy = Opc == TII -> getCallFrameDestroyOpcode ( ) ; uint64_t CalleePopAmount = IsDestroy ? I -> getOperand ( <NUM_LIT> ) . getImm ( ) : <NUM_LIT> ; if ( ! hasReservedCallFrame ( MF ) ) { Amount = alignTo ( Amount , getStackAlign ( ) ) ; assert ( isUInt < <NUM_LIT> > ( Amount ) && <STR_LIT> ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register SPReg = MFI -> getStackPtrOffsetReg ( ) ; Amount *= getScratchScaleFactor ( ST ) ; if ( IsDestroy ) Amount = - Amount ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , SPReg ) . addReg ( SPReg ) . addImm ( Amount ) ; } else if ( CalleePopAmount != <NUM_LIT> ) { llvm_unreachable ( <STR_LIT> ) ; } return MBB . erase ( I


<s> initLiveRegs ( LiveRegs , TRI , FuncInfo , MF , MBB , MBBI , true ) ; MCPhysReg TmpVGPR = findScratchNonCalleeSaveRegister ( MRI , LiveRegs , <STR_LIT> ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , TmpVGPR ) . addReg ( TRI . getSubReg ( RetAddrReg , <STR_LIT> ) ) ; int DwordOff = <NUM_LIT> ; buildPrologSpill ( ST , TRI , * FuncInfo , LiveRegs , MF , MBB , MBBI , TmpVGPR , FI , DwordOff ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , TmpVGPR ) . addReg ( TRI . getSubReg ( RetAddrReg , <STR_LIT> ) ) ; DwordOff = <NUM_LIT> ; buildPrologSpill ( ST , TRI , * FuncInfo , LiveRegs , MF , MBB , MBBI , TmpVGPR , FI , DwordOff ) ; buildCFI ( MBB , MBBI , DL , MCCFIInstruction :: createOffset ( nullptr , MCRI -> getDwarfRegNum ( <STR_LIT> , false ) , MFI . getObjectOffset ( FI ) * ST . getWavefrontSize ( ) ) ) ; } if ( EXECSaveIndex && spilledToMemory ( MF , * EXECSaveIndex ) ) { const int FI = * EXECSaveIndex ; assert ( ! MFI . isDeadObjectIndex ( FI ) ) ; initLiveRegs ( LiveRegs , TRI , FuncInfo , MF , MBB , MBBI , true ) ; MCPhysReg TmpVGPR = findScratchNonCalleeSaveRegister ( MRI , LiveRegs , <STR_LIT> ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , TmpVGPR ) . addReg ( TRI . getSubReg ( <STR_LIT> , <STR_LIT> ) ) ; int DwordOff = <NUM_LIT> ; buildPrologSpill ( ST , TRI , * FuncInfo , LiveRegs , MF , MBB , MBBI , TmpVGPR , FI , DwordOff ) ; if ( ! ST . isWave32 ( ) ) { BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , TmpVGPR ) . addReg ( TRI . getSubReg ( <STR_LIT> , <STR_LIT> ) ) ; DwordOff = <NUM_LIT> ; buildPrologSpill ( ST , TRI , * FuncInfo , LiveRegs , MF , MBB , MBBI , TmpVGPR , FI , DwordOff ) ; } buildCFI ( MBB , MBBI , DL , MCCFIInstruction :: createOffset ( nullptr , MCRI -> getDwarfRegNum ( <STR_LIT> , false ) , MFI . getObjectOffset ( FI ) * ST . getWavefrontSize ( ) ) ) ; } } if ( ! emitSpillsToMem ) { if ( RASaveIndex && ! spilledToMemory ( MF , * RASaveIndex ) ) { MCRegister RetAddrReg = TRI . getReturnAddressReg ( MF ) ; if ( ! MBB . isLiveIn ( RetAddrReg ) ) MBB . addLiveIn (


<s> const MCInstrDesc & SAndB32 = TII -> get ( <STR_LIT> ) ; BuildMI ( MBB , I , DL , SAndB32 , FlatScrInitHi ) . addReg ( FlatScrInitHi ) . addImm ( <NUM_LIT> ) ; } else { Register FlatScratchInitReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; assert ( FlatScratchInitReg ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; MRI . addLiveIn ( FlatScratchInitReg ) ; MBB . addLiveIn ( FlatScratchInitReg ) ; FlatScrInitLo = TRI -> getSubReg ( FlatScratchInitReg , <STR_LIT> ) ; FlatScrInitHi = TRI -> getSubReg ( FlatScratchInitReg , <STR_LIT> ) ; } if ( ST . flatScratchIsPointer ( ) ) { if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX10 ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , FlatScrInitLo ) . addReg ( FlatScrInitLo ) . addReg ( ScratchWaveOffsetReg ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , FlatScrInitHi ) . addReg ( FlatScrInitHi ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( FlatScrInitLo ) . addImm ( int16_t ( <STR_LIT> :: ID_FLAT_SCR_LO | ( <NUM_LIT> << <STR_LIT> :: WIDTH_M1_SHIFT_ ) ) ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( FlatScrInitHi ) . addImm ( int16_t ( <STR_LIT> :: ID_FLAT_SCR_HI | ( <NUM_LIT> << <STR_LIT> :: WIDTH_M1_SHIFT_ ) ) ) ; return ; } BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitLo ) . addReg ( ScratchWaveOffsetReg ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitHi ) . addImm ( <NUM_LIT> ) ; return ; } assert ( ST . getGeneration ( ) < <ISA_LIT>Subtarget :: GFX9 ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg (


<s> if ( ST . isAmdHsaOrMesa ( F ) ) { PreloadedScratchRsrcReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; if ( ScratchRsrcReg && PreloadedScratchRsrcReg ) { MRI . addLiveIn ( PreloadedScratchRsrcReg ) ; MBB . addLiveIn ( PreloadedScratchRsrcReg ) ; } } Register ScratchWaveOffsetReg ; if ( TRI -> isSubRegisterEq ( ScratchRsrcReg , PreloadedScratchWaveOffsetReg ) ) { ArrayRef < MCPhysReg > AllSGPRs = TRI -> getAllSGPR32 ( MF ) ; unsigned NumPreloaded = MFI -> getNumPreloadedSGPRs ( ) ; AllSGPRs = AllSGPRs . slice ( std :: min ( static_cast < unsigned > ( AllSGPRs . size ( ) ) , NumPreloaded ) ) ; Register GITPtrLoReg = MFI -> getGITPtrLoReg ( MF ) ; for ( MCPhysReg Reg : AllSGPRs ) { if ( ! MRI . isPhysRegUsed ( Reg ) && MRI . isAllocatable ( Reg ) && ! TRI -> isSubRegisterEq ( ScratchRsrcReg , Reg ) && GITPtrLoReg != Reg ) { ScratchWaveOffsetReg = Reg ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchWaveOffsetReg ) . addReg ( PreloadedScratchWaveOffsetReg , RegState :: Kill ) ; break ; } } } else { ScratchWaveOffsetReg = PreloadedScratchWaveOffsetReg ; } assert ( ScratchWaveOffsetReg ) ; if ( requiresStackPointerReference ( MF ) ) { Register SPReg = MFI -> getStackPtrOffsetReg ( ) ; assert ( SPReg != <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , SPReg ) . addImm ( FrameInfo . getStackSize ( ) * getScratchScaleFactor ( ST ) ) ; } if ( hasFP ( MF ) ) { Register FPReg = MFI -> getFrameOffsetReg ( ) ; assert ( FPReg != <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , FPReg ) . addImm ( <NUM_LIT> ) ; } bool NeedsFlatScratchInit = MFI -> hasFlatScratchInit ( ) && ( MRI . isPhysRegUsed ( <STR_LIT> ) || FrameInfo . hasCalls ( ) || ( ! allStackObjectsAreDead ( MF ) && ST . enableFlatScratch ( ) ) ) ; if ( ( NeedsFlatScratchInit || ScratchRsrcReg ) && ! ST . flatScratchIsArchitected ( ) ) { MRI . addLiveIn ( PreloadedScratchWaveOffsetReg ) ; MBB . addLiveIn ( PreloadedScratchWaveOffsetReg ) ; } if ( NeedsFlatScratchInit ) { emitEntryFunctionFlatScratchInit ( MF , MBB , I , DL , ScratchWaveOffsetReg ) ; } if ( ScratchRsrcReg ) { emitEntryFunctionScratchRsrcRegSetup ( MF , MBB , I , DL , PreloadedScratchRsrcReg , ScratchRsrcReg ,


<s> const <ISA_LIT>RegisterInfo & TRI = ST . getInstrInfo ( ) -> getRegisterInfo ( ) ; Register StackPtrReg = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> getStackPtrOffsetReg ( ) ; buildCFI ( MBB , MBBI , DL , MCCFIInstruction :: createLLVMDefAspaceCfa ( nullptr , MCRI -> getDwarfRegNum ( StackPtrReg , false ) , <NUM_LIT> , <NUM_LIT> ) ) ; static const char PCEncodedInst [ ] = { dwarf :: DW_CFA_expression , <NUM_LIT> , <NUM_LIT> , static_cast < char > ( dwarf :: DW_OP_regx ) , <NUM_LIT> , static_cast < char > ( dwarf :: DW_OP_piece ) , <NUM_LIT> , static_cast < char > ( dwarf :: DW_OP_regx ) , <NUM_LIT> , static_cast < char > ( dwarf :: DW_OP_piece ) , <NUM_LIT> } ; buildCFI ( MBB , MBBI , DL , MCCFIInstruction :: createEscape ( nullptr , StringRef ( PCEncodedInst , sizeof ( PCEncodedInst ) ) ) ) ; BitVector IsCalleeSaved ( TRI . getNumRegs ( )


<s> static void encodeDwarfRegisterLocation ( int DwarfReg , raw_ostream & OS ) { if ( DwarfReg < <NUM_LIT> ) { OS << uint8_t ( dwarf :: DW_OP_reg0 + DwarfReg ) ; } else { OS << uint8_t ( dwarf ::


<s> return MFI . hasVarSizedObjects ( ) || MFI . hasStackMap (


<s> ArrayRef < MCPhysReg > AllSGPR128s = TRI -> getAllSGPR128 ( MF ) ; AllSGPR128s = AllSGPR128s . slice ( std :: min ( static_cast < unsigned > ( AllSGPR128s . size ( ) ) , NumPreloaded ) ) ; Register GITPtrLoReg = MFI -> getGITPtrLoReg ( MF ) ; for ( MCPhysReg Reg : AllSGPR128s ) { if ( ! MRI . isPhysRegUsed ( Reg ) && MRI . isAllocatable ( Reg ) && !

TRI -> isSubRegisterEq ( Reg , GITPtrLoReg ) ) {


<s> FrameReg = RI -> getFrameRegister ( MF ) ; return StackOffset :: getFixed ( MF . getFrameInfo ( )


<s> const <ISA_LIT>RegisterInfo * RI = MF . getSubtarget < GCNSubtarget > ( ) . getRegisterInfo ( ) ; FrameReg = RI -> getFrameRegister ( MF


<s> static unsigned getScratchScaleFactor ( const


<s> return ST . enableFlatScratch ( ) ? <NUM_LIT> :


<s> if ( ! MFI -> allocateSGPRSpillToVGPR ( MF , NewFI ) ) llvm_unreachable ( <STR_LIT> ) ; FrameIndex = NewFI ; LLVM_DEBUG ( auto Spill = MFI -> getSGPRToVGPRSpills ( NewFI ) . front ( ) ; dbgs ( ) << <STR_LIT> << ( IsFP ? <STR_LIT> : <STR_LIT> ) << <STR_LIT> << printReg ( Spill . VGPR , TRI ) << ':' << Spill . Lane << '\n' ) ; return ; } TempSGPR = findScratchNonCalleeSaveRegister ( MF . getRegInfo ( ) , LiveRegs , <STR_LIT> , true ) ; if ( ! TempSGPR ) { int NewFI = FrameInfo . CreateStackObject ( <NUM_LIT> , Align ( <NUM_LIT> ) , true , nullptr , TargetStackID :: SGPRSpill ) ; if ( TRI -> spillSGPRToVGPR ( ) && MFI -> allocateSGPRSpillToVGPR ( MF , NewFI ) ) { FrameIndex = NewFI ; LLVM_DEBUG ( auto Spill = MFI -> getSGPRToVGPRSpills ( NewFI ) . front ( ) ; dbgs ( ) << ( IsFP ? <STR_LIT> : <STR_LIT> ) << <STR_LIT> << printReg ( Spill . VGPR , TRI ) << ':' << Spill . Lane << '\n' ; ) ; } else


<s> static void getVGPRSpillLaneOrTempRegister ( MachineFunction & MF , LivePhysRegs & LiveRegs , Register & TempSGPR , Optional < int > & FrameIndex , bool IsFP ) { <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; if ( MFI -> haveFreeLanesForSGPRSpill ( MF , <NUM_LIT> ) ) { int NewFI = FrameInfo . CreateStackObject ( <NUM_LIT> , Align ( <NUM_LIT> ) , true , nullptr , TargetStackID :: SGPRSpill ) ; if ( ! MFI -> allocateSGPRSpillToVGPR ( MF ,

NewFI ) ) llvm_unreachable ( <STR_LIT> ) ;


<s> return frameTriviallyRequiresSP ( MFI ) || MFI . isFrameAddressTaken ( ) || MF . getSubtarget < GCNSubtarget > ( ) . getRegisterInfo (

) -> hasStackRealignment ( MF ) || MF . getTarget ( ) . Options . DisableFramePointerElim ( MF ) ;


<s> return MFI . getStackSize ( ) != <NUM_LIT> ; } return frameTriviallyRequiresSP ( MFI ) || MFI . isFrameAddressTaken ( ) || MF . getSubtarget < GCNSubtarget > ( ) . getRegisterInfo ( ) -> hasStackRealignment ( MF ) || MF . getTarget ( ) .

Options . DisableFramePointerElim ( MF ) ;


<s> bool SeenDbgInstr = false ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator Next ; for ( auto I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; I = Next ) { MachineInstr & MI = * I ; Next = std :: next ( I ) ; if ( MI . isDebugInstr ( ) ) SeenDbgInstr = true ; if ( TII -> isVGPRSpill ( MI ) ) { unsigned FIOp = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vaddr ) ; int FI = MI . getOperand ( FIOp ) . getIndex ( ) ; Register VReg = TII -> getNamedOperand ( MI , <STR_LIT> :: vdata ) -> getReg ( ) ; if ( FuncInfo -> allocateVGPRSpillToAGPR ( MF , FI , TRI -> isAGPR ( MRI , VReg ) ) ) { RS -> enterBasicBlock ( MBB ) ; TRI -> eliminateFrameIndex ( MI , <NUM_LIT> , FIOp , RS ) ; SpillFIs . set ( FI ) ; continue ; } } } } for ( MachineBasicBlock & MBB : MF ) { for ( MCPhysReg Reg : FuncInfo -> getVGPRSpillAGPRs ( ) ) MBB . addLiveIn ( Reg ) ; for ( MCPhysReg Reg : FuncInfo -> getAGPRSpillVGPRs (


<s> const <ISA_LIT>InstrInfo * TII = MF . getSubtarget < GCNSubtarget > ( ) . getInstrInfo ( ) ; for ( const CalleeSavedInfo & CS : C<ISA_LIT> ) { unsigned Reg = CS . getReg ( ) ; if ( CS . isSpilledToReg ( ) ) { BuildMI ( MBB , MBBI , DebugLoc ( ) , TII -> get ( TargetOpcode :: COPY ) , CS

. getDstReg ( ) ) . addReg ( Reg , getKillRegState ( true ) ) ;


<s> static bool spilledToMemory ( const MachineFunction & MF , int SaveIndex ) { const MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; return MFI . getStackID ( SaveIndex )


<s> return MFI . getStackID ( SaveIndex ) != TargetStackID ::


<s> auto & MBB = * CI . First -> getParent ( ) ; auto ClauseMI = BuildMI ( MBB , * CI . First , DebugLoc ( ) , <ISA_LIT>I -> get

( <STR_LIT> ) ) . addImm ( CI . Length - <NUM_LIT> ) ;


<s> if ( CI . First == CI . Last ) return false ; assert ( CI . Length <= <NUM_LIT> && <STR_LIT> ) ; auto & MBB = * CI . First -> getParent


<s> } if ( <ISA_LIT>InstrInfo :: isFLAT ( MI ) ) return HARDCLAUSE_FLAT ; if ( <ISA_LIT>InstrInfo :: isSMRD ( MI ) ) return HARDCLAUSE_SMEM ; } if ( MI . getOpcode ( ) == <STR_LIT> ) return HARDCLAUSE_INTERNAL ; if ( MI . isMetaInstruction ( ) )


<s> } if ( <ISA_LIT>InstrInfo :: isFLAT ( MI ) ) return HARDCLAUSE_FLAT ; if ( <ISA_LIT>InstrInfo :: isSMRD ( MI ) ) return HARDCLAUSE_SMEM ; } if ( MI . getOpcode ( ) == <STR_LIT> ) return HARDCLAUSE_INTERNAL ; if ( MI . isMetaInstruction


<s> bool Dummy2 ; unsigned Dummy3 ; SmallVector < const MachineOperand * , <NUM_LIT> > BaseOps ; if ( Type <= LAST_REAL_HARDCLAUSE_TYPE ) { if ( ! <ISA_LIT>I -> getMemOperandsWithOffsetWidth ( MI , BaseOps , Dummy1 , Dummy2 , Dummy3 , TRI ) ) { Type = HARDCLAUSE_ILLEGAL ; } } if ( CI . Length == <NUM_LIT> || ( CI . Length && Type != HARDCLAUSE_INTERNAL && Type != HARDCLAUSE_IGNORE && ( Type != CI . Type || ! <ISA_LIT>I -> shouldClusterMemOps ( CI . BaseOps , BaseOps , <NUM_LIT> , <NUM_LIT> ) ) ) ) { Changed |= emitClause ( CI ,


<s> const TargetRegisterInfo * TRI = ST . getRegisterInfo ( ) ; bool Changed = false ; for ( auto & MBB : MF ) { ClauseInfo CI ; for ( auto & MI : MBB ) { HardClauseType Type = getHardClauseType ( MI ) ; int64_t Dummy1 ; bool Dummy2 ; SmallVector < const MachineOperand * , <NUM_LIT> > BaseOps ; if ( Type <= LAST_REAL_HARDCLAUSE_TYPE ) { if ( ! <ISA_LIT>I -> getMemOperandsWithOffset ( MI , BaseOps , Dummy1 , Dummy2 , TRI ) ) { Type = HARDCLAUSE_ILLEGAL ; } } if ( CI . Length == <NUM_LIT> || ( CI . Length && Type != HARDCLAUSE_INTERNAL && ( Type != CI . Type || ! <ISA_LIT>I -> shouldClusterMemOps ( CI . BaseOps , BaseOps , <NUM_LIT> ) ) ) ) { Changed |= emitClause ( CI , <ISA_LIT>I ) ; CI = ClauseInfo ( ) ; } if ( CI . Length ) { ++ CI . Length ; if ( Type != HARDCLAUSE_INTERNAL ) { CI . Last = & MI ; CI . BaseOps = std :: move (


<s> for ( auto & MBB : MF ) { ClauseInfo CI ; for ( auto & MI : MBB ) { HardClauseType Type = getHardClauseType ( MI ) ; int64_t Dummy1 ; bool Dummy2 ; unsigned Dummy3 ; SmallVector < const MachineOperand * , <NUM_LIT> > BaseOps ; if ( Type <= LAST_REAL_HARDCLAUSE_TYPE ) { if ( ! <ISA_LIT>I -> getMemOperandsWithOffsetWidth ( MI , BaseOps , Dummy1 , Dummy2 , Dummy3 , TRI ) ) { Type = HARDCLAUSE_ILLEGAL ; } } if ( CI . Length == <NUM_LIT> || ( CI . Length && Type != HARDCLAUSE_INTERNAL && ( Type != CI . Type || ! <ISA_LIT>I -> shouldClusterMemOps (

CI . BaseOps , BaseOps , <NUM_LIT> , <NUM_LIT> ) ) ) ) {


<s> auto ClauseMI = BuildMI ( MBB , * CI . First , DebugLoc ( ) , <ISA_LIT>I -> get ( <STR_LIT> ) ) . addImm ( Size - <NUM_LIT> ) ; finalizeBundle ( MBB , ClauseMI -> getIterator ( ) , std :: next


<s> HardClauseType getHardClauseType ( const MachineInstr & MI ) { if ( MI . mayLoad ( ) ) { if ( <ISA_LIT>InstrInfo :: isVMEM ( MI ) || <ISA_LIT>InstrInfo :: isSegmentSpecificFLAT ( MI ) ) return HARDCLAUSE_VMEM ; if ( <ISA_LIT>InstrInfo :: isFLAT (


<s> HardClauseType getHardClauseType ( const MachineInstr & MI ) { if ( MI . mayLoad ( ) ) { if ( <ISA_LIT>InstrInfo :: isVMEM ( MI ) || <ISA_LIT>InstrInfo :: isSegmentSpecificFLAT (


<s> for ( auto & MI : MBB ) { HardClauseType Type = getHardClauseType ( MI ) ; int64_t Dummy1 ; bool Dummy2 ; unsigned Dummy3 ; SmallVector < const MachineOperand * , <NUM_LIT> > BaseOps ; if ( Type <= LAST_REAL_HARDCLAUSE_TYPE ) { if ( ! <ISA_LIT>I -> getMemOperandsWithOffsetWidth ( MI , BaseOps , Dummy1 , Dummy2 , Dummy3 , TRI ) ) { Type = HARDCLAUSE_ILLEGAL ; } } if ( CI . Length == <NUM_LIT> || ( CI . Length && Type != HARDCLAUSE_INTERNAL && ( Type != CI . Type || ! <ISA_LIT>I -> shouldClusterMemOps ( CI . BaseOps , BaseOps , <NUM_LIT> , <NUM_LIT> ) ) ) ) { Changed |= emitClause ( CI , <ISA_LIT>I ) ; CI = ClauseInfo ( ) ; } if ( CI . Length ) { ++ CI . Length ; if ( Type != HARDCLAUSE_INTERNAL ) { CI . Last = & MI ; CI . BaseOps = std :: move


<s> if ( MI -> isDebugValue ( ) || ! MI -> getDebugLoc ( ) ) { continue ; } auto DL = MI -> getDebugLoc ( ) ; auto CL = DL . getLine ( ) ; auto LineToInstEntry = LineToInst . find ( CL ) ; if ( LineToInstEntry == LineToInst . end ( ) ) { BuildMI ( * MBB , * MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; LineToInst . insert ( std :: make_pair ( CL , MI ) ) ; } else { LineToInstEntry -> second = MI ; } } } for ( auto LineToInstEntry = LineToInst . begin ( ) ; LineToInstEntry != LineToInst . end ( ) ; ++ LineToInstEntry ) { auto MBB = LineToInstEntry -> second -> getParent ( ) ; auto DL = LineToInstEntry -> second -> getDebugLoc ( ) ; MachineBasicBlock :: iterator MI = LineToInstEntry -> second ; ++ MI ; if ( MI != MBB -> end ( ) ) { BuildMI ( * MBB , * MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; } } MachineBasicBlock & MBB = MF . front ( ) ; MachineInstr & MI = MBB .


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { MachineFunctionPass :: getAnalysisUsage ( AU


<s> MachineBasicBlock * <ISA_LIT>InsertSkips :: insertSkipBlock ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator I ) const { MachineFunction * MF = MBB


<s> break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } assert ( MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) ; if ( TRI -> isVGPR ( MBB . getParent ( ) -> getRegInfo ( ) , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { Opcode = <STR_LIT> ( Opcode ) ; BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; } else { BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) . addReg ( <STR_LIT> , RegState :: Define ) . addImm ( <NUM_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( <NUM_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( <NUM_LIT> ) ; } break ; } case <STR_LIT> : { const MachineOperand & Op = MI . getOperand ( <NUM_LIT> ) ; int64_t KillVal = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; assert ( KillVal == <NUM_LIT> || KillVal == - <NUM_LIT> ) ; if ( Op . isImm ( ) ) { int64_t Imm = Op . getImm ( ) ; assert ( Imm == <NUM_LIT> || Imm == - <NUM_LIT> ) ; if ( Imm == KillVal ) BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addImm ( <NUM_LIT> ) ; break ; } unsigned Opcode = KillVal ? <STR_LIT> : <STR_LIT> ; BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . add ( Op ) ; break ; } default : llvm_unreachable (


<s> switch ( Opc ) { case TargetOpcode :: IMPLICIT_DEF : case TargetOpcode :: KILL : case TargetOpcode :: BUNDLE : case TargetOpcode :: CFI_INSTRUCTION : case TargetOpcode :: EH_LABEL : case TargetOpcode ::


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : ExecBranchStack . push_back ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ; MadeChange |= skipMaskBranch ( MI , MBB ) ; break ; case <STR_LIT> : if ( MBB . isLayoutSuccessor ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ) { MI . eraseFromParent ( ) ; } else if ( HaveSkipBlock ) { MI . eraseFromParent ( ) ; } break ; case <STR_LIT> : case <STR_LIT> : MadeChange = true ; kill ( MI ) ; if ( ExecBranchStack . empty ( ) ) { if ( skipIfDead ( MI , * NextBB ) ) { HaveSkipBlock = true ; NextBB = std :: next ( BI ) ; BE = MF . end ( ) ; } } else { HaveKill = true ; } MI . eraseFromParent ( ) ; break ; case <STR_LIT> : assert ( ! MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ) ; if ( BI != -- MF . end ( ) || I != MBB . getFirstTerminator ( ) ) { if ( ! EmptyMBBAtEnd ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF .


<s> unsigned NumInstr = <NUM_LIT> ; const MachineFunction * MF = From . getParent ( ) ; for ( MachineFunction :: const_iterator MBBI ( & From ) , ToI ( & To ) , End = MF -> end ( ) ; MBBI != End && MBBI != ToI ; ++ MBBI ) { const MachineBasicBlock & MBB = * MBBI ; for ( MachineBasicBlock :: const_iterator I = MBB . begin ( ) , E = MBB . end ( ) ; NumInstr < SkipThreshold && I != E ; ++ I ) { if ( opcodeEmitsNoInsts ( I -> getOpcode ( ) ) ) continue ; if ( I -> getOpcode ( ) == <STR_LIT> || I -> getOpcode ( ) == <STR_LIT> ) return true ; if ( ( I -> getOpcode ( ) == <STR_LIT> ) || ( I -> getOpcode ( ) == <STR_LIT> ) ) { return true ; } if ( I -> isInlineAsm ( ) ) { const MCAsmInfo * MAI = MF -> getTarget ( ) .


<s> BuildMI ( & MBB , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( & NextBB ) ; MachineBasicBlock :: iterator Insert = SkipBB -> begin ( ) ; BuildMI ( * SkipBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( * SkipBB , Insert , DL , TII -> get (


<s> for ( ; M != E ; ++ M ) { if ( M -> definesRegister ( SReg , TRI ) ) break ; if ( M -> modifiesRegister ( SReg , TRI ) ) return Changed ; ReadsSreg |= M -> readsRegister ( SReg , TRI ) ; } if ( M == E || ! M -> isMoveImmediate ( ) || ! M -> getOperand ( <NUM_LIT> ) . isImm ( ) || M -> getOperand ( <NUM_LIT> ) . getImm ( ) != - <NUM_LIT> ) return Changed ; if ( ! ReadsSreg && Op2 . isKill ( ) ) { A -> getOperand ( <NUM_LIT> ) . ChangeToImmediate ( - <NUM_LIT> ) ; M -> eraseFromParent ( ) ; } } if ( ! ReadsCond && A -> registerDefIsDead ( <STR_LIT> ) && MI . killsRegister ( CondReg , TRI ) ) A -> eraseFromParent ( ) ; bool IsVCCZ = MI . getOpcode ( ) == <STR_LIT> ; if ( SReg == ExecReg ) { if ( IsVCCZ ) { MI . eraseFromParent ( ) ; return true ; } MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; } else


<s> for ( MachineBasicBlock :: const_iterator I = MBB . begin ( ) , E = MBB . end ( ) ; NumInstr < SkipThreshold && I != E ; ++ I ) { if ( opcodeEmitsNoInsts ( I -> getOpcode ( ) ) ) continue ; if ( I -> getOpcode ( ) == <STR_LIT> || I -> getOpcode ( ) == <STR_LIT> ) return true ; if ( TII -> hasUnwantedEffectsWhenEXECEmpty ( * I ) ) return true ; if ( TII -> isSMRD ( * I ) || TII -> isVMEM ( * I ) || I -> getOpcode ( ) == <STR_LIT> ) return true ; ++


<s> bool ReadsCond = false ; unsigned Threshold = <NUM_LIT> ; for ( ++ A ; A != E ; ++ A ) { if ( ! -- Threshold ) return false ; if ( A -> modifiesRegister ( ExecReg , TRI ) ) return false ; if ( A -> modifiesRegister ( CondReg , TRI ) ) { if ( ! A -> definesRegister ( CondReg , TRI ) || A -> getOpcode ( ) != And ) return false ; break ; } ReadsCond |= A -> readsRegister ( CondReg , TRI ) ; } if ( A == E ) return false ; MachineOperand & Op1 = A -> getOperand ( <NUM_LIT> ) ; MachineOperand & Op2 = A -> getOperand ( <NUM_LIT> ) ; if ( Op1 . getReg ( ) != ExecReg && Op2 . isReg ( ) && Op2 . getReg ( ) == ExecReg ) { TII -> commuteInstruction ( * A ) ; Changed = true ; } if ( Op1 . getReg ( ) != ExecReg ) return Changed ; if ( Op2 . isImm ( ) && Op2 . getImm ( ) != - <NUM_LIT> ) return Changed ; unsigned SReg = <STR_LIT> ; if ( Op2 . isReg ( ) ) { SReg = Op2 . getReg (


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; const GCNSubtarget & ST = MBB . getParent ( ) -> getSubtarget < GCNSubtarget > ( ) ; const bool IsWave32 = ST . isWave32 ( ) ; const unsigned CondReg = TRI -> getVCC ( ) ; const unsigned ExecReg = IsWave32 ? <STR_LIT> : <STR_LIT> ; const unsigned And = IsWave32 ? <STR_LIT> : <STR_LIT> ; MachineBasicBlock :: reverse_iterator A = MI . getReverseIterator ( ) , E = MBB . rend ( ) ; bool ReadsCond = false ; unsigned Threshold = <NUM_LIT> ; for ( ++ A ; A != E ; ++ A ) { if ( ! -- Threshold ) return false ; if ( A -> modifiesRegister ( ExecReg , TRI ) ) return false ; if ( A -> modifiesRegister ( CondReg , TRI ) ) { if ( ! A -> definesRegister ( CondReg , TRI ) || A -> getOpcode ( ) != And ) return false ; break ; } ReadsCond |= A -> readsRegister ( CondReg , TRI ) ; } if ( A == E ) return false ; MachineOperand & Op1 = A -> getOperand ( <NUM_LIT> ) ; MachineOperand & Op2 = A -> getOperand ( <NUM_LIT> ) ; if ( Op1 . getReg ( ) != ExecReg && Op2 . isReg ( ) && Op2 . getReg ( ) == ExecReg ) { TII -> commuteInstruction ( * A ) ; Changed = true ; } if ( Op1 . getReg ( ) != ExecReg ) return Changed ; if ( Op2 . isImm ( ) && Op2 . getImm ( ) != - <NUM_LIT> ) return Changed ; unsigned SReg = <STR_LIT> ; if ( Op2 . isReg ( ) ) { SReg = Op2 . getReg ( ) ; auto M = std :: next ( A ) ; bool ReadsSreg = false ; for ( ; M != E ; ++ M ) { if ( M -> definesRegister ( SReg , TRI ) ) break ; if ( M -> modifiesRegister ( SReg , TRI ) ) return Changed ; ReadsSreg |= M -> readsRegister ( SReg , TRI ) ; } if ( M == E || ! M -> isMoveImmediate ( ) || ! M -> getOperand ( <NUM_LIT> ) . isImm ( )

|| M -> getOperand ( <NUM_LIT> ) . getImm ( ) != - <NUM_LIT> ) return Changed ;


<s> CallingConv :: ID CallConv = MBB . getParent ( ) -> getFunction ( ) -> getCallingConv ( ) ; assert ( CallConv == CallingConv :: <ISA_LIT>_PS || CallConv == CallingConv :: <ISA_LIT>_GS ) ; if ( Op . isImm ( ) ) { if ( Op . getImm ( ) &


<s> const MachineFunction * MF = From . getParent ( ) ; for ( MachineFunction :: const_iterator MBBI ( & From ) , ToI ( & To ) , End = MF -> end ( ) ; MBBI != End && MBBI != ToI ; ++ MBBI ) { const MachineBasicBlock & MBB = * MBBI ; for ( MachineBasicBlock :: const_iterator I = MBB . begin ( ) , E = MBB . end


<s> <ISA_LIT>InsertSkips ( ) : MachineFunctionPass ( ID ) , TRI ( nullptr ) , TII ( nullptr ) , SkipThreshold


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; MachineFunction * MF = MBB . getParent ( ) ; if ( MF -> getFunction ( ) -> getCallingConv ( ) != CallingConv :: <ISA_LIT>_PS || ! shouldSkip ( MBB , MBB . getParent ( ) -> back ( ) ) ) return false ; MachineBasicBlock * SkipBB = insertSkipBlock ( MBB , MI . getIterator ( ) ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; BuildMI ( & MBB , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( & NextBB ) ; MachineBasicBlock :: iterator Insert = SkipBB -> begin ( ) ; BuildMI ( * SkipBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( * SkipBB , Insert , DL , TII -> get


<s> MachineBasicBlock :: iterator Insert = SkipBB -> begin ( ) ; BuildMI ( * SkipBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState ::

Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ;


<s> void <ISA_LIT>InsertSkips :: kill ( MachineInstr & MI ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; const MachineOperand & Op = MI . getOperand ( <NUM_LIT> ) ; CallingConv :: ID CallConv = MBB . getParent ( ) -> getFunction ( ) -> getCallingConv ( ) ; assert ( CallConv == CallingConv :: <ISA_LIT>_PS || CallConv == CallingConv :: <ISA_LIT>_GS ) ; if ( Op . isImm ( ) ) { if ( Op . getImm ( ) & <NUM_LIT>


<s> SmallVector < MachineBasicBlock * , <NUM_LIT> > ExecBranchStack ; MachineFunction :: iterator NextBB ; MachineBasicBlock * EmptyMBBAtEnd = nullptr ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; BI = NextBB ) { NextBB = std :: next ( BI ) ; MachineBasicBlock & MBB = * BI ; bool HaveSkipBlock = false ; if ( ! ExecBranchStack . empty ( ) && ExecBranchStack . back ( ) == & MBB ) { ExecBranchStack . pop_back ( ) ; } if ( HaveKill && ExecBranchStack . empty ( ) ) { HaveKill = false ; } MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : ExecBranchStack . push_back ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ; MadeChange |= skipMaskBranch ( MI , MBB ) ; break ; case <STR_LIT> : if ( MBB . isLayoutSuccessor ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ) { MI . eraseFromParent ( ) ; } else if ( HaveSkipBlock ) { MI . eraseFromParent ( ) ; } break ; case <STR_LIT> : MadeChange = true ; kill ( MI


<s> SmallVector < MachineBasicBlock * , <NUM_LIT> > ExecBranchStack ; MachineFunction :: iterator NextBB ; MachineBasicBlock * EmptyMBBAtEnd = nullptr ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; BI = NextBB ) { NextBB = std :: next ( BI ) ; MachineBasicBlock & MBB = * BI ; bool HaveSkipBlock = false ; if ( ! ExecBranchStack . empty ( ) && ExecBranchStack . back ( ) == & MBB ) { ExecBranchStack . pop_back ( ) ; } if ( HaveKill && ExecBranchStack . empty ( ) ) { HaveKill = false ; } MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : ExecBranchStack . push_back ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ; MadeChange |= skipMaskBranch ( MI , MBB ) ; break ; case <STR_LIT> : if ( MBB . isLayoutSuccessor ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ) { MI . eraseFromParent (


<s> case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } const GCNSubtarget & ST = MBB . getParent ( ) -> getSubtarget < GCNSubtarget > ( ) ; if ( ST . hasNoSdstCMPX ( ) ) Opcode = <STR_LIT> ( Opcode ) ; assert ( MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) ; if ( TRI -> isVGPR ( MBB . getParent ( ) -> getRegInfo ( ) , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { Opcode = <STR_LIT> ( Opcode ) ; BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; } else { auto I = BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) ; if ( ! ST . hasNoSdstCMPX ( ) ) I . addReg ( <STR_LIT> , RegState :: Define ) ; I . addImm ( <NUM_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( <NUM_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; I . addImm (


<s> EarlyExitBlock = MF -> CreateMachineBasicBlock ( ) ; MF -> insert ( MF -> end ( ) , EarlyExitBlock ) ; generatePsEndPgm ( * EarlyExitBlock , EarlyExitBlock -> end


<s> static void generatePsEndPgm ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator I , DebugLoc DL , const <ISA_LIT>InstrInfo * TII ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT>

, RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ;


<s> case <STR_LIT> : if ( MBB . isLayoutSuccessor ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ) { assert ( & MI == & MBB . back ( ) ) ; MI . eraseFromParent ( ) ; MadeChange = true ; } break ; case <STR_LIT> : case <STR_LIT> : { MadeChange = true ; bool CanKill = kill ( MI ) ; if ( CanKill && MF . getFunction ( ) . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS && dominatesAllReachable ( MBB ) ) { KillInstrs . push_back ( & MI ) ; } else { MI . eraseFromParent ( ) ; } break ; } case <STR_LIT> : assert ( ! MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ) ; if ( & MBB != & MF . back ( ) || & MI != & MBB . back ( ) ) { if ( ! EmptyMBBAtEnd ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } MBB . addSuccessor ( EmptyMBBAtEnd ) ; BuildMI ( MBB , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( EmptyMBBAtEnd ) ; MI . eraseFromParent ( ) ; MDT -> getBase ( ) . insertEdge ( & MBB , EmptyMBBAtEnd ) ; } break ; default : break ; } } } for ( MachineInstr * Kill : KillInstrs ) { skipIfDead ( * Kill -> getParent ( ) , std :: next ( Kill -> getIterator ( ) ) , Kill -> getDebugLoc ( ) ) ; Kill -> eraseFromParent ( ) ; } KillInstrs . clear (


<s> auto NextBBI = std :: next ( MBB . getIterator ( ) ) ; bool NoSuccessor = llvm :: find ( MBB . successors ( ) , & * NextBBI ) == MBB . succ_end ( ) ; MachineBasicBlock * SkipBB ; if ( NoSuccessor ) { SkipBB = & MBB ; } else { SkipBB = MF -> CreateMachineBasicBlock ( ) ; MF -> insert ( NextBBI , SkipBB ) ; BuildMI ( & MBB , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( & * NextBBI ) ; MBB . addSuccessor ( SkipBB ) ; MDT -> addNewBlock ( SkipBB , &


<s> MachineBasicBlock :: iterator Insert = SkipBB -> begin ( ) ; BuildMI ( * SkipBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) ; BuildMI ( * SkipBB , Insert , DL , TII -> get (


<s> createEarlyExitBlock ( MBB ) ; NextBBI = std :: next ( MBB . getIterator ( ) ) ; } auto BranchMI = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( EarlyExitBlock ) ; auto Next = std :: next ( BranchMI -> getIterator ( ) ) ; if ( Next != MBB . end ( ) && ! Next -> isTerminator ( ) ) { MachineBasicBlock * SplitBB = MF -> CreateMachineBasicBlock ( MBB . getBasicBlock ( ) ) ; MF -> insert ( NextBBI , SplitBB ) ; SplitBB -> splice ( SplitBB -> begin ( ) , & MBB , I , MBB . end


<s> MachineFunction :: iterator NextBB ; MachineBasicBlock * EmptyMBBAtEnd = nullptr ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; BI = NextBB ) { NextBB = std :: next ( BI ) ; MachineBasicBlock & MBB = * BI ; bool HaveSkipBlock = false ; if ( ! ExecBranchStack . empty ( ) && ExecBranchStack . back ( ) == & MBB ) { ExecBranchStack . pop_back ( ) ; } if ( HaveKill && ExecBranchStack . empty ( ) ) { HaveKill = false ; } MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : ExecBranchStack . push_back ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ; MadeChange |= skipMaskBranch ( MI , MBB ) ; break ; case <STR_LIT> : if ( MBB . isLayoutSuccessor ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ) { MI . eraseFromParent ( ) ; } else if ( HaveSkipBlock ) { MI . eraseFromParent ( ) ; } break ; case <STR_LIT> : MadeChange = true ; kill ( MI ) ; if ( ExecBranchStack . empty ( ) ) { if ( skipIfDead ( MI , * NextBB ) ) { HaveSkipBlock = true ; NextBB = std :: next ( BI ) ; BE = MF . end ( ) ; } } else { HaveKill = true ; } MI . eraseFromParent (


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : { unsigned Opcode = <NUM_LIT> ; switch ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) ) { case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } assert ( MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) ; MachineInstr * NewMI ; if ( TRI -> isVGPR ( MBB . getParent ( ) -> getRegInfo ( ) , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { Opcode = <STR_LIT> ( Opcode ) ; NewMI = BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; } else { NewMI = BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) . addReg ( <STR_LIT> , RegState :: Define ) . addImm ( <NUM_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( <NUM_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( <NUM_LIT> ) ; } if ( NewMI -> hasExtraSrcRegAllocReq ( ) ) for ( MachineOperand & NewMO : NewMI -> uses ( ) ) if ( NewMO . isReg ( ) && NewMO . isUse ( ) ) NewMO . setIsRenamable ( false ) ; break ; } case <STR_LIT> : { const MachineOperand & Op = MI .


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; const DebugLoc DL = MI . getDebugLoc ( ) ; ensureEarlyExitBlock ( MBB , true ) ; auto BranchMI = BuildMI ( MBB , MI , DL , TII ->

get ( <STR_LIT> ) ) . addMBB ( EarlyExitBlock ) ;


<s> if ( IsPS ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <STR_LIT> :: ET_NULL ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; } BuildMI ( MBB , I , DL , TII -> get (


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MDT = & getAnalysis < MachineDominatorTree > ( ) ; SkipThreshold = SkipThresholdFlag ; SmallVector < MachineInstr * , <NUM_LIT> > KillInstrs ; SmallVector < MachineInstr * , <NUM_LIT> > EarlyTermInstrs ; bool MadeChange = false ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : MadeChange |= skipMaskBranch ( MI , MBB ) ; break ; case <STR_LIT> : if ( MBB . isLayoutSuccessor ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ) { assert ( & MI == & MBB . back ( ) ) ; MI . eraseFromParent ( ) ; MadeChange = true ; } break ; case <STR_LIT>


<s> } break ; case <STR_LIT> : case <STR_LIT> : { MadeChange = true ; bool CanKill = kill ( MI ) ; if ( CanKill && MF . getFunction ( ) . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS && dominatesAllReachable ( MBB ) ) { KillInstrs . push_back ( & MI ) ; } else { MI . eraseFromParent ( ) ; } break ; } case <STR_LIT> : assert ( ! MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ) ; if ( & MBB != & MF . back ( ) || & MI != & MBB . back ( ) ) { if ( ! EmptyMBBAtEnd ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } MBB . addSuccessor ( EmptyMBBAtEnd ) ; BuildMI ( MBB , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( EmptyMBBAtEnd ) ; MI . eraseFromParent ( ) ; MDT -> getBase ( ) . insertEdge ( & MBB , EmptyMBBAtEnd ) ; } break ; case <STR_LIT> : case <STR_LIT> : MadeChange |= optimizeVccBranch ( MI ) ; break ; default : break ; } } } for ( MachineInstr * Kill : KillInstrs ) { skipIfDead ( * Kill -> getParent ( ) , std :: next (

Kill -> getIterator ( ) ) , Kill -> getDebugLoc ( ) ) ;


<s> MachineFunction * MF = MBB . getParent ( ) ; DebugLoc DL ; if ( ! EarlyExitBlock ) { EarlyExitBlock = MF -> CreateMachineBasicBlock ( ) ; MF -> insert ( MF -> end ( ) , EarlyExitBlock ) ; generatePsEndPgm ( * EarlyExitBlock , EarlyExitBlock -> end ( ) , DL , TII ) ; EarlyExitClearsExec = false ; } if ( ClearExec && ! EarlyExitClearsExec ) { const GCNSubtarget & ST = MF -> getSubtarget < GCNSubtarget > ( ) ; unsigned Mov = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; Register Exec = ST . isWave32 ( )


<s> BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <STR_LIT> :: ET_NULL ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addImm


<s> AU . addRequired < MachineDominatorTree > ( ) ; AU . addPreserved < MachineDominatorTree > ( ) ; MachineFunctionPass :: getAnalysisUsage ( AU


<s> case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } const GCNSubtarget & ST = MBB . getParent ( ) -> getSubtarget < GCNSubtarget > ( ) ; if ( ST . hasNoSdstCMPX ( ) ) Opcode = <STR_LIT> ( Opcode ) ; assert ( MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) ; if ( TRI -> isVGPR ( MBB . getParent ( ) -> getRegInfo ( ) , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { Opcode = <STR_LIT> ( Opcode ) ; BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; } else { auto I = BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) ; if ( ! ST . hasNoSdstCMPX ( ) ) I . addReg ( <STR_LIT> , RegState :: Define ) ; I . addImm ( <NUM_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( <NUM_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; I . addImm ( <NUM_LIT> ) ; } return true ; } case <STR_LIT> : { const MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; const GCNSubtarget & ST = MF -> getSubtarget < GCNSubtarget > ( ) ; unsigned Exec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; const MachineOperand & Op = MI . getOperand ( <NUM_LIT> ) ; int64_t KillVal = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; assert ( KillVal == <NUM_LIT> || KillVal == - <NUM_LIT> ) ; if ( Op . isImm ( ) ) { int64_t Imm = Op . getImm ( ) ; assert ( Imm == <NUM_LIT> || Imm == - <NUM_LIT> ) ; if ( Imm == KillVal ) { BuildMI ( MBB , & MI , DL , TII -> get ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , Exec ) . addImm ( <NUM_LIT> ) ; return true ; } return


<s> break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } const GCNSubtarget & ST = MBB . getParent ( ) -> getSubtarget < GCNSubtarget > ( ) ; if ( ST . hasNoSdstCMPX ( ) ) Opcode = <STR_LIT> ( Opcode ) ; assert ( MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) ; if ( TRI -> isVGPR ( MBB . getParent ( ) -> getRegInfo ( ) , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { Opcode = <STR_LIT> ( Opcode ) ; BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; } else { auto I = BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) ; if ( ! ST . hasNoSdstCMPX ( ) ) I . addReg ( <STR_LIT> , RegState :: Define ) ; I . addImm ( <NUM_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( <NUM_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; I . addImm ( <NUM_LIT> ) ; } return true ; } case <STR_LIT> : { const MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; const GCNSubtarget & ST = MF -> getSubtarget < GCNSubtarget > ( ) ; unsigned Exec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; const MachineOperand & Op = MI . getOperand ( <NUM_LIT> ) ; int64_t KillVal = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; assert ( KillVal == <NUM_LIT> || KillVal == - <NUM_LIT>


<s> if ( MI . isMetaInstruction ( ) ) return true ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : return true


<s> bool <ISA_LIT>InsertSkips :: runOnMachineFunction ( MachineFunction & MF ) { const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MDT = & getAnalysis < MachineDominatorTree > ( ) ; SkipThreshold = SkipThresholdFlag ; SmallVector < MachineInstr * , <NUM_LIT> > KillInstrs ; bool MadeChange = false ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : MadeChange |= skipMaskBranch ( MI , MBB ) ; break ; case <STR_LIT> : if ( MBB . isLayoutSuccessor ( MI . getOperand ( <NUM_LIT> ) . getMBB ( )


<s> if ( TII -> hasUnwantedEffectsWhenEXECEmpty ( * I ) ) return true ; if ( TII -> isSMRD ( * I ) || TII -> isVMEM ( * I ) || TII -> isFLAT ( * I ) || I -> getOpcode ( ) == <STR_LIT> ) return true ; ++ NumInstr ; if ( NumInstr >= SkipThreshold ) return true ; } } return false


<s> assert ( MF -> getFunction ( ) . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS ) ; auto NextBBI = std :: next ( MBB . getIterator ( ) ) ; bool NoSuccessor = I == MBB . end ( ) && ! llvm :: is_contained ( MBB . successors ( ) , & * NextBBI ) ; if ( NoSuccessor ) { generatePsEndPgm ( MBB , I , DL , TII ) ; } else { ensureEarlyExitBlock ( MBB , false ) ; MachineInstr * BranchMI = BuildMI ( MBB , I , DL , TII ->


<s> bool <ISA_LIT>InsertSkips :: skipMaskBranch ( MachineInstr & MI , MachineBasicBlock & SrcMBB ) { MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( ! shouldSkip ( * * SrcMBB . succ_begin ( ) , * DestBB ) ) return false ; const DebugLoc & DL = MI


<s> KillWaitBrackets . push_back ( llvm :: make_unique <

BlockWaitcntBrackets > ( * Bracket ) ) ;


<s> void addKillWaitBracket ( BlockWaitcntBrackets * Bracket ) { KillWaitBrackets . push_back ( llvm :: make_unique < BlockWaitcntBrackets


<s> case LGKM_CNT : { if ( EventUBs [ SMEM_ACCESS ] > ScoreLBs [ LGKM_CNT ] && EventUBs [ SMEM_ACCESS ] <= ScoreUBs [ LGKM_CNT ] ) { return true ; } int NumEventTypes = <NUM_LIT> ; if ( EventUBs [ LDS_ACCESS ] > ScoreLBs [ LGKM_CNT ] && EventUBs [ LDS_ACCESS ] <= ScoreUBs [ LGKM_CNT ] ) { NumEventTypes ++ ; } if ( EventUBs [ GDS_ACCESS ] > ScoreLBs [ LGKM_CNT ] && EventUBs [ GDS_ACCESS ] <= ScoreUBs [ LGKM_CNT ] ) { NumEventTypes ++ ; } if ( EventUBs [ SQ_MESSAGE ] > ScoreLBs [ LGKM_CNT ] && EventUBs [ SQ_MESSAGE ] <= ScoreUBs [ LGKM_CNT ] ) { NumEventTypes ++ ; } if ( NumEventTypes <= <NUM_LIT>


<s> } int NumEventTypes = <NUM_LIT> ; if ( EventUBs [ LDS_ACCESS ] > ScoreLBs [ LGKM_CNT ] && EventUBs [ LDS_ACCESS ] <= ScoreUBs [ LGKM_CNT ] ) { NumEventTypes ++ ; } if ( EventUBs [ GDS_ACCESS ] > ScoreLBs [ LGKM_CNT ] && EventUBs [ GDS_ACCESS ] <= ScoreUBs [ LGKM_CNT ] ) { NumEventTypes ++ ; } if ( EventUBs [ SQ_MESSAGE ] > ScoreLBs [ LGKM_CNT ] && EventUBs [ SQ_MESSAGE ] <= ScoreUBs [ LGKM_CNT ] ) { NumEventTypes ++ ; } if ( NumEventTypes <= <NUM_LIT> ) { return false ; } break ; } case EXP_CNT : { if ( MixedExpTypes ) return true ; int NumEventTypes = <NUM_LIT> ; if ( EventUBs [ EXP_GPR_LOCK ] > ScoreLBs [ EXP_CNT ] && EventUBs [ EXP_GPR_LOCK ] <= ScoreUBs [ EXP_CNT ] ) { NumEventTypes ++ ; } if ( EventUBs [ GDS_GPR_LOCK ] > ScoreLBs [ EXP_CNT ] && EventUBs [ GDS_GPR_LOCK ] <= ScoreUBs [ EXP_CNT ] ) { NumEventTypes ++ ; } if ( EventUBs [ VMW_GPR_LOCK ] > ScoreLBs [ EXP_CNT ] && EventUBs [ VMW_GPR_LOCK ] <= ScoreUBs [ EXP_CNT ] ) { NumEventTypes ++ ; } if ( EventUBs [ EXP_PARAM_ACCESS ] > ScoreLBs [ EXP_CNT ] && EventUBs [ EXP_PARAM_ACCESS ] <= ScoreUBs [ EXP_CNT ] ) { NumEventTypes ++ ; } if ( EventUBs [ EXP_POS_ACCESS ] > ScoreLBs [ EXP_CNT ] && EventUBs [ EXP_POS_ACCESS ] <= ScoreUBs [ EXP_CNT ] ) { NumEventTypes ++


<s> return VM_CNT ; case LDS_ACCESS : case GDS_ACCESS : case SQ_MESSAGE : case SMEM_ACCESS : return LGKM_CNT ; case EXP_GPR_LOCK : case


<s> if ( PredEventUB > KillWaitBrackets [ I ] -> getScoreLB ( T ) ) { int NewEventUB = MaxPending [ T ] + PredEventUB - KillWaitBrackets [ I ] -> getScoreUB ( T ) ; if ( NewEventUB


<s> AU . addRequired < MachineLoopInfo > ( ) ; MachineFunctionPass :: getAnalysisUsage (


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . setPreservesCFG (


<s> assert ( W < NUM_WAIT_EVENTS ) ; return EventUBs [ W


<s> bool getRevisitLoop ( ) const { return RevisitLoop


<s> assert ( T < NUM_INST_CNTS ) ; if ( T >= NUM_INST_CNTS ) return <NUM_LIT> ; return ScoreLBs [


<s> bool hasPendingSMEM ( ) const { return ( EventUBs [ SMEM_ACCESS ] > ScoreLBs [ LGKM_CNT ] && EventUBs [ SMEM_ACCESS ]


<s> bool hasPendingSMEM ( ) const { return ( EventUBs [ SMEM_ACCESS ] > ScoreLBs [ LGKM_CNT ] && EventUBs

[ SMEM_ACCESS ] <= ScoreUBs [ LGKM_CNT ] ) ;


<s> BlockWaitcntBrackets * ScoreBrackets = BlockWaitcntBracketsMap [ & Block ] . get ( ) ; DEBUG ( { dbgs ( ) << <STR_LIT> << Block . getNumber ( ) ; ScoreBrackets -> dump (


<s> mergeInputScoreBrackets ( Block ) ; BlockWaitcntBrackets * ScoreBrackets = BlockWaitcntBracketsMap [ & Block


<s> MachineBasicBlock * <ISA_LIT>InsertWaitcnts :: loopBottom ( const MachineLoop * Loop ) { MachineBasicBlock * Bottom = Loop -> getHeader


<s> if ( MI . memoperands_empty ( ) ) return true ; for ( const MachineMemOperand * Memop : MI . memoperands ( ) ) { unsigned AS = Memop ->


<s> bool mixedExpTypes ( ) const { return MixedExpTypes


<s> OS << RelScore << <STR_LIT> << J << <STR_LIT> ; } else { OS << RelScore << <STR_LIT> ; } } if ( T == LGKM_CNT ) { for ( int J = <NUM_LIT> ; J <= getMaxSGPR ( ) ; J ++ ) { int RegScore = getRegScore ( J + NUM_ALL_VGPRS , LGKM_CNT ) ; if ( RegScore <= LB ) continue ; int RelScore = RegScore - LB - <NUM_LIT> ; OS << RelScore << <STR_LIT> << J << <STR_LIT> ; } } } OS << '\n' ; } OS <<


<s> void setMixedExpTypes ( bool MixedExpTypesIn ) { MixedExpTypes =


<s> void setRegScore ( int GprNo , InstCounterType T , int32_t Val ) { if ( GprNo < NUM_ALL_VGPRS ) { if ( GprNo > VgprUB ) { VgprUB = GprNo ; } VgprScores [ T ] [ GprNo ] = Val ; } else { assert ( T == LGKM_CNT ) ; if ( GprNo - NUM_ALL_VGPRS > SgprUB ) { SgprUB = GprNo


<s> void setRevisitLoop ( bool RevisitLoopIn ) { RevisitLoop =


<s> if ( T >= NUM_INST_CNTS ) return ; ScoreLBs [ T ] =


<s> assert ( T < NUM_INST_CNTS ) ; if ( T >= NUM_INST_CNTS ) return ; ScoreLBs [ T


<s> void setScoreUB ( InstCounterType T , int32_t Val ) { assert ( T < NUM_INST_CNTS ) ; if ( T >= NUM_INST_CNTS ) return ; ScoreUBs [ T ]


<s> void setWaitAtBeginning ( ) { WaitAtBeginning =


<s> unsigned int NeedWait = <NUM_LIT> ; if ( ScoreToWait == - <NUM_LIT> ) { NeedWait = CNT_MASK ( T ) ; setScoreLB ( T , getScoreUB ( T ) ) ; return NeedWait ; } const int32_t LB = getScoreLB ( T ) ; const int32_t UB = getScoreUB ( T ) ; if ( ( UB >= ScoreToWait ) && ( ScoreToWait > LB ) ) { if ( T == VM_CNT && hasPendingFlat ( ) ) { NeedWait = CNT_MASK ( T ) ; setScoreLB ( T , getScoreUB ( T ) ) ; } else if ( counterOutOfOrder ( T ) ) { NeedWait = CNT_MASK ( T ) ; setScoreLB ( T , getScoreUB ( T ) ) ; } else


<s> ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; } } else if ( TII -> isFLAT ( Inst ) ) { assert ( Inst . mayLoad ( ) || Inst . mayStore ( ) ) ; if ( TII -> usesVM_CNT ( Inst ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; if ( TII -> usesLGKM_CNT ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; if ( mayAccessLDSThroughFlat ( Inst ) ) ScoreBrackets -> setPendingFlat ( ) ; } } else if ( <ISA_LIT>InstrInfo :: isVMEM ( Inst ) && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; if ( ( Inst . mayStore ( ) || <STR_LIT> ( Inst . getOpcode ( ) ) != - <NUM_LIT> ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMW_GPR_LOCK , Inst ) ; } } else if ( TII -> isSMRD ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS , Inst ) ; } else { switch ( Inst . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SQ_MESSAGE , Inst ) ; break ; case <STR_LIT> : case <STR_LIT>


<s> void addKillWaitBracket ( BlockWaitcntBrackets * Bracket ) { KillWaitBrackets . push_back ( make_unique < BlockWaitcntBrackets > ( * Bracket


<s> KillWaitBrackets . push_back ( make_unique < BlockWaitcntBrackets > ( *


<s> BlockWaitcntBrackets ( ) : WaitAtBeginning ( false ) , ValidLoop ( false ) , MixedExpTypes ( false ) , LoopRegion ( NULL ) , PostOrder ( <NUM_LIT> ) , Waitcnt ( NULL ) , VgprUB ( <NUM_LIT> ) , SgprUB ( <NUM_LIT> ) { for ( enum InstCounterType T = VM_CNT ; T < NUM_INST_CNTS ; T = ( enum InstCounterType ) ( T +


<s> MBB . push_back ( Waitcnt ) ; return ; } MachineBasicBlock :: iterator It = MBB . end ( ) ; MachineInstr * MI = & * ( -- It ) ; if ( MI -> isBranch ( ) ) { MBB . insert ( It , Waitcnt ) ; } else


<s> LLVM_DEBUG ( { dbgs ( ) << <STR_LIT> << Block . getNumber ( ) << <STR_LIT> ; ScoreBrackets -> dump (


<s> bool isForceEmitWaitcnt ( ) const { for ( enum InstCounterType T = VM_CNT ; T < NUM_INST_CNTS ; T = ( enum InstCounterType ) ( T + <NUM_LIT> ) ) if ( ForceEmitWaitcnt [ T ] )


<s> T = ( enum InstCounterType ) ( T + <NUM_LIT> ) ) if ( ForceEmitWaitcnt [ T ] )


<s> if ( MBB == Block && MBB -> isSuccessor ( Loop -> getHeader ( )


<s> bool <ISA_LIT>InsertWaitcnts :: isWaitcntStronger ( unsigned LHS , unsigned RHS ) { if ( <STR_LIT> ( IV , LHS ) > <STR_LIT> ( IV , RHS ) ) return false ; if ( <STR_LIT> ( IV , LHS ) > <STR_LIT> ( IV , RHS ) ) return false ; if ( <STR_LIT> ( IV , LHS ) > <STR_LIT> ( IV , RHS ) ) return false ; return true


<s> void print ( ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT>


<s> RegInterval Interval = getRegInterval ( MI , TII , MRI , TRI , OpNo , false ) ; LLVM_DEBUG ( { const MachineOperand & Opnd = MI -> getOperand ( OpNo ) ; assert ( TRI -> isVGPR ( * MRI , Opnd . getReg


<s> void BlockWaitcntBrackets :: setExpScore ( const MachineInstr * MI , const <ISA_LIT>InstrInfo * TII , const <ISA_LIT>RegisterInfo * TRI , const MachineRegisterInfo * MRI , unsigned OpNo , int32_t Val ) { RegInterval Interval = getRegInterval ( MI , TII , MRI , TRI , OpNo , false ) ; LLVM_DEBUG


<s> assert ( Inst . mayLoad ( ) || Inst . mayStore ( ) ) ; if ( TII -> usesVM_CNT ( Inst ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; if ( TII -> usesLGKM_CNT ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; if ( mayAccessLDSThroughFlat ( Inst ) ) ScoreBrackets -> setPendingFlat ( ) ; } } else if ( <ISA_LIT>InstrInfo :: isVMEM ( Inst ) && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; if ( ST -> vmemWriteNeedsExpWaitcnt ( ) && ( Inst . mayStore ( ) || <STR_LIT> ( Inst . getOpcode ( ) ) != - <NUM_LIT> ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMW_GPR_LOCK , Inst ) ; } } else if ( TII -> isSMRD ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS , Inst ) ; } else { switch ( Inst . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SQ_MESSAGE , Inst ) ; break ; case <STR_LIT> : case <STR_LIT> : { int Imm = TII -> getNamedOperand ( Inst , <STR_LIT> :: tgt ) -> getImm ( ) ; if ( Imm >= <NUM_LIT> && Imm <= <NUM_LIT> ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , EXP_PARAM_ACCESS , Inst ) ; else if ( Imm >= <NUM_LIT> && Imm <= <NUM_LIT> ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , EXP_POS_ACCESS , Inst ) ; else ScoreBrackets -> updateByEvent ( TII , TRI , MRI , EXP_GPR_LOCK ,


<s> BlockWaitcntBrackets * ScoreBrackets = BlockWaitcntBracketsMap [ & MBB ] . get ( ) ; if ( ! ScoreBrackets ) { BlockWaitcntBracketsMap [ & MBB ] = llvm :: make_unique < BlockWaitcntBrackets > ( ST ) ; ScoreBrackets = BlockWaitcntBracketsMap [ & MBB ] . get ( ) ; } ScoreBrackets -> setPostOrder ( MBB . getNumber ( ) ) ; MachineLoop * ContainingLoop = MLI -> getLoopFor ( & MBB ) ; if ( ContainingLoop && LoopWaitcntDataMap [ ContainingLoop ] == nullptr ) LoopWaitcntDataMap [ ContainingLoop ] = llvm :: make_unique < LoopWaitcntData > ( ) ; if ( ContainingLoop && ContainingLoop -> getHeader ( ) == & MBB ) { unsigned Count = countNumBottomBlocks ( ContainingLoop ) ; if ( ( std :: count ( BlockWaitcntProcessedSet . begin ( ) , BlockWaitcntProcessedSet . end ( ) , & MBB ) < ( int ) Count ) ) { BlockWaitcntBracketsMap [ & MBB ] -> setRevisitLoop ( true ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << ContainingLoop -> getHeader ( ) -> getNumber ( ) << '\n' ; ) ; } } insertWaitcntInBlock ( MF , MBB ) ; BlockWaitcntProcessedSet . push_back ( & MBB ) ; if ( ContainingLoop && isLoopBottom ( ContainingLoop , & MBB ) && std :: count ( BlockWaitcntProcessedSet . begin ( ) , BlockWaitcntProcessedSet . end ( ) , & MBB ) == <NUM_LIT> ) { MachineBasicBlock * EntryBB = ContainingLoop -> getHeader ( ) ; BlockWaitcntBrackets * EntrySB = BlockWaitcntBracketsMap [ EntryBB ] . get ( ) ; if ( EntrySB && EntrySB -> getRevisitLoop ( ) ) { EntrySB -> setRevisitLoop ( false ) ; J = I ; int32_t PostOrder = EntrySB -> getPostOrder ( ) ; for ( ReversePostOrderTraversal < MachineFunction * > :: rpo_iterator X = RPOT . begin ( ) , Y = RPOT . end ( ) ; X != Y ; ++ X ) { MachineBasicBlock & MBBX = * * X ; if ( MBBX . getNumber ( ) == PostOrder ) { I = X ; break ; } } LoopWaitcntData * WaitcntData = LoopWaitcntDataMap [ ContainingLoop ] . get ( ) ; WaitcntData -> incIterCnt ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << EntryBB -> getNumber ( ) << '\n' ; ) ; continue ; } else { LoopWaitcntData * WaitcntData = LoopWaitcntDataMap [ ContainingLoop ] . get ( ) ; WaitcntData -> resetIterCnt ( ) ; } } J = I ; ++ I ; } SmallVector < MachineBasicBlock * , <NUM_LIT> > EndPgmBlocks ; bool HaveScalarStores = false ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = *


<s> NeedWait = CNT_MASK ( T ) ; setScoreLB ( T , getScoreUB ( T ) ) ; return NeedWait ; } const int32_t LB = getScoreLB ( T ) ; const int32_t UB = getScoreUB ( T ) ; if ( ( UB >= ScoreToWait ) && ( ScoreToWait > LB


<s> } if ( TII -> usesLGKM_CNT ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; if ( mayAccessLDSThroughFlat ( Inst ) ) ScoreBrackets -> setPendingFlat ( ) ; } } else if ( <ISA_LIT>InstrInfo :: isVMEM ( Inst ) && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { if ( ! ST -> hasVscnt ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; else if ( ( Inst . mayLoad ( ) && <STR_LIT> ( Inst . getOpcode ( ) ) == - <NUM_LIT> ) || ( TII -> isMIMG ( Inst ) && ! Inst . mayLoad ( ) && ! Inst . mayStore ( ) ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_READ_ACCESS , Inst ) ; else if ( Inst . mayStore ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_WRITE_ACCESS , Inst ) ; if ( ST -> vmemWriteNeedsExpWaitcnt ( ) && ( Inst . mayStore ( ) || <STR_LIT> ( Inst . getOpcode ( ) ) != - <NUM_LIT> ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMW_GPR_LOCK , Inst ) ; } } else if ( TII -> isSMRD ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS , Inst


<s> void <ISA_LIT>InsertWaitcnts :: updateEventWaitcntAfter ( MachineInstr & Inst , WaitcntBrackets * ScoreBrackets ) { if ( TII -> isDS ( Inst ) && TII -> usesLGKM_CNT ( Inst ) ) { if ( TII -> isAlwaysGDS ( Inst . getOpcode ( ) ) || TII -> hasModifiersSet ( Inst , <STR_LIT> :: gds ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , GDS_ACCESS , Inst ) ; ScoreBrackets -> updateByEvent ( TII , TRI , MRI , GDS_GPR_LOCK , Inst ) ; } else { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; } } else if ( TII -> isFLAT ( Inst ) ) { assert ( Inst . mayLoadOrStore ( ) ) ; if ( TII -> usesVM_CNT ( Inst ) ) { if ( ! ST -> hasVscnt ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; else if ( Inst . mayLoad ( ) && <STR_LIT> ( Inst . getOpcode ( ) ) == - <NUM_LIT> ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_READ_ACCESS , Inst ) ; else ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_WRITE_ACCESS , Inst ) ; } if ( TII -> usesLGKM_CNT ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; if ( mayAccessLDSThroughFlat ( Inst ) ) ScoreBrackets -> setPendingFlat ( ) ; } } else if ( <ISA_LIT>InstrInfo :: isVMEM ( Inst ) && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { if ( ! ST -> hasVscnt ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; else if ( ( Inst . mayLoad ( ) && <STR_LIT> ( Inst . getOpcode ( ) ) == - <NUM_LIT> ) || ( TII -> isMIMG ( Inst ) && ! Inst . mayLoad ( ) && ! Inst . mayStore ( ) ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_READ_ACCESS , Inst ) ; else if ( Inst . mayStore ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_WRITE_ACCESS , Inst ) ; if ( ST -> vmemWriteNeedsExpWaitcnt ( ) && ( Inst . mayStore ( ) || <STR_LIT> ( Inst . getOpcode ( ) ) != - <NUM_LIT> ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMW_GPR_LOCK , Inst ) ; } } else if ( TII -> isSMRD ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS , Inst ) ; } else if ( Inst . isCall ( )


<s> RegInterval WaitcntBrackets :: getRegInterval ( const MachineInstr * MI , const <ISA_LIT>InstrInfo * TII , const MachineRegisterInfo * MRI , const <ISA_LIT>RegisterInfo * TRI , unsigned OpNo ) const { const MachineOperand & Op = MI -> getOperand ( OpNo ) ; if ( ! TRI -> isInAllocatableClass ( Op . getReg ( ) ) ) return { - <NUM_LIT> , - <NUM_LIT> } ; assert ( ! Op . getSubReg ( ) || ! Op . isUndef ( ) ) ; RegInterval Result ; unsigned Reg = TRI -> getEncodingValue ( <STR_LIT> ( Op . getReg ( ) , * ST ) ) ; if ( TRI -> isVectorRegister ( * MRI , Op . getReg ( ) ) ) { assert ( Reg >= Encoding . VGPR0 && Reg <= Encoding . VGPRL ) ; Result . first = Reg - Encoding . VGPR0 ; if ( TRI -> isAGPR ( * MRI , Op . getReg ( ) ) ) Result . first += AGPR_OFFSET ; assert ( Result . first >= <NUM_LIT> && Result . first < SQ_MAX_PGM_VGPRS ) ; } else if ( TRI -> isSGPRReg ( * MRI , Op . getReg ( ) ) ) { assert ( Reg >= Encoding . SGPR0 && Reg < SQ_MAX_PGM_SGPRS ) ; Result . first = Reg - Encoding . SGPR0 + NUM_ALL_VGPRS ; assert ( Result . first >= NUM_ALL_VGPRS && Result . first < SQ_MAX_PGM_SGPRS + NUM_ALL_VGPRS ) ; } else return { - <NUM_LIT> , - <NUM_LIT> } ; const TargetRegisterClass * RC = TII -> getOpRegClass ( * MI , OpNo ) ; unsigned Size = TRI -> getRegSizeInBits ( * RC


<s> assert ( ! Op . getSubReg ( ) || ! Op . isUndef ( ) ) ; RegInterval Result ; unsigned Reg = TRI -> getEncodingValue ( <STR_LIT> ( Op . getReg ( ) , * ST ) ) ; if ( TRI -> isVectorRegister ( * MRI , Op . getReg ( ) ) ) { assert ( Reg >= Encoding . VGPR0 && Reg <= Encoding . VGPRL ) ; Result . first = Reg - Encoding . VGPR0 ; if ( TRI -> isAGPR ( * MRI , Op . getReg ( ) ) ) Result . first += AGPR_OFFSET ; assert ( Result . first >= <NUM_LIT> && Result . first < SQ_MAX_PGM_VGPRS ) ; } else if ( TRI -> isSGPRReg ( * MRI , Op . getReg (


<s> const <STR_LIT> * Info = <STR_LIT> ( Inst . getOpcode ( ) ) ; const <STR_LIT> * BaseInfo = <STR_LIT> ( Info ->


<s> auto inst_counter_types ( ) { return enum_seq ( VM_CNT , NUM_INST_CNTS


<s> if ( ST -> isTgSplitEnabled ( ) ) return false ; if ( MI . memoperands_empty ( ) ) return true ; for ( const MachineMemOperand * Memop : MI . memoperands (


<s> if ( NewUB < ScoreLBs [ T ] ) report_fatal_error ( <STR_LIT> ) ; MergeInfo M ; M . OldLB = ScoreLBs [ T ] ; M . OtherLB = Other . ScoreLBs [ T ] ; M . MyShift = NewUB - ScoreUBs [ T ] ; M . OtherShift = NewUB - Other . ScoreUBs [ T ] ; ScoreUBs [ T ] = NewUB ; StrictDom |= mergeScore ( M , LastFlat [ T ] , Other . LastFlat [ T ] ) ; bool RegStrictDom = false ; for ( int J = <NUM_LIT> ; J <= VgprUB ; J ++ ) { RegStrictDom |= mergeScore ( M , VgprScores [ T ] [ J ] , Other . VgprScores [ T ] [ J ] ) ; } if ( T == VM_CNT ) { for ( int J = <NUM_LIT> ; J <= VgprUB ; J ++ ) { unsigned char NewVmemTypes = VgprVmemTypes [ J ] | Other . VgprVmemTypes [ J ] ; RegStrictDom |= NewVmemTypes != VgprVmemTypes [ J ] ; VgprVmemTypes [ J ] = NewVmemTypes ; } } if ( T == LGKM_CNT ) { for ( int J = <NUM_LIT> ; J <= SgprUB ; J ++ ) { RegStrictDom |= mergeScore ( M , SgprScores [ J ] , Other . SgprScores [ J ] ) ; } } if ( RegStrictDom ) StrictDom = true ; } return StrictDom


<s> Limits . VscntMax = ST -> hasVscnt ( ) ? <NUM_LIT> : <NUM_LIT> ; unsigned NumVGPRsMax = ST -> getAddressableNumVGPRs ( ) ; unsigned NumSGPRsMax = ST -> getAddressableNumSGPRs ( ) ; assert ( NumVGPRsMax <= SQ_MAX_PGM_VGPRS ) ; assert ( NumSGPRsMax <= SQ_MAX_PGM_SGPRS ) ; RegisterEncoding Encoding = { } ; Encoding . VGPR0 = TRI -> getEncodingValue ( <STR_LIT> ) ; Encoding . VGPRL = Encoding . VGPR0 + NumVGPRsMax - <NUM_LIT> ; Encoding . SGPR0 = TRI -> getEncodingValue ( <STR_LIT> ) ; Encoding . SGPRL = Encoding . SGPR0 + NumSGPRsMax - <NUM_LIT> ; TrackedWaitcntSet . clear ( ) ; BlockInfos . clear ( ) ; bool Modified = false ; if ( ! MFI -> isEntryFunction ( ) ) { MachineBasicBlock & EntryBB = MF . front ( ) ; MachineBasicBlock :: iterator I = EntryBB . begin ( ) ; for ( MachineBasicBlock :: iterator E = EntryBB . end ( ) ; I != E && ( I -> isPHI ( ) || I -> isMetaInstruction ( ) ) ; ++ I ) ; BuildMI ( EntryBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; if ( ST -> hasVscnt ( ) ) BuildMI ( EntryBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) ; Modified = true ; } for ( auto * MBB : ReversePostOrderTraversal < MachineFunction * > ( & MF ) ) BlockInfos . insert ( { MBB , BlockInfo ( MBB ) } ) ; std :: unique_ptr < WaitcntBrackets > Brackets ; bool Repeat ; do { Repeat = false ; for ( auto BII = BlockInfos . begin ( ) , BIE = BlockInfos . end ( ) ; BII != BIE ; ++ BII ) { BlockInfo & BI = BII -> second ; if ( ! BI . Dirty ) continue ; if ( BI . Incoming ) { if ( ! Brackets ) Brackets = std :: make_unique < WaitcntBrackets > ( * BI . Incoming ) ; else * Brackets = * BI . Incoming ; } else { if ( ! Brackets ) Brackets = std :: make_unique < WaitcntBrackets > ( ST , Limits , Encoding ) ; else * Brackets = WaitcntBrackets ( ST , Limits , Encoding ) ; } Modified |= insertWaitcntInBlock ( MF , * BI . MBB , * Brackets ) ; BI . Dirty = false ; if ( Brackets -> hasPending ( ) ) { BlockInfo * MoveBracketsToSucc = nullptr ; for ( MachineBasicBlock * Succ : BI . MBB -> successors ( ) ) { auto SuccBII = BlockInfos . find ( Succ ) ; BlockInfo & SuccBI =


<s> void WaitcntBrackets :: setExpScore ( const MachineInstr * MI , const <ISA_LIT>InstrInfo * TII , const <ISA_LIT>RegisterInfo * TRI , const MachineRegisterInfo * MRI , unsigned OpNo , unsigned Val ) { RegInterval Interval = getRegInterval ( MI , TII ,


<s> void WaitcntBrackets :: setExpScore ( const MachineInstr * MI , const <ISA_LIT>InstrInfo * TII , const <ISA_LIT>RegisterInfo * TRI , const MachineRegisterInfo * MRI , unsigned OpNo


<s> void WaitcntBrackets :: simplifyWaitcnt ( InstCounterType T ,


<s> if ( TII -> isDS ( Inst ) && TII -> usesLGKM_CNT ( Inst ) ) { if ( TII -> isAlwaysGDS ( Inst . getOpcode ( ) ) || TII -> hasModifiersSet ( Inst , <STR_LIT> :: gds ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , GDS_ACCESS , Inst ) ; ScoreBrackets -> updateByEvent ( TII , TRI , MRI , GDS_GPR_LOCK , Inst ) ; } else { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; } } else if ( TII -> isFLAT ( Inst ) ) { assert ( Inst . mayLoadOrStore ( ) ) ; int FlatASCount = <NUM_LIT> ; if ( mayAccessVMEMThroughFlat ( Inst ) ) { ++ FlatASCount ; if ( ! ST -> hasVscnt ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; else if ( Inst . mayLoad ( ) && ! <ISA_LIT>InstrInfo :: isAtomicNoRet ( Inst ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_READ_ACCESS , Inst ) ; else ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_WRITE_ACCESS , Inst ) ; } if ( mayAccessLDSThroughFlat ( Inst ) ) { ++ FlatASCount ; ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; } assert ( FlatASCount ) ; if ( FlatASCount > <NUM_LIT> ) ScoreBrackets -> setPendingFlat


<s> if ( FlatASCount > <NUM_LIT> ) ScoreBrackets -> setPendingFlat ( ) ; } else if ( <ISA_LIT>InstrInfo :: isVMEM ( Inst ) && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { if ( ! ST -> hasVscnt ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; else if ( ( Inst . mayLoad ( ) && <STR_LIT> ( Inst . getOpcode ( ) ) == - <NUM_LIT> ) || ( TII -> isMIMG ( Inst ) && ! Inst . mayLoad ( ) && ! Inst . mayStore ( ) ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_READ_ACCESS , Inst ) ; else if ( Inst . mayStore ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_WRITE_ACCESS , Inst ) ; if ( ST -> vmemWriteNeedsExpWaitcnt ( ) && ( Inst . mayStore ( ) || <STR_LIT> ( Inst . getOpcode ( ) ) != - <NUM_LIT> ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMW_GPR_LOCK , Inst ) ; } } else if ( TII -> isSMRD ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS , Inst ) ; } else if ( Inst . isCall ( ) ) { if ( callWaitsOnFunctionReturn ( Inst ) ) { ScoreBrackets -> applyWaitcnt ( <STR_LIT> :: allZero ( ST -> hasVscnt ( ) ) ) ; } else { ScoreBrackets -> applyWaitcnt ( <STR_LIT> ( ) ) ; } } else if ( <ISA_LIT>InstrInfo :: isEXP


<s> if ( Count >= UB ) return ; if ( Count != <NUM_LIT> ) { if ( counterOutOfOrder ( T ) ) return ; setScoreLB ( T , std :: max ( getScoreLB ( T ) , UB - Count ) ) ; } else { setScoreLB ( T


<s> static bool callWaitsOnFunctionEntry ( const MachineInstr &


<s> static bool callWaitsOnFunctionReturn ( const


<s> if ( T == LGKM_CNT && hasPendingEvent ( SMEM_ACCESS )


<s> if ( T == LGKM_CNT && hasPendingEvent ( SMEM_ACCESS ) ) return true ; return hasMixedPendingEvents ( T


<s> const unsigned UB = getScoreUB ( T ) ; if ( ( UB >= ScoreToWait ) && ( ScoreToWait > LB ) ) { if ( ( T == VM_CNT || T == LGKM_CNT ) && hasPendingFlat ( ) && ! ST -> hasFlatLgkmVMemCountInOrder ( ) ) { addWait ( Wait , T , <NUM_LIT>


<s> if ( WaitEventMaskForInst [ VS_CNT ] & ( <NUM_LIT> << E ) ) return VS_CNT ; assert ( WaitEventMaskForInst [ EXP_CNT ] & ( <NUM_LIT> << E ) ) ; return


<s> if ( WaitEventMaskForInst [ LGKM_CNT ] & ( <NUM_LIT> << E ) ) return LGKM_CNT ; if ( WaitEventMaskForInst [ VS_CNT ] & ( <NUM_LIT> << E ) ) return VS_CNT ; assert ( WaitEventMaskForInst [ EXP_CNT ] &


<s> AU . addRequired < MachinePostDominatorTree > ( ) ; MachineFunctionPass :: getAnalysisUsage


<s> if ( TRI -> isVGPR ( * MRI , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . VGPR0 && Reg <= RegisterEncoding . VGPRL ) ; Result . first = Reg - RegisterEncoding . VGPR0 ; assert ( Result . first >= <NUM_LIT> && Result . first < SQ_MAX_PGM_VGPRS ) ; } else if ( TRI -> isSGPRReg ( * MRI , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . SGPR0 && Reg < SQ_MAX_PGM_SGPRS ) ; Result . first = Reg -

RegisterEncoding . SGPR0 + NUM_ALL_VGPRS ;


<s> assert ( ! Op . getSubReg ( ) || ! Op . isUndef ( ) ) ; RegInterval Result ; unsigned Reg = TRI -> getEncodingValue ( <STR_LIT> ( Op . getReg ( ) , * ST ) ) ; if ( TRI -> isVGPR ( * MRI , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . VGPR0 && Reg <= RegisterEncoding . VGPRL ) ; Result . first = Reg - RegisterEncoding . VGPR0 ; assert ( Result . first >= <NUM_LIT> && Result . first < SQ_MAX_PGM_VGPRS ) ; } else if ( TRI -> isSGPRReg ( * MRI , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . SGPR0 && Reg < SQ_MAX_PGM_SGPRS ) ; Result . first = Reg - RegisterEncoding .


<s> return VgprScores [ T ] [ GprNo ] ; } assert ( T == LGKM_CNT ) ; return SgprScores [ GprNo


<s> unsigned getScoreLB ( InstCounterType T ) const


<s> assert ( T < NUM_INST_CNTS ) ; return ScoreUBs [


<s> const <STR_LIT> * Info = <STR_LIT> ( Inst . getOpcode ( ) ) ; return <STR_LIT> ( Info -> BaseOpcode ) -> Sampler ? VMEM_SAMPLER


<s> VmemType getVmemType ( const MachineInstr & Inst ) { assert ( <ISA_LIT>InstrInfo :: isVMEM ( Inst ) ) ; if ( ! <ISA_LIT>InstrInfo :: isMIMG ( Inst ) ) return VMEM_NOSAMPLER ; const <STR_LIT> * Info = <STR_LIT> ( Inst . getOpcode ( ) ) ; return <STR_LIT> ( Info -> BaseOpcode ) -> Sampler ? VMEM_SAMPLER


<s> bool hasOtherPendingVmemTypes ( int GprNo , VmemType


<s> return VgprVmemTypes [ GprNo ] & ~ ( <NUM_LIT> << V


<s> bool <ISA_LIT>InsertWaitcnts :: mayAccessLDSThroughFlat ( const MachineInstr & MI ) const { assert ( TII -> isFLAT ( MI ) ) ; if ( ! TII -> usesLGKM_CNT ( MI ) )


<s> bool <ISA_LIT>InsertWaitcnts :: mayAccessVMEMThroughFlat ( const MachineInstr & MI ) const { assert ( TII -> isFLAT ( MI ) ) ; assert ( TII -> usesVM_CNT ( MI ) ) ; if ( MI . memoperands_empty ( ) ) return true ; for ( const MachineMemOperand * Memop : MI . memoperands ( ) ) { unsigned AS = Memop -> getAddrSpace (


<s> for ( const MachineMemOperand * Memop : MI . memoperands ( ) ) { unsigned AS = Memop -> getAddrSpace ( ) ; assert ( AS != <ISA_LIT>AS :: REGION_ADDRESS ) ; if ( AS != <ISA_LIT>AS :: LOCAL_ADDRESS ) return true ; } return false


<s> PendingEvents |= OtherEvents ; const unsigned MyPending = ScoreUBs [ T ] - ScoreLBs [ T ] ; const unsigned OtherPending = Other . ScoreUBs [ T ] - Other . ScoreLBs [ T ] ; const unsigned NewUB = ScoreLBs [ T ] + std :: max ( MyPending , OtherPending ) ; if ( NewUB < ScoreLBs [ T ] ) report_fatal_error ( <STR_LIT> ) ; MergeInfo M ; M . OldLB = ScoreLBs [ T ] ; M . OtherLB = Other . ScoreLBs [ T ] ; M . MyShift = NewUB - ScoreUBs [ T ] ; M . OtherShift = NewUB - Other . ScoreUBs [ T ] ; ScoreUBs [ T ] = NewUB ; StrictDom |= mergeScore ( M , LastFlat [ T ] , Other . LastFlat [ T ] ) ; bool RegStrictDom =


<s> for ( auto T : inst_counter_types ( ) ) { unsigned LB = getScoreLB ( T ) ; unsigned UB = getScoreUB ( T ) ; switch ( T ) { case VM_CNT : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; case LGKM_CNT : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; case EXP_CNT : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; case VS_CNT : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; default : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; } if ( LB < UB ) { for ( int J = <NUM_LIT> ; J <= VgprUB ; J ++ ) { unsigned RegScore = getRegScore ( J , T ) ; if ( RegScore <= LB ) continue ; unsigned RelScore = RegScore - LB - <NUM_LIT> ; if ( J < SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS ) { OS << RelScore << <STR_LIT> << J << <STR_LIT> ; } else


<s> TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; IV = <STR_LIT> ( ST -> getCPU ( ) ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; PDT = & getAnalysis < MachinePostDominatorTree > ( ) ; ForceEmitZeroWaitcnts = ForceEmitZeroFlag ; for ( auto T : inst_counter_types ( ) ) ForceEmitWaitcnt [ T ] = false ; HardwareLimits . VmcntMax = <STR_LIT> ( IV ) ; HardwareLimits . ExpcntMax = <STR_LIT> ( IV ) ; HardwareLimits . LgkmcntMax = <STR_LIT> ( IV ) ; HardwareLimits . VscntMax = ST -> hasVscnt ( ) ? <NUM_LIT> : <NUM_LIT> ; unsigned NumVGPRsMax = ST -> getAddressableNumVGPRs ( ) ; unsigned NumSGPRsMax = ST -> getAddressableNumSGPRs ( ) ; assert ( NumVGPRsMax <= SQ_MAX_PGM_VGPRS ) ; assert ( NumSGPRsMax <= SQ_MAX_PGM_SGPRS ) ; RegisterEncoding . VGPR0 = TRI -> getEncodingValue ( <STR_LIT> ) ; RegisterEncoding . VGPRL = RegisterEncoding . VGPR0 + NumVGPRsMax - <NUM_LIT> ; RegisterEncoding . SGPR0 = TRI -> getEncodingValue ( <STR_LIT> ) ; RegisterEncoding . SGPRL = RegisterEncoding . SGPR0 + NumSGPRsMax - <NUM_LIT> ; TrackedWaitcntSet . clear ( ) ; BlockInfos . clear ( ) ; for ( auto * MBB : ReversePostOrderTraversal < MachineFunction * > ( & MF ) ) BlockInfos . insert ( { MBB , BlockInfo ( MBB ) } ) ; std :: unique_ptr < WaitcntBrackets > Brackets ; bool Modified = false ; bool Repeat ; do { Repeat = false ; for ( auto BII = BlockInfos . begin ( ) , BIE = BlockInfos . end ( ) ; BII != BIE ; ++ BII ) { BlockInfo & BI = BII -> second ; if ( ! BI . Dirty ) continue ; if ( BI . Incoming ) { if ( ! Brackets ) Brackets = std :: make_unique < WaitcntBrackets > ( * BI . Incoming ) ; else * Brackets = * BI . Incoming ; } else { if ( ! Brackets ) Brackets = std :: make_unique < WaitcntBrackets > ( ST ) ; else * Brackets = WaitcntBrackets ( ST ) ; } Modified |= insertWaitcntInBlock ( MF , * BI . MBB , * Brackets ) ; BI . Dirty = false ; if ( Brackets -> hasPending ( ) ) { BlockInfo * MoveBracketsToSucc = nullptr ; for ( MachineBasicBlock * Succ : BI . MBB -> successors ( ) ) { auto SuccBII = BlockInfos . find ( Succ ) ; BlockInfo & SuccBI = SuccBII -> second ; if ( ! SuccBI . Incoming ) { SuccBI . Dirty = true ; if ( SuccBII <= BII ) Repeat = true ; if ( ! MoveBracketsToSucc ) { MoveBracketsToSucc = & SuccBI ; } else { SuccBI . Incoming = std :: make_unique < WaitcntBrackets > ( * Brackets ) ; } } else if ( SuccBI . Incoming -> merge ( * Brackets ) ) { SuccBI . Dirty


<s> assert ( T < NUM_INST_CNTS ) ; ScoreLBs [ T ] =


<s> void setScoreLB ( InstCounterType T , unsigned Val ) { assert ( T < NUM_INST_CNTS )


<s> unsigned UB = ScoreUBs [ T ] - getWaitCountMax ( EXP_CNT ) ; if ( ScoreLBs [ T ] < UB && UB < ScoreUBs [ T ] ) ScoreLBs [


<s> if ( ! ST -> hasVscnt ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; else if ( ( Inst . mayLoad ( ) && <STR_LIT> ( Inst . getOpcode ( ) ) == - <NUM_LIT> ) || ( TII -> isMIMG ( Inst ) && ! Inst . mayLoad ( ) && ! Inst . mayStore ( ) ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_READ_ACCESS , Inst ) ; else if ( Inst . mayStore ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_WRITE_ACCESS , Inst ) ; if ( ST -> vmemWriteNeedsExpWaitcnt ( ) && ( Inst . mayStore ( ) || <STR_LIT> ( Inst . getOpcode ( ) ) != - <NUM_LIT> ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMW_GPR_LOCK , Inst ) ; } } else if ( TII -> isSMRD ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS , Inst ) ; } else if ( Inst . isCall ( ) ) { if ( callWaitsOnFunctionReturn ( Inst ) ) { ScoreBrackets -> applyWaitcnt ( <STR_LIT> :: allZero ( ST -> hasVscnt ( ) ) ) ; } else { ScoreBrackets -> applyWaitcnt ( <STR_LIT> ( ) ) ; } } else { switch ( Inst . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT>


<s> BlockWaitcntBrackets ( ) : WaitAtBeginning ( false ) , RevisitLoop ( false ) , ValidLoop ( false ) , MixedExpTypes ( false ) , LoopRegion ( NULL ) , PostOrder ( <NUM_LIT> ) , Waitcnt ( NULL ) , VgprUB ( <NUM_LIT> ) , SgprUB ( <NUM_LIT> ) { for ( enum InstCounterType T = VM_CNT


<s> for ( enum InstCounterType T = VM_CNT ; T < NUM_INST_CNTS ; T = ( enum InstCounterType ) ( T + <NUM_LIT> ) ) { memset ( VgprScores [ T ] , <NUM_LIT> , sizeof ( VgprScores [


<s> static bool mayWriteLDSThroughDMA ( const MachineInstr & MI ) { return <ISA_LIT>InstrInfo :: isVALU ( MI ) && ( <ISA_LIT>InstrInfo :: isMUBUF ( MI ) || <ISA_LIT>InstrInfo ::

isFLAT ( MI ) ) && MI . getOpcode ( ) != <STR_LIT> ;


<s> setExpScore ( & Inst , TII , TRI , MRI , AddrOpIdx , CurrScore ) ; } if ( Inst . mayStore ( ) ) { if ( <STR_LIT> ( Inst . getOpcode ( ) , <STR_LIT> :: data0 ) != - <NUM_LIT> ) { setExpScore ( & Inst , TII , TRI , MRI , <STR_LIT> ( Inst . getOpcode ( ) , <STR_LIT> :: data0 ) , CurrScore ) ; } if ( <STR_LIT> ( Inst . getOpcode ( ) , <STR_LIT> :: data1 ) != - <NUM_LIT> ) { setExpScore ( & Inst , TII , TRI , MRI , <STR_LIT> ( Inst . getOpcode ( ) , <STR_LIT> :: data1 ) , CurrScore ) ; } } else if ( <ISA_LIT>InstrInfo :: isAtomicRet ( Inst ) && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { for ( unsigned I = <NUM_LIT> , E = Inst . getNumOperands ( ) ; I != E ; ++ I ) { const MachineOperand & Op = Inst . getOperand ( I ) ; if ( Op . isReg ( ) && ! Op . isDef ( ) && TRI -> isVectorRegister ( * MRI , Op . getReg ( ) ) ) { setExpScore ( & Inst , TII , TRI , MRI , I , CurrScore ) ; } } } } else if ( TII -> isFLAT ( Inst ) ) { if ( Inst . mayStore ( ) ) { setExpScore ( & Inst , TII , TRI , MRI , <STR_LIT> ( Inst . getOpcode ( ) , <STR_LIT> :: data ) , CurrScore ) ; } else if ( <ISA_LIT>InstrInfo :: isAtomicRet ( Inst ) ) { setExpScore ( & Inst , TII , TRI , MRI , <STR_LIT> ( Inst . getOpcode ( ) , <STR_LIT> :: data ) , CurrScore ) ; } } else if ( TII -> isMIMG ( Inst ) ) { if ( Inst . mayStore ( ) ) { setExpScore ( & Inst , TII , TRI , MRI , <NUM_LIT> , CurrScore ) ; } else if ( <ISA_LIT>InstrInfo :: isAtomicRet ( Inst ) ) { setExpScore ( & Inst , TII , TRI , MRI , <STR_LIT> ( Inst . getOpcode ( ) , <STR_LIT> :: data ) , CurrScore ) ; } } else if ( TII -> isMTBUF ( Inst ) ) { if ( Inst . mayStore ( ) ) { setExpScore ( & Inst , TII , TRI , MRI , <NUM_LIT> , CurrScore ) ; } } else if ( TII -> isMUBUF ( Inst ) ) { if ( Inst . mayStore (


<s> if ( ( T == VM_CNT || T == LGKM_CNT ) && hasPendingFlat ( ) && ! ST -> hasFlatLgkmVMemCountInOrder ( ) ) { addWait ( Wait , T , <NUM_LIT> ) ; } else if ( counterOutOfOrder ( T ) ) { addWait ( Wait , T ,


<s> const uint32_t UB = getScoreUB ( T ) ; if ( ( UB >= ScoreToWait ) && ( ScoreToWait > LB ) ) { if ( ( T == VM_CNT || T == LGKM_CNT ) && hasPendingFlat ( ) && ! ST -> hasFlatLgkmVMemCountInOrder ( ) ) { addWait ( Wait , T , <NUM_LIT> ) ; } else if ( counterOutOfOrder ( T ) ) { addWait ( Wait , T , <NUM_LIT> ) ; } else


<s> const MachineRegisterInfo & MRIA = * MRI ; unsigned Reg = TRI -> getEncodingValue ( Op . getReg ( ) ) ; if ( TRI -> isVGPR ( MRIA , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . VGPR0 && Reg <= RegisterEncoding . VGPRL ) ; Result . first = Reg - RegisterEncoding . VGPR0 ; assert ( Result . first >= <NUM_LIT> && Result . first < SQ_MAX_PGM_VGPRS ) ; } else if ( TRI -> isSGPRReg ( MRIA , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . SGPR0 &&


<s> else if ( Inst . mayLoad ( ) && <STR_LIT> ( Inst . getOpcode ( ) ) == - <NUM_LIT> ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_READ_ACCESS , Inst ) ; else ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_WRITE_ACCESS , Inst ) ; } if ( TII -> usesLGKM_CNT ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; if ( mayAccessLDSThroughFlat ( Inst ) ) ScoreBrackets -> setPendingFlat ( ) ; } } else if ( <ISA_LIT>InstrInfo :: isVMEM ( Inst ) && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { if ( ! ST -> hasVscnt ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; else if ( ( Inst . mayLoad ( ) && <STR_LIT> ( Inst . getOpcode ( ) ) == - <NUM_LIT> ) || ( TII -> isMIMG ( Inst ) && ! Inst . mayLoad ( ) && ! Inst . mayStore ( ) ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_READ_ACCESS , Inst ) ; else if ( Inst . mayStore ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_WRITE_ACCESS , Inst ) ; if ( ST -> vmemWriteNeedsExpWaitcnt ( ) && ( Inst . mayStore ( ) || <STR_LIT> ( Inst . getOpcode ( ) ) != - <NUM_LIT> ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMW_GPR_LOCK , Inst ) ; } } else if ( TII -> isSMRD ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS , Inst ) ; } else if ( Inst . isCall ( ) ) { if ( callWaitsOnFunctionReturn ( Inst ) ) { ScoreBrackets -> applyWaitcnt ( <STR_LIT> :: allZero ( ST -> hasVscnt ( )


<s> if ( ! TRI -> isInAllocatableClass ( Op . getReg ( ) ) || TRI -> isAGPR ( * MRI , Op . getReg ( ) ) ) return { - <NUM_LIT> , - <NUM_LIT> } ; assert ( ! Op . getSubReg ( ) || ! Op . isUndef ( ) ) ; RegInterval Result ; unsigned Reg = TRI -> getEncodingValue ( Op . getReg ( ) ) ; if ( TRI -> isVGPR ( * MRI , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . VGPR0 && Reg <= RegisterEncoding . VGPRL ) ; Result . first = Reg - RegisterEncoding . VGPR0 ; assert ( Result . first >= <NUM_LIT> && Result


<s> RegInterval WaitcntBrackets :: getRegInterval ( const MachineInstr * MI , const <ISA_LIT>InstrInfo * TII , const MachineRegisterInfo * MRI , const <ISA_LIT>RegisterInfo * TRI , unsigned OpNo ) const { const MachineOperand & Op = MI -> getOperand ( OpNo ) ; assert ( Op . isReg ( ) ) ; if ( ! TRI -> isInAllocatableClass ( Op . getReg ( ) ) || TRI -> isAGPR ( * MRI , Op . getReg ( ) ) ) return { - <NUM_LIT> , - <NUM_LIT> } ; assert ( ! Op . getSubReg ( ) || ! Op . isUndef ( ) ) ; RegInterval


<s> if ( mayAccessLDSThroughFlat ( Inst ) ) { ++ FlatASCount ; ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; } assert ( FlatASCount ) ; if ( FlatASCount > <NUM_LIT> ) ScoreBrackets -> setPendingFlat ( ) ; } else if ( <ISA_LIT>InstrInfo :: isVMEM ( Inst ) && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { if ( ! ST -> hasVscnt ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; else if ( ( Inst . mayLoad ( ) && <STR_LIT> ( Inst . getOpcode ( ) ) == - <NUM_LIT> ) || ( TII -> isMIMG ( Inst ) && ! Inst . mayLoad ( ) && ! Inst . mayStore ( ) ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_READ_ACCESS , Inst ) ; else if ( Inst . mayStore ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_WRITE_ACCESS , Inst ) ; if ( ST -> vmemWriteNeedsExpWaitcnt ( ) && ( Inst . mayStore ( ) || <STR_LIT> ( Inst . getOpcode ( ) ) != - <NUM_LIT> ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMW_GPR_LOCK , Inst ) ; } } else if ( TII -> isSMRD ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS


<s> else if ( Inst . mayLoad ( ) && <STR_LIT> ( Inst . getOpcode ( ) ) == - <NUM_LIT> ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_READ_ACCESS , Inst ) ; else ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_WRITE_ACCESS , Inst ) ; } if ( TII -> usesLGKM_CNT ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; if ( mayAccessLDSThroughFlat ( Inst ) ) ScoreBrackets -> setPendingFlat ( ) ; } } else if ( <ISA_LIT>InstrInfo :: isVMEM ( Inst ) && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { if ( ! ST -> hasVscnt ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; else if ( ( Inst . mayLoad ( ) && <STR_LIT> ( Inst . getOpcode ( ) ) == - <NUM_LIT> ) || ( TII -> isMIMG ( Inst ) && ! Inst . mayLoad ( ) && ! Inst . mayStore ( ) ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_READ_ACCESS , Inst ) ; else if ( Inst . mayStore ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_WRITE_ACCESS , Inst ) ; if ( ST -> vmemWriteNeedsExpWaitcnt ( ) && ( Inst . mayStore ( ) || <STR_LIT> ( Inst . getOpcode ( ) ) != - <NUM_LIT> ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMW_GPR_LOCK , Inst ) ; } } else if ( TII -> isSMRD ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS ,


<s> II -> eraseFromParent ( ) ; Modified = true ; } } else { assert ( II -> getOpcode ( ) == <STR_LIT> ) ; assert ( II -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) ; if ( ! TrackedWaitcntSet . count ( & * II ) ) { unsigned OldVSCnt = TII -> getNamedOperand ( * II , <STR_LIT> :: simm16 ) -> getImm ( ) ; Wait . VsCnt = std :: min ( Wait . VsCnt , OldVSCnt ) ; } if ( ! WaitcntVsCntInstr ) { WaitcntVsCntInstr = & * II ; } else { II -> eraseFromParent ( ) ; Modified = true ; } } } if ( WaitcntInstr ) { if ( Wait . hasWaitExceptVsCnt ( ) ) { unsigned NewEnc = <STR_LIT> ( IV , Wait ) ; unsigned OldEnc = WaitcntInstr -> getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( OldEnc != NewEnc ) { WaitcntInstr -> getOperand ( <NUM_LIT> ) . setImm ( NewEnc ) ; Modified = true ; } ScoreBrackets . applyWaitcnt ( Wait ) ; Wait . VmCnt = ~ <NUM_LIT> ; Wait . LgkmCnt = ~ <NUM_LIT> ; Wait . ExpCnt = ~ <NUM_LIT> ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << MI << <STR_LIT> << * WaitcntInstr << '\n' ) ; } else { WaitcntInstr -> eraseFromParent ( ) ; Modified = true ; } } if ( WaitcntVsCntInstr ) { if ( Wait . hasWaitVsCnt ( ) ) { assert ( ST -> hasVscnt ( ) ) ; unsigned OldVSCnt = TII -> getNamedOperand ( * WaitcntVsCntInstr


<s> RegInterval Result ; unsigned Reg = TRI -> getEncodingValue ( <STR_LIT> ( Op . getReg ( ) , * ST ) ) ; if ( TRI -> isVectorRegister ( * MRI , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . VGPR0 && Reg <= RegisterEncoding . VGPRL ) ; Result . first = Reg - RegisterEncoding . VGPR0 ; if ( TRI -> isAGPR ( * MRI , Op . getReg ( ) ) ) Result . first += AGPR_OFFSET ; assert ( Result . first >= <NUM_LIT> && Result . first < SQ_MAX_PGM_VGPRS ) ; } else if ( TRI -> isSGPRReg ( * MRI , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . SGPR0 && Reg < SQ_MAX_PGM_SGPRS ) ; Result . first = Reg - RegisterEncoding . SGPR0 + NUM_ALL_VGPRS ; assert ( Result . first >= NUM_ALL_VGPRS && Result . first < SQ_MAX_PGM_SGPRS + NUM_ALL_VGPRS ) ; } else return { - <NUM_LIT>


<s> if ( TRI -> isVectorRegister ( * MRI , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . VGPR0 && Reg <= RegisterEncoding . VGPRL ) ; Result . first = Reg - RegisterEncoding . VGPR0 ; if ( TRI -> isAGPR ( * MRI , Op . getReg ( ) ) ) Result . first += AGPR_OFFSET ; assert ( Result . first >= <NUM_LIT> && Result . first < SQ_MAX_PGM_VGPRS ) ; } else if ( TRI -> isSGPRReg ( * MRI , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . SGPR0


<s> BlockWaitcntProcessedSet . clear ( ) ; ReversePostOrderTraversal < MachineFunction * > RPOT ( & MF ) ; bool Modified = false ; for ( ReversePostOrderTraversal < MachineFunction * > :: rpo_iterator I = RPOT . begin ( ) , E = RPOT . end ( ) , J = RPOT . begin ( ) ; I != E ; ) { MachineBasicBlock & MBB = * * I ; BlockVisitedSet . insert ( & MBB ) ; BlockWaitcntBrackets * ScoreBrackets = BlockWaitcntBracketsMap [ & MBB ] . get ( ) ; if ( ! ScoreBrackets ) { BlockWaitcntBracketsMap [ & MBB ] = llvm :: make_unique < BlockWaitcntBrackets > ( ST ) ; ScoreBrackets = BlockWaitcntBracketsMap [ & MBB ] . get ( ) ; } ScoreBrackets -> setPostOrder ( MBB . getNumber ( ) ) ; MachineLoop * ContainingLoop = MLI -> getLoopFor ( & MBB ) ; if ( ContainingLoop && LoopWaitcntDataMap [ ContainingLoop ] == nullptr ) LoopWaitcntDataMap [ ContainingLoop ] = llvm :: make_unique < LoopWaitcntData > ( ) ; if ( ContainingLoop && ContainingLoop -> getHeader ( ) == & MBB ) { unsigned Count = countNumBottomBlocks ( ContainingLoop ) ; if ( ( std :: count ( BlockWaitcntProcessedSet . begin ( ) , BlockWaitcntProcessedSet . end ( ) , & MBB ) < Count ) ) { BlockWaitcntBracketsMap [ & MBB ] -> setRevisitLoop ( true ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << ContainingLoop -> getHeader ( ) -> getNumber ( ) << '\n' ; ) ; } } insertWaitcntInBlock ( MF , MBB ) ; BlockWaitcntProcessedSet . push_back ( & MBB ) ; if ( ContainingLoop && isLoopBottom ( ContainingLoop , & MBB ) && std :: count ( BlockWaitcntProcessedSet . begin ( ) , BlockWaitcntProcessedSet . end ( ) , & MBB ) == <NUM_LIT> ) { MachineBasicBlock * EntryBB = ContainingLoop -> getHeader ( ) ; BlockWaitcntBrackets * EntrySB = BlockWaitcntBracketsMap [ EntryBB ] . get ( ) ; if ( EntrySB && EntrySB -> getRevisitLoop ( ) ) { EntrySB -> setRevisitLoop ( false ) ; J = I ; int32_t PostOrder = EntrySB -> getPostOrder ( ) ; for ( ReversePostOrderTraversal < MachineFunction * > :: rpo_iterator X = RPOT . begin ( ) , Y = RPOT . end ( ) ; X != Y ; ++ X ) { MachineBasicBlock & MBBX = * * X ; if ( MBBX . getNumber ( ) == PostOrder ) { I = X ; break ; } } LoopWaitcntData * WaitcntData = LoopWaitcntDataMap [ ContainingLoop ] . get ( ) ; WaitcntData -> incIterCnt ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << EntryBB -> getNumber ( ) << '\n' ; ) ; continue ; } else { LoopWaitcntData * WaitcntData = LoopWaitcntDataMap [ ContainingLoop ] . get ( ) ; WaitcntData -> resetIterCnt (


<s> unsigned Reg = TRI -> getEncodingValue ( Op . getReg ( ) ) ; if ( TRI -> isVGPR ( MRIA , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . VGPR0 && Reg <= RegisterEncoding . VGPRL ) ; Result . first = Reg - RegisterEncoding . VGPR0 ; assert ( Result . first >= <NUM_LIT> && Result . first < SQ_MAX_PGM_VGPRS ) ; } else if ( TRI -> isSGPRReg ( MRIA , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . SGPR0 && Reg < SQ_MAX_PGM_SGPRS ) ; Result . first = Reg - RegisterEncoding . SGPR0 + NUM_ALL_VGPRS ; assert ( Result . first >= NUM_ALL_VGPRS && Result . first < SQ_MAX_PGM_SGPRS + NUM_ALL_VGPRS ) ; } else return { - <NUM_LIT> , - <NUM_LIT> } ; const MachineInstr & MIA = * MI ; const TargetRegisterClass * RC = TII -> getOpRegClass ( MIA , OpNo ) ; unsigned Size = RC ->