<s> void GCNScheduleDAGMILive :: computeBlockPressure ( const MachineBasicBlock * MBB ) { GCNDownwardRPTracker RPTracker ( * LIS ) ; const MachineBasicBlock * OnlySucc = nullptr ; if ( MBB -> succ_size ( ) == <NUM_LIT> && ! ( * MBB -> succ_begin ( ) ) -> empty ( ) ) { SlotIndexes * Ind = LIS -> getSlotIndexes ( ) ; if ( Ind -> getMBBStartIdx ( MBB ) < Ind -> getMBBStartIdx ( * MBB -> succ_begin ( ) ) ) OnlySucc = * MBB -> succ_begin ( ) ; } size_t CurRegion = RegionIdx ; for ( size_t E = Regions . size ( ) ; CurRegion != E ; ++ CurRegion ) if ( Regions [ CurRegion ] . first -> getParent ( ) != MBB ) break ; -- CurRegion ; auto I = MBB -> begin ( ) ; auto LiveInIt = MBBLiveIns . find ( MBB ) ; if ( LiveInIt != MBBLiveIns . end ( ) ) { auto LiveIn = std :: move ( LiveInIt -> second ) ; RPTracker . reset ( * MBB -> begin ( ) , & LiveIn ) ; MBBLiveIns . erase ( LiveInIt ) ; } else { auto & Rgn = Regions [ CurRegion ] ; I = Rgn . first ; auto * NonDbgMI = & * skipDebugInstructionsForward ( Rgn . first , Rgn . second ) ; auto LRS = BBLiveInMap . lookup ( NonDbgMI ) ; assert ( isEqual ( getLiveRegsBefore ( * NonDbgMI , * LIS ) , LRS ) ) ; RPTracker . reset ( * I , & LRS ) ; } for ( ; ; ) { I = RPTracker . getNext ( ) ; if ( Regions [ CurRegion ] . first == I ) { LiveIns [ CurRegion ] = RPTracker . getLiveRegs ( ) ; RPTracker . clearMaxPressure ( ) ; } if ( Regions [ CurRegion ] . second == I ) { Pressure [ CurRegion ] = RPTracker . moveMaxPressure ( ) ; if ( CurRegion -- == RegionIdx ) break ; } RPTracker . advanceToNext ( ) ; RPTracker . advanceBeforeNext ( ) ; } if ( OnlySucc ) { if ( I != MBB -> end ( ) ) { RPTracker . advanceToNext ( ) ; RPTracker . advance ( MBB -> end ( ) ) ; } RPTracker . reset ( * OnlySucc -> begin ( ) , & RPTracker . getLiveRegs ( ) ) ; RPTracker . advanceBeforeNext ( ) ; MBBLiveIns [ OnlySucc ] =


<s> } } if ( Stage == UnclusteredReschedule ) SavedMutations . swap ( Mutations ) ; for ( auto Region : Regions ) { if ( Stage == UnclusteredReschedule && ! RescheduleRegions [ RegionIdx ] ) { ++ RegionIdx ; continue ; } RegionBegin = Region . first ; RegionEnd = Region . second ; if ( RegionBegin -> getParent ( ) != MBB ) { if ( MBB ) finishBlock ( ) ; MBB = RegionBegin -> getParent ( ) ; startBlock ( MBB ) ; if ( Stage == InitialSchedule ) computeBlockPressure ( MBB ) ; } unsigned NumRegionInstrs = std :: distance ( begin ( ) , end ( ) ) ; enterRegion ( MBB , begin ( ) , end ( ) , NumRegionInstrs ) ; if ( begin ( ) == end ( ) || begin ( ) == std :: prev ( end ( ) ) ) { exitRegion ( ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT>


<s> MachineBasicBlock * MBB = nullptr ; if ( Stage > InitialSchedule ) { if ( ! LIS ) break ; if ( Stage == UnclusteredReschedule ) { if ( RescheduleRegions . none ( ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } if ( Stage == ClusteredLowOccupancyReschedule ) { if ( StartingOccupancy <= MinOccupancy ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT> ) ; S . setTargetOccupancy ( MinOccupancy ) ; } } if ( Stage == UnclusteredReschedule ) SavedMutations . swap ( Mutations ) ; for ( auto Region : Regions ) { if ( Stage == UnclusteredReschedule && ! RescheduleRegions [ RegionIdx ] ) { ++ RegionIdx ; continue ; } RegionBegin = Region . first ; RegionEnd = Region . second ; if ( RegionBegin -> getParent ( ) != MBB ) { if ( MBB ) finishBlock ( ) ; MBB = RegionBegin -> getParent ( ) ; startBlock ( MBB ) ; if ( Stage == InitialSchedule ) computeBlockPressure ( MBB ) ; } unsigned NumRegionInstrs = std :: distance ( begin ( ) , end ( ) ) ; enterRegion ( MBB , begin ( ) , end ( ) , NumRegionInstrs ) ; if ( begin ( ) == end ( ) || begin ( ) == std :: prev ( end ( ) ) ) { exitRegion ( ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << MF . getName ( ) << <STR_LIT> << printMBBReference ( * MBB ) << <STR_LIT> << MBB -> getName ( ) << <STR_LIT> << * begin ( ) << <STR_LIT> ; if ( RegionEnd != MBB -> end ( ) ) dbgs ( ) << * RegionEnd ; else dbgs (


<s> ScheduleDAGMILive ( C , std :: move ( S ) ) , ST ( MF . getSubtarget < GCNSubtarget > ( ) ) , MFI ( * MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ) , StartingOccupancy ( MFI . getOccupancy ( ) ) , MinOccupancy ( StartingOccupancy ) , Stage ( Collect ) , RegionIdx ( <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << StartingOccupancy <<


<s> RPTracker . advance ( begin ( ) , end ( ) , & LiveIns


<s> unsigned NewSGPRPressure = Pressure [ <STR_LIT> :: SReg_32 ] ; unsigned NewVGPRPressure = Pressure [ <STR_LIT> :: VGPR_32 ] ; const unsigned MaxVGPRPressureInc = <NUM_LIT> ; bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >= VGPRExcessLimit ; bool ShouldTrackSGPRs = ! ShouldTrackVGPRs && SGPRPressure >= SGPRExcessLimit ; if ( ShouldTrackVGPRs && NewVGPRPressure >= VGPRExcessLimit ) { Cand . RPDelta . Excess = PressureChange ( <STR_LIT> :: VGPR_32 ) ; Cand . RPDelta . Excess . setUnitInc ( NewVGPRPressure - VGPRExcessLimit ) ; } if ( ShouldTrackSGPRs && NewSGPRPressure >= SGPRExcessLimit ) { Cand . RPDelta . Excess = PressureChange ( <STR_LIT> :: SReg_32 ) ; Cand . RPDelta . Excess . setUnitInc ( NewSGPRPressure - SGPRExcessLimit ) ; } int SGPRDelta = NewSGPRPressure - SGPRCriticalLimit ; int VGPRDelta = NewVGPRPressure -


<s> SGPRCriticalLimit = ST . getMaxNumSGPRs ( TargetOccupancy , true ) ; VGPRCriticalLimit = ST . getMaxNumVGPRs ( TargetOccupancy ) ; } else { SGPRCriticalLimit = SRI -> getRegPressureSetLimit ( DAG -> MF , <STR_LIT> :: SReg_32 ) ; VGPRCriticalLimit = SRI -> getRegPressureSetLimit ( DAG -> MF , <STR_LIT> :: VGPR_32 ) ; } SGPRCriticalLimit -= ErrorMargin


<s> ReadyQueue & Q = Zone . Available ; for ( SUnit * SU : Q ) { SchedCandidate TryCand ( ZonePolicy ) ; initCandidate ( TryCand , SU , Zone . isTop ( ) , RPTracker , SRI , SGPRPressure , VGPRPressure ) ; SchedBoundary * ZoneArg = Cand . AtTop == TryCand . AtTop ? & Zone : nullptr ; GenericScheduler :: tryCandidate ( Cand , TryCand ,


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; PressureAfter . print ( dbgs ( ) ) ) ; if ( PressureAfter . getSGPRNum ( ) <= S . SGPRCriticalLimit && PressureAfter . getVGPRNum ( ) <= S . VGPRCriticalLimit ) { Pressure [ RegionIdx ] = PressureAfter ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return ; } unsigned Occ = MFI . getOccupancy ( ) ; unsigned WavesAfter = std :: min ( Occ , PressureAfter . getOccupancy ( ST ) ) ; unsigned WavesBefore = std :: min ( Occ , PressureBefore . getOccupancy ( ST ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << WavesBefore << <STR_LIT> << WavesAfter << <STR_LIT> ) ; unsigned NewOccupancy = std :: max ( WavesAfter , WavesBefore ) ; if ( WavesAfter < WavesBefore && WavesAfter < MinOccupancy && WavesAfter >= MFI . getMinAllowedOccupancy ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MFI . getMinAllowedOccupancy ( ) << <STR_LIT> ) ; NewOccupancy = WavesAfter ; } if ( NewOccupancy < MinOccupancy ) { MinOccupancy = NewOccupancy ; MFI . limitOccupancy ( MinOccupancy ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT> ) ; } unsigned MaxVGPRs = ST . getMaxNumVGPRs ( MF ) ; unsigned MaxSGPRs = ST . getMaxNumSGPRs ( MF ) ; if ( PressureAfter . getVGPRNum ( ) > MaxVGPRs || PressureAfter . getSGPRNum ( ) > MaxSGPRs ) RescheduleRegions [ RegionIdx ] = true ; if ( WavesAfter >= MinOccupancy ) { if ( Stage == UnclusteredReschedule && ! PressureAfter . less ( ST , PressureBefore ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } else if ( WavesAfter > MFI . getMinWavesPerEU ( ) || PressureAfter . less ( ST , PressureBefore ) || ! RescheduleRegions [ RegionIdx ] ) { Pressure [ RegionIdx ] = PressureAfter ; return ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; RescheduleRegions [ RegionIdx ] =


<s> bool hasAtomicFaddInsts ( ) const { return HasAtomicFaddRtnInsts ||


<s> bool hasCompressedExport ( ) const { return ! GFX11Insts


<s> bool hasDelayAlu ( ) const { return GFX11Insts


<s> bool hasFlatScratchSVSMode ( ) const { return GFX940Insts


<s> bool hasLdsDirect ( ) const { return getGeneration ( ) >= GFX11


<s> bool hasLegacyGeometry ( ) const { return getGeneration ( ) < GFX11


<s> bool hasNullExportTarget ( ) const { return !


<s> bool hasPermLane64 ( ) const { return getGeneration (


<s> return getGeneration ( ) == <ISA_LIT>Subtarget ::


<s> bool hasVALUPartialForwardingHazard ( ) const { return getGeneration ( ) >=


<s> bool hasVALUTransUseHazard ( ) const { return getGeneration ( )


<s> bool hasVALUTransUseHazard ( ) const { return getGeneration (


<s> bool hasVOPDInsts ( ) const { return HasVOPDInsts


<s> bool d16PreservesUnusedBits ( ) const { return hasD16LoadStore ( ) && ! TargetID


<s> bool flatScratchIsArchitected ( ) const { return HasArchitectedFlatScratch


<s> unsigned getMaxNumAGPRs ( const Function &


<s> const <STR_LIT> :: <ISA_LIT>TargetID &


<s> return isAmdHsaOS ( ) ? TrapHandlerAbi :: AMDHSA


<s> TrapHandlerAbi getTrapHandlerAbi ( ) const { return isAmdHsaOS ( ) ? TrapHandlerAbi :: AMDHSA


<s> bool hasDot7Insts ( ) const { return HasDot7Insts


<s> bool hasExtendedImageInsts ( ) const { return HasExtendedImageInsts


<s> bool hasFlatScratchSTMode ( ) const { return hasFlatScratchInsts ( ) && ( hasGFX10_3Insts ( )


<s> bool hasFlatScratchSVSMode ( ) const { return


<s> bool hasGFX10_AEncoding ( ) const { return


<s> bool hasLDSFPAtomicAdd ( ) const { return


<s> bool hasLshlAddB64 ( ) const { return


<s> bool hasNegativeScratchOffsetBug ( )


<s> bool hasNegativeScratchOffsetBug ( ) const { return NegativeScratchOffsetBug


<s> bool hasNegativeUnalignedScratchOffsetBug ( ) const { return


<s> bool isTgSplitEnabled ( ) const { return


<s> bool needsAlignedVGPRs ( ) const { return


<s> default : LLVM_FALLTHROUGH ; case CallingConv :: <ISA_LIT>_GS : LLVM_FALLTHROUGH ; case CallingConv :: <ISA_LIT>_CS : LLVM_FALLTHROUGH ; case CallingConv :: <ISA_LIT>_VS : RsrcReg = R_028868_SQ_PGM_RESOURCES_VS ; break ; case CallingConv :: <ISA_LIT>_PS : RsrcReg = R_028850_SQ_PGM_RESOURCES_PS ; break ; } } OutStreamer -> emitIntValue ( RsrcReg , <NUM_LIT> ) ; OutStreamer -> emitIntValue ( S_NUM_GPRS ( MaxGPR + <NUM_LIT> ) | S_STACK_<ISA_LIT>ZE ( MFI -> CFStackSize ) , <NUM_LIT> ) ; OutStreamer -> emitIntValue ( R_02880C_DB_SHADER_CONTROL , <NUM_LIT> ) ; OutStreamer -> emitIntValue ( S_02880C_KILL_ENABLE ( killPixel ) , <NUM_LIT> ) ; if ( <STR_LIT> ( MF . getFunction ( ) . getCallingConv ( ) ) ) { OutStreamer -> emitIntValue ( R_0288E8_SQ_LDS_ALLOC , <NUM_LIT> ) ; OutStreamer -> emitIntValue ( alignTo ( MFI -> getLDSSize ( ) , <NUM_LIT> ) >>


<s> EmitProgramInfo<ISA_LIT> ( MF ) ; emitFunctionBody ( ) ; if ( isVerbose ( ) ) { MCSectionELF * CommentSection = Context . getELFSection ( <STR_LIT> , ELF :: SHT_PROGBITS , <NUM_LIT> ) ; OutStreamer -> SwitchSection ( CommentSection ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > (


<s> MCSectionELF * ConfigSection = Context . getELFSection ( <STR_LIT> , ELF :: SHT_PROGBITS , <NUM_LIT> ) ; OutStreamer -> SwitchSection ( ConfigSection ) ; EmitProgramInfo<ISA_LIT> ( MF ) ; EmitFunctionBody ( ) ; if ( isVerbose


<s> return new <ISA_LIT>AsmPrinter ( TM , std :: move (


<s> const <ISA_LIT>Subtarget & STM = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const <ISA_LIT>RegisterInfo * RI = STM . getRegisterInfo ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; for ( const MachineBasicBlock & MBB : MF ) { for ( const MachineInstr & MI : MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) killPixel = true ; unsigned numOperands = MI . getNumOperands ( ) ; for ( unsigned op_idx = <NUM_LIT> ; op_idx < numOperands ; op_idx ++ ) { const MachineOperand & MO = MI . getOperand ( op_idx ) ; if ( ! MO . isReg ( ) ) continue ; unsigned HWReg = RI -> getHWRegIndex ( MO . getReg ( ) ) ; if ( HWReg > <NUM_LIT> ) continue ; MaxGPR = std :: max ( MaxGPR , HWReg ) ; } } } unsigned RsrcReg ; if ( STM . getGeneration ( ) >= <ISA_LIT>Subtarget :: EVERGREEN ) { switch ( MF . getFunction ( ) . getCallingConv ( ) ) { default : LLVM_FALLTHROUGH ; case CallingConv :: <ISA_LIT>_CS : RsrcReg = R_0288D4_SQ_PGM_RESOURCES_LS ; break ; case CallingConv :: <ISA_LIT>_GS : RsrcReg = R_028878_SQ_PGM_RESOURCES_GS ; break ; case CallingConv :: <ISA_LIT>_PS :

RsrcReg = R_028844_SQ_PGM_RESOURCES_PS ; break ;


<s> StringRef <ISA_LIT>AsmPrinter :: getPassName ( ) const { return <STR_LIT>


<s> MF . ensureAlignment ( <NUM_LIT> ) ; SetupMachineFunction ( MF ) ; MCContext & Context = getObjFileLowering ( ) . getContext ( ) ; MCSectionELF * ConfigSection = Context . getELFSection ( <STR_LIT> , ELF :: SHT_PROGBITS , <NUM_LIT> ) ; OutStreamer -> SwitchSection ( ConfigSection ) ; EmitProgramInfo<ISA_LIT> ( MF ) ; EmitFunctionBody ( ) ; if ( isVerbose ( ) ) { MCSectionELF * CommentSection = Context . getELFSection ( <STR_LIT> , ELF :: SHT_PROGBITS , <NUM_LIT> ) ; OutStreamer -> SwitchSection ( CommentSection ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; OutStreamer -> emitRawComment ( Twine ( <STR_LIT> + Twine ( MFI -> CFStackSize ) ) ) ; } return false


<s> OutStreamer -> SwitchSection ( ConfigSection ) ; EmitProgramInfo<ISA_LIT> ( MF ) ; EmitFunctionBody ( ) ; if ( isVerbose ( ) ) { MCSectionELF * CommentSection = Context . getELFSection ( <STR_LIT> , ELF :: SHT_PROGBITS , <NUM_LIT> ) ; OutStreamer -> SwitchSection ( CommentSection ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; OutStreamer -> emitRawComment ( Twine ( <STR_LIT> + Twine ( MFI -> CFStackSize ) ) ) ; } return false


<s> return new <ISA_LIT>ClauseMergePass (


<s> bool <ISA_LIT>ClauseMergePass :: runOnMachineFunction ( MachineFunction & MF ) { if ( skipFunction ( * MF . getFunction ( ) ) ) return false ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = ST . getInstrInfo ( ) ; for ( MachineFunction :: iterator BB = MF . begin ( ) , BB_E = MF . end ( ) ; BB != BB_E ; ++ BB ) { MachineBasicBlock & MBB = * BB ; MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; MachineBasicBlock :: iterator LatestCFAlu = E ; while ( I != E ) { MachineInstr & MI = * I ++ ; if ( ( ! TII -> canBeConsideredALU ( MI ) && ! isCFAlu ( MI ) ) || TII -> mustBeLastInClause ( MI . getOpcode ( ) ) ) LatestCFAlu = E ; if ( ! isCFAlu ( MI ) ) continue ; cleanPotentialDisabledCFAlu ( MI ) ; if ( LatestCFAlu != E && mergeIfPossible ( * LatestCFAlu , MI ) ) { MI . eraseFromParent ( ) ; } else { assert ( MI . getOperand ( <NUM_LIT> ) .


<s> int CntIdx = TII -> getOperandIdx ( <STR_LIT> , <STR_LIT> :: COUNT ) ; MachineBasicBlock :: iterator I = CFAlu , E = CFAlu -> getParent ( ) -> end ( ) ; I ++ ; do { while ( I != E && ! isCFAlu ( I ) ) I ++ ; if ( I == E )


<s> const char * <ISA_LIT>ClauseMergePass :: getPassName (


<s> const char * <ISA_LIT>ClauseMergePass ::


<s> case <STR_LIT> : case <STR_LIT> : return true ; default : return false


<s> DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( RootCFAlu -> getOpcode ( ) == <STR_LIT> ) return false ; int Mode0Idx = TII -> getOperandIdx ( <STR_LIT> , <STR_LIT> :: KCACHE_MODE0 ) ; int KBank0Idx = TII -> getOperandIdx ( <STR_LIT> , <STR_LIT> :: KCACHE_BANK0 ) ; int KBank0LineIdx = TII -> getOperandIdx ( <STR_LIT> , <STR_LIT> :: KCACHE_ADDR0 ) ; if ( LatrCFAlu -> getOperand ( Mode0Idx ) . getImm ( ) && RootCFAlu -> getOperand ( Mode0Idx ) . getImm ( ) && ( LatrCFAlu -> getOperand ( KBank0Idx ) . getImm ( ) != RootCFAlu -> getOperand ( KBank0Idx ) . getImm ( ) || LatrCFAlu -> getOperand ( KBank0LineIdx ) . getImm ( ) != RootCFAlu -> getOperand ( KBank0LineIdx ) . getImm ( ) ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } int Mode1Idx = TII -> getOperandIdx ( <STR_LIT> , <STR_LIT> :: KCACHE_MODE1 ) ; int KBank1Idx = TII -> getOperandIdx ( <STR_LIT> , <STR_LIT> ::


<s> unsigned <ISA_LIT>ClauseMergePass :: getCFAluSize ( const MachineInstr & MI ) const { assert ( isCFAlu ( MI ) ) ; return MI . getOperand ( TII -> getOperandIdx ( MI . getOpcode ( ) , <STR_LIT> :: COUNT ) ) .


<s> bool <ISA_LIT>ClauseMergePass :: isCFAluEnabled ( const MachineInstr & MI ) const { assert ( isCFAlu (


<s> bool <ISA_LIT>ClauseMergePass :: isCFAluEnabled ( const MachineInstr & MI ) const { assert ( isCFAlu ( MI )


<s> if ( LatrCFAlu . getOperand ( Mode0Idx ) . getImm ( ) && RootCFAlu . getOperand ( Mode0Idx ) . getImm ( ) && ( LatrCFAlu . getOperand ( KBank0Idx ) . getImm ( ) != RootCFAlu . getOperand ( KBank0Idx ) . getImm ( ) || LatrCFAlu . getOperand ( KBank0LineIdx ) . getImm ( ) != RootCFAlu . getOperand ( KBank0LineIdx ) . getImm ( ) ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } int Mode1Idx = TII -> getOperandIdx ( <STR_LIT> , <STR_LIT> :: KCACHE_MODE1 ) ; int KBank1Idx = TII -> getOperandIdx ( <STR_LIT> , <STR_LIT> :: KCACHE_BANK1 ) ; int KBank1LineIdx = TII -> getOperandIdx ( <STR_LIT> , <STR_LIT> :: KCACHE_ADDR1 ) ; if ( LatrCFAlu . getOperand ( Mode1Idx ) . getImm ( ) && RootCFAlu . getOperand ( Mode1Idx ) . getImm ( ) && ( LatrCFAlu . getOperand ( KBank1Idx ) . getImm ( ) != RootCFAlu . getOperand ( KBank1Idx ) . getImm ( ) || LatrCFAlu . getOperand ( KBank1LineIdx ) . getImm ( ) != RootCFAlu . getOperand ( KBank1LineIdx ) . getImm ( ) ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( LatrCFAlu . getOperand ( Mode0Idx ) . getImm ( ) ) { RootCFAlu . getOperand ( Mode0Idx ) . setImm ( LatrCFAlu . getOperand ( Mode0Idx ) . getImm ( ) ) ; RootCFAlu . getOperand ( KBank0Idx ) . setImm ( LatrCFAlu . getOperand ( KBank0Idx ) . getImm ( ) ) ; RootCFAlu . getOperand ( KBank0LineIdx ) . setImm ( LatrCFAlu . getOperand ( KBank0LineIdx ) . getImm ( ) ) ; } if ( LatrCFAlu . getOperand ( Mode1Idx ) . getImm ( ) ) { RootCFAlu . getOperand ( Mode1Idx ) . setImm ( LatrCFAlu . getOperand ( Mode1Idx ) . getImm ( ) ) ; RootCFAlu . getOperand ( KBank1Idx ) . setImm ( LatrCFAlu . getOperand ( KBank1Idx ) . getImm (


<s> if ( skipFunction ( * MF . getFunction ( ) ) ) return false ; TII = static_cast < const <ISA_LIT>InstrInfo * > ( MF . getSubtarget ( ) . getInstrInfo ( ) ) ; for ( MachineFunction :: iterator BB = MF . begin ( ) , BB_E = MF . end ( ) ; BB != BB_E ; ++ BB ) { MachineBasicBlock & MBB = * BB ; MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; MachineBasicBlock :: iterator LatestCFAlu = E ; while ( I != E ) { MachineInstr * MI = I ++ ; if ( ( ! TII -> canBeConsideredALU ( MI ) && ! isCFAlu ( MI ) ) || TII -> mustBeLastInClause ( MI -> getOpcode (


<s> MachineBasicBlock :: iterator LatestCFAlu = E ; while ( I != E ) { MachineInstr * MI = I ++ ; if ( ( ! TII -> canBeConsideredALU ( MI ) && ! isCFAlu ( MI ) ) || TII -> mustBeLastInClause ( MI -> getOpcode ( ) ) ) LatestCFAlu = E ; if ( ! isCFAlu ( MI ) ) continue ; cleanPotentialDisabledCFAlu ( MI ) ; if ( LatestCFAlu != E && mergeIfPossible ( LatestCFAlu , MI ) ) { MI -> eraseFromParent ( ) ; }


<s> for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; MachineBasicBlock :: iterator LatestCFAlu = E ; while ( I != E ) { MachineInstr & MI = * I ++ ; if ( ( ! TII -> canBeConsideredALU ( MI ) && ! isCFAlu ( MI ) ) || TII -> mustBeLastInClause ( MI . getOpcode ( )


<s> MachineBasicBlock :: iterator I = CFAlu , E = CFAlu . getParent ( ) -> end ( ) ; I ++ ; do { while ( I != E && ! isCFAlu ( * I ) ) I ++ ; if ( I == E ) return ; MachineInstr & MI = *


<s> llvm :: FunctionPass * llvm :: create<ISA_LIT>ClauseMergePass (


<s> llvm :: FunctionPass * llvm :: create<ISA_LIT>ClauseMergePass ( ) { return new <ISA_LIT>ClauseMergePass


<s> return MI . getOperand ( TII -> getOperandIdx ( MI . getOpcode ( ) , <STR_LIT> :: COUNT ) )


<s> return MI . getOperand ( TII -> getOperandIdx ( MI . getOpcode ( )


<s> static bool isCFAlu ( const MachineInstr & MI ) { switch ( MI . getOpcode ( ) ) { case <STR_LIT>


<s> bool <ISA_LIT>ClauseMergePass :: isCFAluEnabled ( const MachineInstr & MI ) const { assert ( isCFAlu ( MI


<s> bool <ISA_LIT>ClauseMergePass :: isCFAluEnabled ( const MachineInstr & MI ) const { assert ( isCFAlu ( MI ) ) ; return MI . getOperand ( TII -> getOperandIdx ( MI . getOpcode

( ) , <STR_LIT> :: Enabled ) ) . getImm ( ) ;


<s> bool <ISA_LIT>ClauseMergePass :: runOnMachineFunction ( MachineFunction & MF ) { if ( skipFunction ( MF . getFunction ( ) ) ) return false ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = ST . getInstrInfo ( ) ; for ( MachineFunction :: iterator BB = MF . begin ( ) , BB_E = MF . end ( ) ; BB != BB_E ; ++ BB ) { MachineBasicBlock & MBB = * BB ; MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; MachineBasicBlock :: iterator LatestCFAlu = E ; while ( I != E ) { MachineInstr & MI = * I ++ ; if ( ( ! TII -> canBeConsideredALU ( MI ) && ! isCFAlu ( MI ) ) || TII -> mustBeLastInClause ( MI . getOpcode ( ) ) ) LatestCFAlu = E ; if ( ! isCFAlu ( MI ) ) continue ; cleanPotentialDisabledCFAlu (


<s> void CounterPropagateAddr ( const std :: set < MachineInstr * > & MIs , unsigned Addr ) const { for ( MachineInstr * MI : MIs


<s> for ( MachineInstr * MI : MIs ) { CounterPropagateAddr ( * MI ,


<s> return new <ISA_LIT>ControlFlowFinalizer ( )


<s> FunctionPass * llvm :: create<ISA_LIT>ControlFlowFinalizer ( ) { return new <ISA_LIT>ControlFlowFinalizer


<s> CounterPropagateAddr ( * Clause . first , CfCount ) ; MachineBasicBlock * BB = Clause . first -> getParent ( ) ; BuildMI ( BB , DL , TII -> get ( <STR_LIT> ) )


<s> static const unsigned LiteralRegs [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; const SmallVector < std :: pair < MachineOperand * , int64_t > , <NUM_LIT> > Srcs = TII -> getSrcs ( MI ) ; for ( const auto & Src : Srcs ) { if ( Src . first -> getReg ( ) != <STR_LIT> ) continue ; int64_t Imm = Src . second ; std :: vector < MachineOperand * > :: iterator It = llvm :: find_if ( Lits , [ & ] ( MachineOperand * val ) { return val -> isImm ( ) && ( val -> getImm ( ) ==


<s> static const unsigned LiteralRegs [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; const SmallVector < std :: pair < MachineOperand * , int64_t > , <NUM_LIT> > Srcs = TII -> getSrcs ( MI ) ; for ( const auto & Src : Srcs ) { if ( Src . first -> getReg ( ) != <STR_LIT> ) continue ; int64_t Imm = Src . second ; std :: vector < MachineOperand * > :: iterator It = llvm :: find_if ( Lits , [ & ] ( MachineOperand * val ) { return val -> isImm ( ) && ( val -> getImm ( ) == Imm ) ; } ) ; MachineOperand & Operand = MI . getOperand ( TII -> getOperandIdx ( MI . getOpcode ( ) , <STR_LIT> :: literal ) ) ; if ( It != Lits . end ( ) ) { unsigned Index = It - Lits


<s> unsigned LiteralPair0 = Literals [ i ] ; unsigned LiteralPair1 = ( i + <NUM_LIT> < e ) ? Literals [ i + <NUM_LIT> ] : <NUM_LIT> ; InsertPos = BuildMI ( MBB , InsertPos -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( LiteralPair0 ) . addImm ( LiteralPair1 ) ; } return


<s> bool IsTrivialInst ( MachineInstr & MI ) const { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : return true ; default : return false


<s> BI -> unbundleFromPred ( ) ; for ( MachineOperand & MO : BI -> operands ( ) ) { if ( MO . isReg ( ) && MO . isInternalRead ( ) ) MO . setIsInternalRead ( false ) ; } getLiteral ( * BI , Literals ) ; ClauseContent . push_back ( & * BI ) ; } I = BI ; DeleteMI . eraseFromParent ( ) ; } else { getLiteral ( * I , Literals ) ; ClauseContent . push_back ( & * I ) ; I ++ ; } for ( unsigned i = <NUM_LIT> , e = Literals . size ( ) ; i < e ; i += <NUM_LIT> ) { MachineInstrBuilder MILit = BuildMI ( MBB , I , I -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; if ( Literals [ i ] -> isImm ( ) ) { MILit . addImm ( Literals [ i ] -> getImm ( ) ) ; } else { MILit . addGlobalAddress ( Literals [ i ] -> getGlobal ( ) , Literals [ i ] -> getOffset ( ) ) ; } if ( i + <NUM_LIT> < e ) { if ( Literals [ i + <NUM_LIT> ] -> isImm ( ) ) { MILit . addImm ( Literals [ i + <NUM_LIT> ] -> getImm ( ) ) ; } else { MILit . addGlobalAddress ( Literals [ i + <NUM_LIT> ] -> getGlobal ( ) , Literals [ i + <NUM_LIT> ] -> getOffset ( ) ) ; } } else MILit . addImm ( <NUM_LIT> ) ; ClauseContent . push_back ( MILit ) ; } } assert ( ClauseContent . size ( ) < <NUM_LIT> &&


<s> if ( ! isWQM ) { if ( ! ST -> hasCaymanISA ( ) && ! branchStackContains ( CFStack :: FIRST_NON_WQM_PUSH ) ) Item = CFStack :: FIRST_NON_WQM_PUSH ; else if ( CurrentEntries > <NUM_LIT> && ST -> getGeneration ( ) > <ISA_LIT>Subtarget :: EVERGREEN && ! ST -> hasCaymanISA ( ) && ! branchStackContains ( CFStack :: FIRST_NON_WQM_PUSH_W_FULL_ENTRY ) ) Item = CFStack :: FIRST_NON_WQM_PUSH_W_FULL_ENTRY ; else Item = CFStack :: SUB_ENTRY ; } else Item = CFStack :: ENTRY ; break ; } BranchStack . push_back


<s> case <STR_LIT> : case <STR_LIT> : if ( ! isWQM ) { if ( ! ST -> hasCaymanISA ( ) && ! branchStackContains ( CFStack :: FIRST_NON_WQM_PUSH ) ) Item = CFStack :: FIRST_NON_WQM_PUSH ; else if ( CurrentEntries > <NUM_LIT> && ST -> getGeneration ( ) > <ISA_LIT>Subtarget :: EVERGREEN && ! ST -> hasCaymanISA ( ) && ! branchStackContains ( CFStack :: FIRST_NON_WQM_PUSH_W_FULL_ENTRY ) ) Item = CFStack :: FIRST_NON_WQM_PUSH_W_FULL_ENTRY ; else Item = CFStack :: SUB_ENTRY ; } else Item = CFStack :: ENTRY ; break ; } BranchStack . push_back ( Item ) ; if ( Item == CFStack :: ENTRY ) CurrentEntries ++ ; else CurrentSubEntries += getSubEntrySize (


<s> if ( Opcode == <STR_LIT> && ST -> hasCaymanISA ( ) && getLoopDepth ( ) > <NUM_LIT> ) return true ; if ( ! ST -> hasCFAluBug ( ) ) return false ; switch ( Opcode ) { default : return false ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( CurrentSubEntries == <NUM_LIT> ) return false ; if ( ST -> getWavefrontSize ( ) == <NUM_LIT> ) { return CurrentSubEntries


<s> unsigned CurrentStackSize = CurrentEntries + ( alignTo ( CurrentSubEntries ,


<s> void CFStack :: updateMaxStackSize ( ) { unsigned CurrentStackSize = CurrentEntries + ( alignTo ( CurrentSubEntries , <NUM_LIT> ) / <NUM_LIT> ) ; MaxStackSize = std :: max ( CurrentStackSize


<s> for ( std :: vector < CFStack :: StackItem > :: const_iterator I = BranchStack . begin ( ) , E = BranchStack . end ( )


<s> for ( std :: vector < CFStack :: StackItem > :: const_iterator I = BranchStack . begin ( ) ,

E = BranchStack . end ( ) ; I != E ; ++ I ) {


<s> Clause . first -> getOperand ( <NUM_LIT> ) . setImm ( <NUM_LIT> ) ; CounterPropagateAddr ( Clause . first , CfCount ) ; MachineBasicBlock * BB = Clause . first -> getParent ( ) ; BuildMI ( BB , InsertPos -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( CfCount ) ; for ( unsigned i = <NUM_LIT> , e = Clause . second . size ( ) ; i <


<s> void getLiteral ( MachineInstr * MI , std :: vector < int64_t > & Lits ) const { static const unsigned LiteralRegs [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; const SmallVector < std :: pair < MachineOperand * , int64_t > , <NUM_LIT> > Srcs = TII -> getSrcs ( MI ) ; for ( unsigned i = <NUM_LIT> , e = Srcs . size ( ) ; i < e ; ++ i ) { if ( Srcs [ i ] . first -> getReg ( ) != <STR_LIT> ) continue ; int64_t Imm = Srcs [ i ] . second ; std :: vector < int64_t > :: iterator It = std :: find ( Lits . begin ( ) , Lits . end ( ) , Imm ) ; if ( It != Lits . end ( ) ) { unsigned Index = It - Lits . begin ( ) ; Srcs [ i ] . first -> setReg ( LiteralRegs [ Index ] ) ; } else { assert ( Lits . size ( ) < <NUM_LIT> && <STR_LIT> ) ; Srcs [ i ] . first -> setReg ( LiteralRegs [ Lits


<s> } else { return <NUM_LIT> ; } case CFStack :: FIRST_NON_WQM_PUSH_W_FULL_ENTRY : assert ( ST -> getGeneration ( ) >= <ISA_LIT>Subtarget :: EVERGREEN ) ; return


<s> unsigned LiteralPair0 = Literals [ i ] ; unsigned LiteralPair1 = ( i + <NUM_LIT> < e ) ? Literals [ i + <NUM_LIT> ] : <NUM_LIT> ; InsertPos = BuildMI ( MBB , InsertPos -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) .

addImm ( LiteralPair0 ) . addImm ( LiteralPair1 ) ;


<s> if ( ! MO . isReg ( ) ) continue ; if ( MO . isDef ( ) ) { unsigned Reg = MO . getReg ( ) ; if ( <STR_LIT> . contains ( Reg ) ) DstMI = Reg ; else DstMI = TRI -> getMatchingSuperReg ( Reg , TRI -> getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ; } if ( MO . isUse ( ) ) { unsigned Reg = MO . getReg


<s> else DstMI = TRI -> getMatchingSuperReg ( Reg , TRI -> getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ; } if ( MO . isUse ( ) ) { unsigned Reg = MO . getReg ( ) ; if ( <STR_LIT> . contains ( Reg ) ) SrcMI = Reg ; else SrcMI = TRI -> getMatchingSuperReg ( Reg , TRI -> getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ; } } if ( ( DstRegs . find ( SrcMI ) == DstRegs . end ( ) ) ) { DstRegs . insert


<s> case <STR_LIT> : case <STR_LIT> : return true ; default : return false


<s> if ( MO . isReg ( ) && MO . isInternalRead ( ) ) MO . setIsInternalRead ( false ) ; } getLiteral ( & * BI , Literals ) ; ClauseContent . push_back ( & * BI ) ; } I = BI ; DeleteMI -> eraseFromParent ( ) ; } else { getLiteral ( I , Literals ) ; ClauseContent . push_back ( I ) ; I ++ ; } for ( unsigned i = <NUM_LIT> , e = Literals . size ( ) ; i < e ; i += <NUM_LIT> ) { unsigned literal0 = Literals [ i ] ; unsigned literal2 = ( i + <NUM_LIT> < e ) ? Literals [ i + <NUM_LIT> ] : <NUM_LIT> ; MachineInstr * MILit = BuildMI ( MBB , I , I -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( literal0 ) . addImm ( literal2 ) ; ClauseContent . push_back (


<s> ClauseFile MakeFetchClause ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator & I ) const { MachineBasicBlock :: iterator ClauseHead = I ; std :: vector < MachineInstr * > ClauseContent ; unsigned AluInstCount = <NUM_LIT> ; bool IsTex = TII -> usesTextureCache ( ClauseHead ) ; std :: set < unsigned > DstRegs ; for ( MachineBasicBlock :: iterator E = MBB . end ( ) ; I != E ; ++ I ) { if ( IsTrivialInst ( I ) ) continue ; if ( AluInstCount >= MaxFetchInst ) break ; if ( ( IsTex && ! TII -> usesTextureCache ( I ) ) || ( ! IsTex && ! TII -> usesVertexCache ( I ) ) ) break ; if ( ! isCompatibleWithClause ( I , DstRegs ) ) break ; AluInstCount ++


<s> CFStack :: StackItem Top = BranchStack . back ( ) ; if ( Top == CFStack :: ENTRY ) CurrentEntries -- ; else CurrentSubEntries -= getSubEntrySize ( Top ) ; BranchStack . pop_back


<s> void CFStack :: popLoop ( ) { CurrentEntries --


<s> void CFStack :: pushLoop ( ) { LoopStack . push_back ( CFStack :: ENTRY


<s> if ( ! ST -> hasCFAluBug ( ) ) return false ; switch ( Opcode ) { default : return false ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( CurrentSubEntries == <NUM_LIT> )


<s> void CFStack :: updateMaxStackSize ( ) { unsigned CurrentStackSize = CurrentEntries + ( RoundUpToAlignment ( CurrentSubEntries , <NUM_LIT> ) / <NUM_LIT> ) ; MaxStackSize = std :: max (


<s> unsigned CurrentStackSize = CurrentEntries + ( RoundUpToAlignment ( CurrentSubEntries , <NUM_LIT> )


<s> CounterPropagateAddr ( * Clause . first , CfCount ) ; MachineBasicBlock * BB = Clause . first -> getParent ( ) ; BuildMI ( BB , DL , TII -> get ( <STR_LIT> ) ) .


<s> for ( MachineInstr :: const_mop_iterator I = MI . operands_begin ( ) , E = MI . operands_end ( ) ; I != E ; ++ I ) { const MachineOperand & MO = * I ; if ( ! MO . isReg ( ) ) continue ; if ( MO . isDef ( ) ) { unsigned Reg = MO . getReg ( ) ; if ( <STR_LIT> . contains ( Reg ) ) DstMI = Reg ; else DstMI = TRI -> getMatchingSuperReg ( Reg , <ISA_LIT>RegisterInfo :: getSubRegFromChannel (

TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ;


<s> bool CFStack :: branchStackContains ( CFStack :: StackItem Item ) { return llvm :: is_contained (


<s> BuildMI ( BB , DL , TII -> get ( <STR_LIT> ) ) . addImm ( CfCount ) ; for ( MachineInstr * MI : Clause . second ) BB -> splice ( InsertPos , BB , MI ) ; CfCount += Clause . second . size


<s> BuildMI ( BB , DL , TII -> get ( <STR_LIT> ) ) . addImm ( CfCount ) ; for ( MachineInstr * MI : Clause . second ) BB -> splice ( InsertPos , BB


<s> MachineBasicBlock * BB = Clause . first -> getParent ( ) ; BuildMI ( BB , DL , TII -> get ( <STR_LIT> ) ) . addImm ( CfCount ) ; for ( MachineInstr * MI : Clause . second ) BB -> splice ( InsertPos


<s> Register Reg = MO . getReg ( ) ; if ( <STR_LIT> . contains ( Reg ) ) DstMI = Reg ; else DstMI = TRI -> getMatchingSuperReg ( Reg , <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ; } if ( MO . isUse ( ) ) { Register Reg = MO . getReg ( ) ; if ( <STR_LIT> . contains ( Reg ) ) SrcMI = Reg ; else SrcMI = TRI -> getMatchingSuperReg ( Reg , <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ; } } if ( ( DstRegs . find ( SrcMI )


<s> void CFStack :: updateMaxStackSize ( ) { unsigned CurrentStackSize = CurrentEntries + divideCeil ( CurrentSubEntries , <NUM_LIT> ) ; MaxStackSize = std :: max (


<s> MaxStackSize = std :: max ( CurrentStackSize ,


<s> Clause . first -> getOperand ( <NUM_LIT> ) . setImm ( <NUM_LIT> ) ; CounterPropagateAddr ( * Clause . first , CfCount ) ; MachineBasicBlock * BB = Clause . first -> getParent ( ) ; BuildMI ( BB , InsertPos -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) )


<s> CounterPropagateAddr ( * Clause . first , CfCount ) ; MachineBasicBlock * BB = Clause . first -> getParent ( ) ; BuildMI ( BB , InsertPos -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( CfCount ) ; for ( unsigned i = <NUM_LIT> , e = Clause . second . size ( ) ; i < e ; ++ i ) { BB -> splice ( InsertPos , BB , Clause . second [ i ] ) ; } CfCount += <NUM_LIT> * Clause . second . size (


<s> break ; case CF_LOOP_CONTINUE : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_JUMP : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_ELSE : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_POP : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_END : if ( ST -> hasCaymanISA ( ) ) { Opcode = <STR_LIT> ; break ; } Opcode = isEg ? <STR_LIT> : <STR_LIT>


<s> bool isEg = ( ST -> getGeneration ( ) >= <ISA_LIT>Subtarget :: EVERGREEN ) ; switch ( CFI ) { case CF_TC : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_VC : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_CALL_FS : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_WHILE_LOOP : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_END_LOOP : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_LOOP_BREAK : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_LOOP_CONTINUE : Opcode = isEg ? <STR_LIT> :


<s> <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; const SmallVector < std :: pair < MachineOperand * , int64_t > , <NUM_LIT> > Srcs = TII -> getSrcs ( MI ) ; for ( const auto & Src : Srcs ) { if ( Src . first -> getReg ( ) != <STR_LIT> ) continue ; int64_t Imm = Src . second ; std :: vector < MachineOperand * > :: iterator It = std :: find_if ( Lits . begin ( ) , Lits . end ( ) , [ & ] ( MachineOperand * val ) { return val -> isImm ( ) && ( val -> getImm ( ) == Imm ) ; } ) ; MachineOperand & Operand = MI . getOperand ( TII -> getOperandIdx ( MI . getOpcode ( ) , <STR_LIT> :: literal ) ) ; if ( It != Lits . end ( ) ) { unsigned Index = It - Lits . begin ( ) ; Src . first -> setReg ( LiteralRegs [ Index ] ) ; } else { assert ( Lits . size ( ) < <NUM_LIT> && <STR_LIT> ) ; Src . first -> setReg ( LiteralRegs [ Lits . size ( ) ] ) ; Lits . push_back ( & Operand


<s> int64_t Imm = Src . second ; std :: vector < MachineOperand * > :: iterator It = std :: find_if ( Lits . begin ( ) , Lits . end ( ) , [ & ] ( MachineOperand * val ) { return val -> isImm ( ) && ( val -> getImm ( ) == Imm ) ; } ) ; MachineOperand & Operand = MI . getOperand ( TII -> getOperandIdx ( MI . getOpcode ( ) ,


<s> switch ( Item ) { default : return <NUM_LIT> ; case CFStack :: FIRST_NON_WQM_PUSH : assert ( ! ST -> hasCaymanISA ( ) ) ; if ( ST -> getGeneration ( ) <= <ISA_LIT>Subtarget ::


<s> if ( ST -> getGeneration ( ) <= <ISA_LIT>Subtarget :: R700 ) { return <NUM_LIT> ; } else { return <NUM_LIT> ; } case CFStack :: FIRST_NON_WQM_PUSH_W_FULL_ENTRY


<s> I != E ; ++ I ) { const MachineOperand & MO = * I ; if ( ! MO . isReg ( ) ) continue ; if ( MO . isDef ( ) ) { unsigned Reg = MO . getReg ( ) ; if ( <STR_LIT> . contains ( Reg ) ) DstMI = Reg ; else DstMI = TRI -> getMatchingSuperReg ( Reg , TRI -> getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ; } if ( MO . isUse ( ) ) { unsigned Reg = MO . getReg ( ) ; if ( <STR_LIT> . contains ( Reg ) ) SrcMI = Reg ; else SrcMI = TRI -> getMatchingSuperReg ( Reg , TRI -> getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ; } } if ( ( DstRegs . find ( SrcMI ) == DstRegs . end ( ) ) ) { DstRegs . insert ( DstMI ) ; return true


<s> ++ I ; continue ; } if ( ! I -> isBundle ( ) && ! TII -> isALUInstr ( I -> getOpcode ( ) ) ) break ; std :: vector < MachineOperand * > Literals ; if ( I -> isBundle ( ) ) { MachineInstr & DeleteMI = * I ; MachineBasicBlock :: instr_iterator BI = I . getInstrIterator ( ) ; while ( ++ BI != E && BI -> isBundledWithPred ( ) ) { BI -> unbundleFromPred ( ) ; for ( MachineOperand & MO : BI -> operands ( ) ) { if ( MO . isReg ( ) && MO . isInternalRead ( ) ) MO . setIsInternalRead ( false ) ; } getLiteral ( * BI , Literals ) ; ClauseContent . push_back ( & * BI ) ; } I = BI ; DeleteMI . eraseFromParent ( ) ; } else { getLiteral ( * I , Literals ) ; ClauseContent . push_back ( & * I ) ; I ++ ; } for ( unsigned i = <NUM_LIT> , e = Literals . size ( ) ; i < e ; i += <NUM_LIT> ) { MachineInstrBuilder MILit = BuildMI ( MBB , I , I -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; if ( Literals [ i ] -> isImm ( ) ) { MILit . addImm ( Literals [ i ] -> getImm ( ) ) ; } else { MILit . addGlobalAddress ( Literals [ i ] -> getGlobal ( ) , Literals [ i ] -> getOffset ( ) ) ; } if ( i + <NUM_LIT> < e ) { if ( Literals [ i + <NUM_LIT> ] -> isImm ( ) ) { MILit . addImm ( Literals [ i + <NUM_LIT> ] -> getImm ( ) ) ; } else { MILit . addGlobalAddress ( Literals [ i + <NUM_LIT> ] -> getGlobal ( ) , Literals [ i + <NUM_LIT> ] -> getOffset ( ) ) ; } } else MILit . addImm ( <NUM_LIT> ) ; ClauseContent . push_back


<s> } case <STR_LIT> : { LastAlu . push_back ( nullptr ) ; MachineInstr * MIb = BuildMI ( MBB , MI , MBB . findDebugLoc ( MI ) , getHWInstrDesc ( CF_JUMP ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; IfThenElseStack . push_back ( MIb ) ; DEBUG ( dbgs ( ) << CfCount << <STR_LIT> ; MIb -> dump ( ) ; ) ; MI -> eraseFromParent ( ) ; CfCount ++ ; break ; } case <STR_LIT> : { MachineInstr * JumpInst = IfThenElseStack . back ( ) ; IfThenElseStack . pop_back ( ) ; CounterPropagateAddr ( * JumpInst , CfCount ) ; MachineInstr * MIb = BuildMI ( MBB , MI , MBB . findDebugLoc ( MI ) , getHWInstrDesc ( CF_ELSE ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; DEBUG ( dbgs ( ) << CfCount << <STR_LIT> ; MIb -> dump ( ) ; ) ; IfThenElseStack . push_back ( MIb ) ; MI -> eraseFromParent ( ) ; CfCount ++ ; break ; } case <STR_LIT> : { CFStack . popBranch ( ) ; if ( LastAlu . back ( ) ) { ToPopAfter . push_back ( LastAlu . back ( ) ) ; } else { MachineInstr * MIb = BuildMI ( MBB , MI , MBB . findDebugLoc ( MI ) , getHWInstrDesc ( CF_POP ) ) . addImm ( CfCount + <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; ( void ) MIb ; DEBUG ( dbgs ( ) << CfCount << <STR_LIT> ; MIb -> dump ( ) ; ) ; CfCount ++ ; } MachineInstr * IfOrElseInst = IfThenElseStack . back ( ) ; IfThenElseStack . pop_back ( ) ; CounterPropagateAddr ( * IfOrElseInst , CfCount ) ; IfOrElseInst -> getOperand ( <NUM_LIT> ) . setImm ( <NUM_LIT> ) ; LastAlu . pop_back ( ) ; MI -> eraseFromParent ( ) ; break ; } case <STR_LIT> : { CfCount ++ ; MachineInstr * MIb = BuildMI ( MBB , MI , MBB . findDebugLoc ( MI ) , getHWInstrDesc ( CF_LOOP_BREAK ) ) . addImm ( <NUM_LIT> ) ; LoopStack . back ( ) . second . insert ( MIb ) ; MI -> eraseFromParent ( ) ; break ; } case <STR_LIT> : { MachineInstr * MIb = BuildMI ( MBB , MI , MBB . findDebugLoc ( MI ) , getHWInstrDesc ( CF_LOOP_CONTINUE ) ) . addImm ( <NUM_LIT> ) ; LoopStack . back ( ) . second . insert ( MIb ) ; MI -> eraseFromParent ( ) ; CfCount ++ ; break ; } case <STR_LIT> : { DebugLoc DL = MBB . findDebugLoc ( MI ) ; BuildMI ( MBB , MI , DL , getHWInstrDesc (


<s> const SmallVector < std :: pair < MachineOperand * , int64_t > , <NUM_LIT> > Srcs = TII -> getSrcs ( MI ) ; for ( const auto & Src : Srcs ) { if ( Src . first -> getReg ( ) != <STR_LIT> ) continue ; int64_t Imm = Src . second ; std :: vector < MachineOperand * > :: iterator It = std :: find_if ( Lits . begin ( ) , Lits . end ( ) , [ & ] ( MachineOperand * val ) { return val -> isImm ( ) && ( val -> getImm ( ) == Imm ) ; } ) ; MachineOperand & Operand = MI -> getOperand ( TII -> getOperandIdx ( MI -> getOpcode ( ) , <STR_LIT> :: literal ) ) ; if ( It != Lits . end ( ) ) { unsigned Index = It - Lits . begin ( ) ; Src . first -> setReg (


<s> Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_VC : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_CALL_FS : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_WHILE_LOOP : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_END_LOOP : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_LOOP_BREAK : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_LOOP_CONTINUE : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_JUMP


<s> CFStack :: StackItem Item = CFStack :: ENTRY ; switch ( Opcode ) { case <STR_LIT> : case <STR_LIT> : if ( ! isWQM ) { if ( ! ST -> hasCaymanISA ( ) && ! branchStackContains ( CFStack :: FIRST_NON_WQM_PUSH ) ) Item = CFStack :: FIRST_NON_WQM_PUSH ; else if ( CurrentEntries > <NUM_LIT> && ST -> getGeneration ( ) > <ISA_LIT>Subtarget :: EVERGREEN && ! ST -> hasCaymanISA ( ) && ! branchStackContains ( CFStack :: FIRST_NON_WQM_PUSH_W_FULL_ENTRY ) ) Item = CFStack :: FIRST_NON_WQM_PUSH_W_FULL_ENTRY ; else Item = CFStack :: SUB_ENTRY ; } else Item = CFStack :: ENTRY ; break ; } BranchStack . push_back ( Item ) ; if ( Item == CFStack :: ENTRY ) CurrentEntries ++ ; else CurrentSubEntries += getSubEntrySize ( Item ) ; updateMaxStackSize (


<s> I = BI ; DeleteMI -> eraseFromParent ( ) ; } else { getLiteral ( I , Literals ) ; ClauseContent . push_back ( I ) ; I ++ ; } for ( unsigned i = <NUM_LIT> , e = Literals . size ( ) ; i < e ; i += <NUM_LIT> ) { unsigned literal0 = Literals [ i ] ; unsigned literal2 = ( i + <NUM_LIT> < e ) ? Literals [ i + <NUM_LIT> ] : <NUM_LIT> ; MachineInstr * MILit = BuildMI ( MBB , I , I -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( literal0 ) . addImm ( literal2 ) ; ClauseContent . push_back ( MILit ) ; } } assert ( ClauseContent . size ( ) <


<s> <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; const SmallVector < std :: pair < MachineOperand * , int64_t > , <NUM_LIT> > Srcs = TII -> getSrcs ( MI ) ; for ( const auto & Src : Srcs ) { if ( Src . first -> getReg ( ) != <STR_LIT> ) continue ; int64_t Imm = Src . second ; std :: vector < MachineOperand * > :: iterator It = find_if ( Lits , [ & ] ( MachineOperand * val ) { return val -> isImm ( ) && ( val -> getImm ( ) == Imm ) ; }


<s> Register Reg = MO . getReg ( ) ; if ( <STR_LIT> . contains ( Reg ) ) DstMI = Reg ; else DstMI = TRI -> getMatchingSuperReg ( Reg , <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ; } if ( MO . isUse ( ) ) { Register Reg = MO . getReg ( ) ; if ( <STR_LIT> . contains ( Reg ) ) SrcMI = Reg ; else SrcMI = TRI -> getMatchingSuperReg ( Reg , <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ; } } if ( ( DstRegs . find ( SrcMI ) == DstRegs . end ( ) ) ) { DstRegs . insert ( DstMI ) ; return


<s> FunctionPass * llvm :: create<ISA_LIT>ControlFlowFinalizer ( TargetMachine & TM ) { return new <ISA_LIT>ControlFlowFinalizer ( TM


<s> for ( MachineBasicBlock :: iterator UseI = Def ; UseI != BBEnd ; ++ UseI ) { AluInstCount += OccupiedDwords ( * UseI ) ; if ( ! SubstituteKCacheBank ( * UseI , KCacheBanks , false ) ) return false ; if ( AluInstCount >= TII -> getMaxAlusPerClause ( ) ) return false ; if ( UseI -> readsRegister ( MOI -> getReg ( ) , & TRI ) )


<s> for ( MachineInstr :: const_mop_iterator MOI = Def -> operands_begin ( ) , MOE = Def -> operands_end ( ) ; MOI != MOE ; ++ MOI ) { if ( ! MOI -> isReg ( ) || ! MOI -> isDef ( ) || TRI . isPhysRegLiveAcrossClauses ( MOI -> getReg ( ) ) ) continue ; unsigned LastUseCount = <NUM_LIT> ; for ( MachineBasicBlock :: iterator UseI = Def ; UseI !=


<s> return new <ISA_LIT>EmitClauseMarkers (


<s> bool isALU ( const MachineInstr & MI ) const { if ( TII -> isALUInstr ( MI . getOpcode ( ) ) ) return true ; if ( TII -> isVector ( MI ) || TII -> isCubeOp ( MI . getOpcode


<s> bool IsTrivialInst ( MachineInstr & MI ) const { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT>


<s> BuildMI ( MBB , ClauseHead , MBB . findDebugLoc ( ClauseHead ) , TII -> get ( Opcode ) ) . addImm ( Address ++ ) . addImm ( KCacheBanks . empty ( ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . first ) . addImm ( ( KCacheBanks . size ( ) < <NUM_LIT> ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . first ) . addImm ( KCacheBanks . empty ( ) ? <NUM_LIT> : <NUM_LIT> ) . addImm ( ( KCacheBanks . size ( ) < <NUM_LIT> ) ? <NUM_LIT> : <NUM_LIT> ) . addImm ( KCacheBanks . empty ( ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . second ) . addImm ( ( KCacheBanks . size ( ) < <NUM_LIT> ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . second ) . addImm ( AluInstCount ) . addImm ( <NUM_LIT> ) ; return


<s> unsigned NumLiteral = <NUM_LIT> ; for ( MachineInstr :: mop_iterator It = MI . operands_begin ( ) , E = MI . operands_end ( ) ; It != E ; ++ It ) { MachineOperand & MO = * It ; if ( MO . isReg ( ) && MO . getReg ( ) == <STR_LIT> ) ++ NumLiteral ; } return <NUM_LIT>


<s> TII = ST . getInstrInfo ( ) ; for ( MachineFunction :: iterator BB = MF . begin ( ) , BB_E = MF . end ( ) ; BB != BB_E ; ++ BB ) { MachineBasicBlock & MBB = * BB ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; if ( I != MBB . end ( ) && I -> getOpcode ( ) == <STR_LIT> ) continue ; for ( MachineBasicBlock :: iterator E = MBB . end ( ) ; I != E ; ) { if ( isALU ( * I ) ) { auto next = MakeALUClause ( MBB , I ) ; assert ( next


<s> const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = ST . getInstrInfo ( ) ; for ( MachineFunction :: iterator BB = MF . begin ( ) , BB_E = MF . end ( ) ; BB != BB_E ; ++ BB ) { MachineBasicBlock & MBB = * BB ; MachineBasicBlock :: iterator I = MBB .


<s> bool SubstituteKCacheBank ( MachineInstr & MI , std :: vector < std :: pair < unsigned , unsigned >> & CachedConsts , bool UpdateInstr = true ) const { std :: vector < std :: pair < unsigned , unsigned >> UsedKCache ; if ( ! TII -> isALUInstr ( MI . getOpcode ( ) ) && MI . getOpcode ( ) != <STR_LIT> ) return true ; const SmallVectorImpl < std :: pair < MachineOperand * , int64_t >> & Consts = TII -> getSrcs ( MI ) ; assert ( ( TII -> isALUInstr ( MI . getOpcode ( ) ) || MI . getOpcode ( ) == <STR_LIT> ) && <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> , n = Consts . size ( ) ; i < n ; ++ i ) { if ( Consts [ i ] . first -> getReg ( ) != <STR_LIT> ) continue ; unsigned Sel = Consts [ i ] . second ; unsigned Chan = Sel & <NUM_LIT> , Index = ( ( Sel >> <NUM_LIT> ) - <NUM_LIT> ) & <NUM_LIT> ; unsigned KCacheIndex = Index *


<s> llvm :: FunctionPass * llvm :: create<ISA_LIT>EmitClauseMarkers (


<s> return new <ISA_LIT>EmitClauseMarkers ( )


<s> if ( TII -> isALUInstr ( MI -> getOpcode ( ) ) ) return true ; if ( TII -> isVector ( * MI ) || TII -> isCubeOp ( MI -> getOpcode ( ) ) ) return true ; switch ( MI -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT>


<s> } if ( ! canClauseLocalKillFitInClause ( AluInstCount , KCacheBanks , I , E ) ) break ; if ( ! SubstituteKCacheBank ( I , KCacheBanks ) ) break ; AluInstCount += OccupiedDwords ( I ) ; } unsigned Opcode = PushBeforeModifier ? <STR_LIT> : <STR_LIT> ; BuildMI ( MBB , ClauseHead , MBB . findDebugLoc ( ClauseHead ) , TII -> get ( Opcode ) ) . addImm ( Address ++ ) . addImm ( KCacheBanks . empty ( ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . first ) . addImm ( ( KCacheBanks . size ( ) < <NUM_LIT> ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . first ) . addImm ( KCacheBanks . empty ( ) ? <NUM_LIT> : <NUM_LIT> ) . addImm ( ( KCacheBanks . size ( ) < <NUM_LIT> ) ? <NUM_LIT> : <NUM_LIT> ) . addImm ( KCacheBanks . empty ( ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . second ) . addImm ( ( KCacheBanks . size ( ) < <NUM_LIT> ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . second ) . addImm ( AluInstCount ) . addImm ( <NUM_LIT> ) ; return


<s> unsigned NumLiteral = <NUM_LIT> ; for ( MachineInstr :: mop_iterator It = MI -> operands_begin ( ) , E = MI -> operands_end ( ) ; It != E ; ++ It ) { MachineOperand & MO = * It ; if ( MO . isReg ( ) && MO . getReg ( ) == <STR_LIT>


<s> bool runOnMachineFunction ( MachineFunction & MF ) override { TII = static_cast < const <ISA_LIT>InstrInfo * > ( MF . getSubtarget ( ) . getInstrInfo ( ) ) ; for ( MachineFunction :: iterator BB = MF . begin ( ) , BB_E = MF . end ( ) ; BB != BB_E ; ++ BB ) { MachineBasicBlock & MBB = * BB ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; if ( I -> getOpcode ( ) == <STR_LIT> ) continue ; for ( MachineBasicBlock :: iterator E = MBB . end ( ) ; I != E ; ) { if ( isALU ( I ) ) I = MakeALUClause ( MBB ,


<s> UsedKCache . push_back ( std :: pair < unsigned , unsigned > ( <NUM_LIT> , KCacheIndex ) ) ; continue ; } if ( CachedConsts [ <NUM_LIT> ] == BankLine ) { UsedKCache . push_back ( std :: pair < unsigned , unsigned > ( <NUM_LIT> , KCacheIndex ) ) ; continue ; } if ( CachedConsts . size ( ) == <NUM_LIT> ) { CachedConsts . push_back ( BankLine ) ; UsedKCache . push_back ( std :: pair < unsigned , unsigned > ( <NUM_LIT> , KCacheIndex ) ) ; continue ; } if ( CachedConsts [ <NUM_LIT> ] == BankLine ) { UsedKCache . push_back ( std :: pair < unsigned , unsigned > ( <NUM_LIT> , KCacheIndex ) ) ; continue ; } return false ; } if ( ! UpdateInstr ) return true ; for ( unsigned i = <NUM_LIT> , j = <NUM_LIT> , n = Consts . size ( ) ; i < n ; ++ i ) { if ( Consts [ i ] . first -> getReg ( ) != <STR_LIT> ) continue ; switch ( UsedKCache [ j ] . first ) { case <NUM_LIT>


<s> for ( MachineInstr :: const_mop_iterator MOI = Def -> operands_begin ( ) , MOE = Def -> operands_end ( ) ; MOI != MOE ; ++ MOI ) { if ( ! MOI -> isReg ( ) || ! MOI -> isDef ( ) || TRI . isPhysRegLiveAcrossClauses ( MOI -> getReg ( ) ) ) continue ; unsigned LastUseCount = <NUM_LIT> ; for ( MachineBasicBlock :: iterator UseI = Def ; UseI != BBEnd ; ++ UseI ) { AluInstCount += OccupiedDwords ( * UseI ) ; if ( ! SubstituteKCacheBank ( * UseI , KCacheBanks , false )


<s> if ( TII -> isALUInstr ( MI . getOpcode ( ) ) ) return true ; if ( TII -> isVector ( MI ) || TII -> isCubeOp ( MI . getOpcode ( ) ) ) return true ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; default : return false


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true


<s> } if ( TII -> mustBeLastInClause ( I -> getOpcode ( ) ) ) { I ++ ; break ; } if ( ! canClauseLocalKillFitInClause ( AluInstCount , KCacheBanks , I , E ) ) break ; if ( ! SubstituteKCacheBank ( * I , KCacheBanks ) ) break ; AluInstCount += OccupiedDwords ( * I ) ; } unsigned Opcode = PushBeforeModifier ? <STR_LIT> : <STR_LIT> ; BuildMI ( MBB , ClauseHead , MBB . findDebugLoc ( ClauseHead ) , TII -> get ( Opcode ) ) . addImm ( Address ++ ) . addImm ( KCacheBanks . empty ( ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . first ) . addImm ( ( KCacheBanks . size ( ) < <NUM_LIT> ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . first ) . addImm ( KCacheBanks . empty ( ) ? <NUM_LIT> : <NUM_LIT> ) . addImm ( ( KCacheBanks . size ( ) < <NUM_LIT> ) ? <NUM_LIT> : <NUM_LIT> ) . addImm ( KCacheBanks . empty ( ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . second ) . addImm ( ( KCacheBanks . size ( ) < <NUM_LIT> ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . second ) . addImm ( AluInstCount ) . addImm ( <NUM_LIT> ) ; return I


<s> } if ( TII -> isLDSRetInstr ( MI . getOpcode ( ) ) ) return <NUM_LIT> ; if ( TII -> isVector ( MI ) || TII -> isCubeOp ( MI . getOpcode ( ) ) || TII -> isReductionOp ( MI . getOpcode ( ) ) ) return <NUM_LIT> ; unsigned NumLiteral =


<s> BB != BB_E ; ++ BB ) { MachineBasicBlock & MBB = * BB ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; if ( I != MBB . end ( ) && I -> getOpcode ( ) == <STR_LIT> ) continue ; for ( MachineBasicBlock :: iterator E = MBB . end ( ) ; I != E ; ) { if ( isALU ( * I ) ) I = MakeALUClause ( MBB , I ) ; else ++ I


<s> } if ( CachedConsts [ <NUM_LIT> ] == BankLine ) { UsedKCache . push_back ( std :: pair < unsigned , unsigned > ( <NUM_LIT> , KCacheIndex ) ) ; continue ; } return false ; } if ( ! UpdateInstr ) return true ; for ( unsigned i = <NUM_LIT> , j = <NUM_LIT> , n = Consts . size ( ) ; i < n ; ++ i ) { if ( Consts [ i ] . first -> getReg ( ) != <STR_LIT> ) continue ; switch ( UsedKCache [ j ] . first ) { case <NUM_LIT> : Consts [ i ] . first -> setReg ( <STR_LIT> . getRegister ( UsedKCache [ j ] . second ) ) ; break ; case <NUM_LIT> : Consts [ i ] . first -> setReg ( <STR_LIT> . getRegister ( UsedKCache [ j ] . second ) ) ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } j ++ ; } return


<s> if ( I != MBB . end ( ) && I -> getOpcode ( ) == <STR_LIT> ) continue ; for ( MachineBasicBlock :: iterator E = MBB . end ( ) ; I != E ; ) { if ( isALU ( * I ) ) { auto next = MakeALUClause ( MBB , I ) ; assert ( next != I ) ; I = next ; } else


<s> TII = ST . getInstrInfo ( ) ; for ( MachineFunction :: iterator BB = MF . begin ( ) , BB_E = MF . end ( ) ; BB != BB_E ; ++ BB ) { MachineBasicBlock & MBB = * BB ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; if ( I != MBB . end ( ) && I -> getOpcode ( ) == <STR_LIT> ) continue ; for ( MachineBasicBlock :: iterator E = MBB . end ( ) ; I != E ; ) { if ( isALU ( * I ) ) { auto next = MakeALUClause ( MBB , I ) ; assert ( next != I ) ; I = next ; } else ++


<s> MachineBasicBlock & MBB = * BB ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; if ( I != MBB . end ( ) && I -> getOpcode ( ) == <STR_LIT> ) continue ; for ( MachineBasicBlock :: iterator E = MBB . end ( ) ; I != E ; ) { if ( isALU ( * I ) ) { auto next = MakeALUClause ( MBB ,


<s> const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; for ( MachineInstr :: const_mop_iterator MOI = Def -> operands_begin ( ) , MOE = Def -> operands_end ( ) ; MOI != MOE ; ++ MOI ) { if ( ! MOI -> isReg ( ) || ! MOI -> isDef ( ) || TRI . isPhysRegLiveAcrossClauses ( MOI -> getReg (


<s> for ( MachineInstr :: const_mop_iterator MOI = Def -> operands_begin ( ) , MOE = Def -> operands_end ( ) ; MOI != MOE ; ++ MOI ) { if ( ! MOI -> isReg ( ) || ! MOI -> isDef ( ) || TRI . isPhysRegLiveAcrossClauses ( MOI -> getReg ( ) ) ) continue ; unsigned LastUseCount = <NUM_LIT> ; for ( MachineBasicBlock :: iterator UseI = Def ; UseI != BBEnd ; ++ UseI ) { AluInstCount += OccupiedDwords ( * UseI ) ; if ( ! SubstituteKCacheBank ( * UseI , KCacheBanks , false ) ) return false ; if ( AluInstCount >= TII -> getMaxAlusPerClause ( ) ) return false ; if ( UseI -> findRegisterUseOperandIdx ( MOI -> getReg ( )


<s> BuildMI ( MBB , ClauseHead , MBB . findDebugLoc ( ClauseHead ) , TII -> get ( Opcode ) ) . addImm ( Address ++ ) . addImm ( KCacheBanks . empty ( ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . first ) . addImm ( ( KCacheBanks . size ( ) < <NUM_LIT> ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . first ) . addImm ( KCacheBanks . empty ( ) ?

<NUM_LIT> : <NUM_LIT> ) . addImm ( ( KCacheBanks . size ( ) < <NUM_LIT> ) ? <NUM_LIT> : <NUM_LIT> ) . addImm ( KCacheBanks . empty ( ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . second ) . addImm ( ( KCacheBanks . size ( ) < <NUM_LIT> ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . second ) . addImm ( AluInstCount ) . addImm ( <NUM_LIT> ) ;


<s> for ( MachineFunction :: iterator BB = MF . begin ( ) , BB_E = MF . end ( ) ; BB != BB_E ; ++ BB ) { MachineBasicBlock & MBB = * BB ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; if ( I -> getOpcode ( ) == <STR_LIT> ) continue ; for ( MachineBasicBlock :: iterator E = MBB . end ( ) ; I != E ; ) { if ( isALU ( * I ) ) I = MakeALUClause ( MBB ,


create<ISA_LIT>ExpandSpecialInstrsPass ( ) {


<s> StringRef getPassName ( ) const override { return


<s> bool Mask = ( Chan != TRI . getHWRegChan ( DstReg ) ) ; unsigned SubDstReg = <STR_LIT> . getRegister ( ( DstBase * <NUM_LIT> ) + Chan ) ; MachineInstr * BMI = TII -> buildSlotOfVectorInstruction ( MBB , & MI , Chan , SubDstReg ) ; if ( Chan > <NUM_LIT> ) { BMI -> bundleWithPred ( ) ; } if ( Mask ) { TII -> addFlag ( * BMI , <NUM_LIT> , MO_FLAG_MASK ) ; } if ( Chan != <NUM_LIT> ) TII -> addFlag ( * BMI , <NUM_LIT> , MO_FLAG_NOT_LAST ) ; unsigned Opcode = BMI -> getOpcode ( ) ; unsigned Src0 = BMI -> getOperand ( TII -> getOperandIdx ( Opcode , <STR_LIT> :: src0 ) ) . getReg ( ) ; unsigned Src1 = BMI -> getOperand ( TII -> getOperandIdx ( Opcode , <STR_LIT> :: src1 ) ) . getReg ( ) ; ( void ) Src0 ; ( void ) Src1 ; if ( ( TRI . getEncodingValue ( Src0 ) & <NUM_LIT> ) < <NUM_LIT> && ( TRI . getEncodingValue ( Src1 ) & <NUM_LIT> ) < <NUM_LIT> ) assert ( TRI . getHWRegChan ( Src0 ) == TRI . getHWRegChan ( Src1 ) ) ; } MI . eraseFromParent ( ) ; continue ; } } bool IsReduction = TII -> isReductionOp ( MI . getOpcode ( ) ) ; bool IsVector = TII -> isVector ( MI ) ; bool IsCube = TII -> isCubeOp ( MI . getOpcode ( ) ) ; if ( ! IsReduction && ! IsVector && ! IsCube ) { continue ; } for ( unsigned Chan = <NUM_LIT> ; Chan < <NUM_LIT> ; Chan ++ ) { unsigned DstReg = MI . getOperand ( TII -> getOperandIdx ( MI , <STR_LIT> :: dst ) ) . getReg ( ) ; unsigned Src0 = MI . getOperand ( TII -> getOperandIdx ( MI , <STR_LIT> :: src0 ) ) . getReg ( ) ; unsigned Src1 = <NUM_LIT> ; if ( ! IsCube ) { int Src1Idx = TII -> getOperandIdx ( MI , <STR_LIT> :: src1 ) ; if ( Src1Idx != - <NUM_LIT> ) { Src1 = MI . getOperand ( Src1Idx ) . getReg ( ) ; } } if ( IsReduction ) { unsigned SubRegIndex = <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( Chan ) ; Src0 = TRI . getSubReg ( Src0 , SubRegIndex ) ; Src1 = TRI . getSubReg ( Src1 , SubRegIndex ) ; } else if ( IsCube ) { static const int CubeSrcSwz [ ] = { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; unsigned SubRegIndex0 = <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( CubeSrcSwz [ Chan ] ) ; unsigned SubRegIndex1 = <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( CubeSrcSwz [ <NUM_LIT> - Chan ] ) ; Src1 = TRI . getSubReg ( Src0 , SubRegIndex1 ) ; Src0 = TRI . getSubReg ( Src0 , SubRegIndex0 ) ; } bool Mask = false ; bool NotLast = true ; if ( IsCube )


<s> void <ISA_LIT>ExpandSpecialInstrsPass :: SetFlagInNewMI ( MachineInstr * NewMI , const MachineInstr * OldMI , unsigned Op ) { int OpIdx = TII -> getOperandIdx ( * OldMI , Op ) ; if ( OpIdx > - <NUM_LIT> ) { uint64_t Val = OldMI -> getOperand ( OpIdx ) . getImm ( ) ; TII -> setImmOperand ( NewMI , Op


<s> for ( unsigned Chan = <NUM_LIT> ; Chan < <NUM_LIT> ; Chan ++ ) { Register DstReg = MI . getOperand ( TII -> getOperandIdx ( MI , <STR_LIT> :: dst ) ) . getReg ( ) ; Register Src0 = MI . getOperand ( TII -> getOperandIdx ( MI , <STR_LIT> :: src0 ) ) . getReg ( ) ; unsigned Src1 = <NUM_LIT> ; if ( ! IsCube ) { int Src1Idx = TII -> getOperandIdx ( MI , <STR_LIT> :: src1 ) ; if ( Src1Idx != - <NUM_LIT> ) { Src1 = MI . getOperand ( Src1Idx ) . getReg ( ) ; } } if ( IsReduction ) { unsigned SubRegIndex = <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( Chan ) ; Src0 = TRI . getSubReg ( Src0 , SubRegIndex ) ; Src1 = TRI . getSubReg ( Src1 , SubRegIndex ) ; } else if ( IsCube ) { static const int CubeSrcSwz [ ] = { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; unsigned SubRegIndex0 = <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( CubeSrcSwz [ Chan ] ) ; unsigned SubRegIndex1 = <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( CubeSrcSwz [ <NUM_LIT> - Chan ] ) ; Src1 = TRI . getSubReg ( Src0 , SubRegIndex1 ) ; Src0 = TRI . getSubReg ( Src0 , SubRegIndex0 ) ; } bool Mask = false ; bool NotLast = true ; if ( IsCube ) { unsigned SubRegIndex = <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( Chan ) ; DstReg = TRI . getSubReg ( DstReg , SubRegIndex ) ; } else { Mask = ( Chan != TRI . getHWRegChan ( DstReg ) ) ; unsigned DstBase = TRI . getEncodingValue ( DstReg ) & HW_REG_MASK ; DstReg = <STR_LIT> . getRegister ( ( DstBase * <NUM_LIT> ) + Chan ) ; } NotLast = ( Chan != <NUM_LIT> ) ; unsigned Opcode = MI . getOpcode ( ) ; switch ( Opcode ) { case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; default : break ; } MachineInstr * NewMI = TII -> buildDefaultInstruction ( MBB , I , Opcode , DstReg , Src0 , Src1 ) ; if ( Chan != <NUM_LIT> ) NewMI -> bundleWithPred ( ) ; if ( Mask ) { TII -> addFlag ( * NewMI , <NUM_LIT> ,


<s> return new <ISA_LIT>ExpandSpecialInstrsPass (


<s> return new <ISA_LIT>ExpandSpecialInstrsPass ( TM


<s> StackOffset <ISA_LIT>FrameLowering :: getFrameIndexReference ( const MachineFunction & MF , int FI , Register & FrameReg ) const { const MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; const <ISA_LIT>RegisterInfo * RI = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getRegisterInfo ( ) ; FrameReg = RI -> getFrameRegister ( MF ) ; unsigned OffsetBytes = <NUM_LIT> * ( getStackWidth ( MF ) * <NUM_LIT> ) ; int UpperBound = FI == - <NUM_LIT> ? MFI . getNumObjects ( ) : FI ; for ( int i = MFI . getObjectIndexBegin (


<s> StackOffset <ISA_LIT>FrameLowering :: getFrameIndexReference ( const MachineFunction & MF , int FI , Register & FrameReg ) const { const MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; const <ISA_LIT>RegisterInfo * RI = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getRegisterInfo ( ) ; FrameReg = RI -> getFrameRegister (


<s> <ISA_LIT>FrameLowering ( D , StackAl


<s> <ISA_LIT>FrameLowering ( StackDirection D , unsigned StackAl , int LAO ,


<s> const MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; const <ISA_LIT>RegisterInfo * RI = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getRegisterInfo ( ) ; FrameReg = RI -> getFrameRegister ( MF ) ; unsigned OffsetBytes = <NUM_LIT> * ( getStackWidth ( MF ) * <NUM_LIT> ) ; int UpperBound = FI == - <NUM_LIT> ? MFI . getNumObjects ( ) : FI ; for ( int i = MFI . getObjectIndexBegin ( ) ; i < UpperBound ; ++ i ) { OffsetBytes = alignTo ( OffsetBytes , MFI . getObjectAlignment ( i ) ) ; OffsetBytes += MFI . getObjectSize ( i ) ; OffsetBytes = alignTo ( OffsetBytes


<s> while ( I != MBB . begin ( ) && std :: prev ( I ) -> getOpcode ( ) == <STR_LIT> ) { MachineBasicBlock :: iterator PriorI = std :: prev ( I ) ; if ( AllowModify ) I -> removeFromParent ( ) ; I = PriorI ; } MachineInstr & LastInst = * I ; unsigned LastOpc = LastInst . getOpcode ( ) ; if ( I == MBB . begin ( ) || ! isJump ( ( -- I ) -> getOpcode ( ) ) ) { if ( LastOpc == <STR_LIT> ) { TBB = LastInst . getOperand ( <NUM_LIT> ) . getMBB ( ) ; return false ; } else if ( LastOpc == <STR_LIT> ) { auto predSet = I ; while ( ! isPredicateSetter ( predSet -> getOpcode ( ) ) ) { predSet = -- I ; } TBB = LastInst . getOperand ( <NUM_LIT> ) . getMBB ( ) ; Cond . push_back ( predSet -> getOperand ( <NUM_LIT> ) ) ; Cond . push_back ( predSet -> getOperand ( <NUM_LIT> ) ) ; Cond . push_back ( MachineOperand :: CreateReg ( <STR_LIT> , false ) ) ; return false ; } return true ; } MachineInstr & SecondLastInst = * I ; unsigned SecondLastOpc = SecondLastInst . getOpcode (


<s> if ( isBranch ( I -> getOpcode ( ) ) ) return true ; if ( ! isJump ( I -> getOpcode ( ) ) ) { return false ; } while ( I != MBB . begin ( ) && std :: prev ( I ) -> getOpcode ( ) == <STR_LIT> ) { MachineBasicBlock :: iterator PriorI = std :: prev ( I ) ; if ( AllowModify ) I -> removeFromParent ( ) ; I = PriorI ; } MachineInstr & LastInst = * I ; unsigned LastOpc = LastInst . getOpcode ( ) ; if ( I == MBB . begin ( ) || ! isJump ( ( -- I ) -> getOpcode ( ) ) ) { if ( LastOpc == <STR_LIT> ) { TBB = LastInst . getOperand ( <NUM_LIT> ) . getMBB ( ) ; return false ; } else if ( LastOpc == <STR_LIT> ) { auto predSet = I ; while ( ! isPredicateSetter ( predSet -> getOpcode ( ) ) ) { predSet = -- I ; } TBB = LastInst . getOperand ( <NUM_LIT> ) . getMBB ( ) ; Cond . push_back ( predSet -> getOperand ( <NUM_LIT> ) ) ; Cond . push_back ( predSet -> getOperand ( <NUM_LIT> ) ) ; Cond . push_back ( MachineOperand :: CreateReg ( <STR_LIT> , false ) ) ; return false ; } return true ; } MachineInstr & SecondLastInst = * I ; unsigned SecondLastOpc = SecondLastInst . getOpcode ( ) ; if ( SecondLastOpc == <STR_LIT> && LastOpc == <STR_LIT> ) { auto predSet = -- I ; while ( ! isPredicateSetter ( predSet -> getOpcode ( ) ) ) { predSet = -- I ; } TBB = SecondLastInst . getOperand ( <NUM_LIT> ) . getMBB


<s> if ( Src1Reg ) { MIB . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; } MIB . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addReg ( Src0Reg ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( - <NUM_LIT> ) ; if ( Src1Reg ) { MIB . addReg ( Src1Reg ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( - <NUM_LIT> ) ; } MIB . addImm ( <NUM_LIT> ) . addReg ( <STR_LIT> ) . addImm ( <NUM_LIT> )


<s> } MachineInstr * MOVA = buildDefaultInstruction ( * MBB , I , <STR_LIT> , <STR_LIT> , OffsetReg ) ; setImmOperand ( * MOVA , <STR_LIT> :: write , <NUM_LIT> ) ; MachineInstrBuilder Mov = buildDefaultInstruction ( * MBB , I , <STR_LIT> , ValueReg , AddrReg ) . addReg ( <STR_LIT> , RegState :: Implicit | RegState :: Kill ) ; setImmOperand ( * Mov , <STR_LIT> :: src0_rel , <NUM_LIT> ) ; return


<s> case <NUM_LIT> : AddrReg = <STR_LIT> . getRegister ( Address ) ; break ; case <NUM_LIT> : AddrReg = <STR_LIT> . getRegister ( Address ) ; break ; case <NUM_LIT> : AddrReg = <STR_LIT> . getRegister ( Address ) ; break ; case <NUM_LIT> : AddrReg = <STR_LIT> . getRegister ( Address ) ; break ; } MachineInstr * MOVA = buildDefaultInstruction ( * MBB , I , <STR_LIT> , <STR_LIT> , OffsetReg ) ; setImmOperand ( * MOVA , <STR_LIT> :: write , <NUM_LIT> ) ; MachineInstrBuilder Mov = buildDefaultInstruction ( * MBB , I , <STR_LIT> , AddrReg , ValueReg ) . addReg ( <STR_LIT> , RegState :: Implicit | RegState :: Kill ) ; setImmOperand ( * Mov , <STR_LIT> :: dst_rel , <NUM_LIT> ) ; return


<s> VectorComponents = <NUM_LIT> ; } else if ( ( <STR_LIT> . contains ( DestReg ) || <STR_LIT> . contains ( DestReg ) ) && ( <STR_LIT> . contains ( SrcReg ) || <STR_LIT> . contains ( SrcReg ) ) ) { VectorComponents = <NUM_LIT> ; } if ( VectorComponents > <NUM_LIT> ) { for ( unsigned I = <NUM_LIT> ; I < VectorComponents ; I ++ ) { unsigned SubRegIndex = <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( I ) ; buildDefaultInstruction ( MBB , MI , <STR_LIT> , RI . getSubReg ( DestReg , SubRegIndex ) , RI . getSubReg ( SrcReg , SubRegIndex ) ) . addReg ( DestReg , RegState :: Define | RegState :: Implicit ) ; } } else { MachineInstr * NewMI = buildDefaultInstruction ( MBB , MI , <STR_LIT> , DestReg , SrcReg ) ; NewMI -> getOperand ( getOperandIdx ( * NewMI , <STR_LIT> :: src0 ) ) .


<s> return MI . findRegisterDefOperandIdx ( <STR_LIT> , false , false , &


<s> bool <ISA_LIT>InstrInfo :: DefinesPredicate ( MachineInstr & MI , std :: vector < MachineOperand > & Pred ) const { return isPredicateSetter ( MI . getOpcode (


<s> int Index = RI . getEncodingValue ( Reg ) & <NUM_LIT> ; if ( Reg == <STR_LIT> ) { Result . push_back ( std :: make_pair ( Index , <NUM_LIT> ) ) ; } if ( PV . find ( Reg ) != PV . end ( ) ) { Result . push_back ( std :: make_pair ( <NUM_LIT> , <NUM_LIT> ) ) ; continue ; } if ( Index > <NUM_LIT> ) { ConstCount ++ ; Result . push_back ( DummyPair ) ; continue ; } unsigned Chan = RI . getHWRegChan ( Reg ) ; Result . push_back ( std :: make_pair ( Index , Chan ) ) ; } for ( ; i < <NUM_LIT> ; ++ i ) Result . push_back ( DummyPair ) ; return Result


<s> if ( It -> getOpcode ( ) == <STR_LIT> || It -> getOpcode ( ) == <STR_LIT> )


<s> bool <ISA_LIT>InstrInfo :: FindSwizzleForVectorSlot ( const std :: vector < std :: vector < std :: pair < int , unsigned >> > & IGSrcs , std :: vector < <ISA_LIT>InstrInfo :: BankSwizzle > & SwzCandidate , const std :: vector < std :: pair < int , unsigned >> & TransSrcs , <ISA_LIT>InstrInfo :: BankSwizzle TransSwz ) const { unsigned ValidUpTo = <NUM_LIT> ; do { ValidUpTo = isLegalUpTo ( IGSrcs , SwzCandidate , TransSrcs , TransSwz ) ; if ( ValidUpTo == IGSrcs . size ( ) ) return true ; } while ( NextPossibleSolution ( SwzCandidate , ValidUpTo )


<s> if ( Literals . size ( ) > <NUM_LIT> ) return false ; if ( Src . first -> getReg ( ) == <STR_LIT> ) Consts . push_back ( Src . second ) ; if ( <STR_LIT> . contains ( Src . first -> getReg ( ) ) || <STR_LIT> . contains ( Src . first -> getReg ( ) ) ) { unsigned Index = RI . getEncodingValue ( Src . first -> getReg ( )


<s> const TargetRegisterClass * IndirectRC = getIndirectAddrRegClass ( ) ; for ( std :: pair < unsigned , unsigned > LI : MRI . liveins ( ) ) { unsigned Reg = LI . first ; if ( TargetRegisterInfo :: isVirtualRegister ( Reg ) || ! IndirectRC -> contains ( Reg ) ) continue ; unsigned RegIndex ; unsigned RegEnd ; for ( RegIndex = <NUM_LIT> , RegEnd = IndirectRC -> getNumRegs ( ) ; RegIndex != RegEnd ; ++ RegIndex ) { if ( IndirectRC -> getRegister ( RegIndex ) == Reg ) break ; } Offset = std :: max ( Offset , ( int ) RegIndex ) ; } return Offset +


<s> const MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; if ( MFI . hasVarSizedObjects ( ) ) { return - <NUM_LIT> ; } if ( MFI . getNumObjects ( ) == <NUM_LIT> ) { return - <NUM_LIT> ; } const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const <ISA_LIT>FrameLowering * TFL = ST . getFrameLowering


<s> const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const <ISA_LIT>FrameLowering * TFL = ST . getFrameLowering ( ) ; unsigned IgnoredFrameReg ; Offset = TFL -> getFrameIndexReference ( MF , - <NUM_LIT> , IgnoredFrameReg ) ; return getIndirectIndexBegin ( MF )


<s> unsigned int <ISA_LIT>InstrInfo :: getPredicationCost ( const MachineInstr & ) const { return


<s> unsigned int <ISA_LIT>InstrInfo :: getPredicationCost ( const


<s> } case <ISA_LIT>InstrInfo :: ALU_VEC_021_SCL_122 : { unsigned Cycles [ <NUM_LIT> ] = { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; return Cycles [ Op ] ; } case <ISA_LIT>InstrInfo :: ALU_VEC_120_SCL_212 : { unsigned Cycles [ <NUM_LIT> ] = { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; return Cycles [ Op ] ; } case <ISA_LIT>InstrInfo :: ALU_VEC_102_SCL_221 : { unsigned Cycles [ <NUM_LIT> ] = { <NUM_LIT> , <NUM_LIT>


<s> if ( Cond . empty ( ) ) { BuildMI ( & MBB , DL , get ( <STR_LIT> ) ) . addMBB ( TBB ) ; return <NUM_LIT> ; } else { MachineInstr * PredSet = findFirstPredicateSetterFrom ( MBB , MBB . end ( ) ) ; assert ( PredSet && <STR_LIT> ) ; addFlag ( * PredSet , <NUM_LIT> , MO_FLAG_PUSH ) ; PredSet -> getOperand ( <NUM_LIT> ) . setImm ( Cond [ <NUM_LIT> ] . getImm ( ) ) ; BuildMI ( & MBB , DL , get ( <STR_LIT> ) ) . addMBB ( TBB ) . addReg ( <STR_LIT> , RegState :: Kill ) ; MachineBasicBlock :: iterator CfAlu = FindLastAluClause ( MBB ) ; if ( CfAlu == MBB . end ( ) ) return <NUM_LIT> ; assert ( CfAlu -> getOpcode ( ) == <STR_LIT> ) ; CfAlu -> setDesc ( get (


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return


<s> static bool isJump ( unsigned Opcode


<s> return isLD<ISA_LIT>nstr ( Opcode ) && getOperandIdx ( Opcode , <STR_LIT> ::


<s> const std :: vector < std :: pair < int , unsigned >> & Srcs = Swizzle ( IGSrcs [ i ] , Swz [ i ] ) ; for ( unsigned j = <NUM_LIT> ; j < <NUM_LIT> ; j ++ ) { const std :: pair < int , unsigned > & Src = Srcs [ j ] ; if ( Src . first < <NUM_LIT> || Src . first == <NUM_LIT> ) continue ; if ( Src . first == GET_REG_INDEX ( RI . getEncodingValue ( <STR_LIT> ) ) ) { if ( Swz [ i ] != <ISA_LIT>InstrInfo :: ALU_VEC_012_SCL_210 && Swz [ i ] != <ISA_LIT>InstrInfo :: ALU_VEC_021_SCL_122 ) { return false ; } continue ; } if ( Vector [ Src . second ] [ j ] < <NUM_LIT> ) Vector [ Src . second ] [ j ] = Src . first ; if ( Vector [ Src . second ] [ j ] != Src . first ) return i ; } } for ( unsigned i = <NUM_LIT> , e = TransSrcs . size ( ) ; i < e ; ++ i ) { const std :: pair < int , unsigned > & Src = TransSrcs [


<s> if ( MI . getParent ( ) -> begin ( ) != MachineBasicBlock :: const_iterator ( MI ) ) return false ; return MI . getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> && MI . getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> ; } else if ( isVector ( MI ) ) { return false ; } else { return TargetInstrInfo :: isPredicable


<s> } else if ( MI . getOpcode ( ) == <STR_LIT> ) { if ( MI . getParent ( ) -> begin ( ) != MachineBasicBlock :: const_iterator ( MI ) ) return false ; return MI . getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> && MI . getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> ; } else if ( isVector ( MI ) ) { return false ; } else


<s> if ( idx < <NUM_LIT> ) return false ; unsigned Reg = MI . getOperand ( idx ) . getReg ( ) ; switch ( Reg ) { default : return


<s> bool <ISA_LIT>InstrInfo :: isProfitableToDupForIfCvt ( MachineBasicBlock & MBB ,

unsigned NumCycles , BranchProbability Probability ) const {


<s> bool <ISA_LIT>InstrInfo :: isProfitableToDupForIfCvt ( MachineBasicBlock & MBB , unsigned NumCycles , BranchProbability Probability ) const { return true


<s> bool <ISA_LIT>InstrInfo :: isProfitableToIfCvt ( MachineBasicBlock & MBB , unsigned NumCycles , unsigned ExtraPredCycles , BranchProbability Probability ) const { return true


<s> return isVectorOnly ( MI . getOpcode (


<s> bool <ISA_LIT>InstrInfo :: isVectorOnly ( const MachineInstr & MI ) const { return isVectorOnly ( MI . getOpcode


<s> return true ; } if ( MI . getOpcode ( ) == <STR_LIT> ) { MI . getOperand ( getOperandIdx ( MI , <STR_LIT> :: pred_sel_X ) ) . setReg ( Pred [ <NUM_LIT> ] . getReg ( ) ) ; MI . getOperand ( getOperandIdx ( MI , <STR_LIT> :: pred_sel_Y ) ) . setReg ( Pred [ <NUM_LIT> ] . getReg ( ) ) ; MI . getOperand ( getOperandIdx ( MI , <STR_LIT> :: pred_sel_Z ) ) . setReg ( Pred [ <NUM_LIT> ] . getReg ( ) ) ; MI . getOperand ( getOperandIdx ( MI , <STR_LIT> :: pred_sel_W ) ) . setReg ( Pred [ <NUM_LIT> ] . getReg ( ) ) ; MachineInstrBuilder MIB ( * MI . getParent ( ) -> getParent ( ) , MI ) ; MIB . addReg ( <STR_LIT> , RegState :: Implicit ) ; return true ; } if ( PIdx != - <NUM_LIT> ) { MachineOperand & PMO = MI .


<s> if ( ! I -> isReg ( ) || ! I -> isUse ( ) || TargetRegisterInfo :: isVirtualRegister ( I -> getReg ( ) ) ) continue ; if ( <STR_LIT> . contains ( I -> getReg


<s> I != E ; ++ I ) { if ( ! I -> isReg ( ) || ! I -> isUse ( ) || TargetRegisterInfo :: isVirtualRegister ( I ->


<s> case <ISA_LIT>InstrInfo :: ALU_VEC_021_SCL_122 : std :: swap ( Src [ <NUM_LIT> ] , Src [ <NUM_LIT> ] ) ; break ; case <ISA_LIT>InstrInfo :: ALU_VEC_102_SCL_221 : std :: swap ( Src [ <NUM_LIT> ] , Src [ <NUM_LIT> ] ) ; break ; case <ISA_LIT>InstrInfo :: ALU_VEC_120_SCL_212 : std :: swap ( Src [ <NUM_LIT> ] , Src [ <NUM_LIT> ] ) ; std :: swap ( Src [ <NUM_LIT> ] , Src [ <NUM_LIT> ] ) ; break ; case <ISA_LIT>InstrInfo :: ALU_VEC_201 : std :: swap ( Src [ <NUM_LIT> ] , Src [ <NUM_LIT> ] ) ; std :: swap ( Src [ <NUM_LIT> ] , Src [


<s> bool <ISA_LIT>InstrInfo :: usesAddressRegister ( MachineInstr & MI ) const { return MI . findRegisterUseOperandIdx ( <STR_LIT> , false , &


<s> MachineBasicBlock :: iterator PriorI = std :: prev ( I ) ; if ( AllowModify ) I -> removeFromParent ( ) ; I = PriorI ; } MachineInstr * LastInst = I ; unsigned LastOpc = LastInst -> getOpcode ( ) ; if ( I == MBB . begin ( ) || ! isJump ( static_cast < MachineInstr * > ( -- I ) -> getOpcode ( ) ) ) { if ( LastOpc == <STR_LIT> ) { TBB = LastInst -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; return false ; } else if ( LastOpc == <STR_LIT> ) { MachineInstr * predSet = I ; while ( ! isPredicateSetter ( predSet -> getOpcode ( ) ) ) { predSet = -- I ; } TBB = LastInst -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; Cond . push_back ( predSet -> getOperand ( <NUM_LIT> ) ) ; Cond . push_back ( predSet -> getOperand ( <NUM_LIT> ) ) ; Cond . push_back ( MachineOperand :: CreateReg ( <STR_LIT> , false ) ) ; return false ; } return true ; } MachineInstr * SecondLastInst = I ; unsigned SecondLastOpc = SecondLastInst -> getOpcode ( ) ; if ( SecondLastOpc == <STR_LIT> && LastOpc == <STR_LIT> ) { MachineInstr * predSet = -- I ; while ( ! isPredicateSetter ( predSet -> getOpcode ( ) ) ) { predSet = -- I ; } TBB = SecondLastInst -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; FBB = LastInst -> getOperand ( <NUM_LIT>


<s> MachineInstrBuilder MIB = BuildMI ( MBB , I , MBB . findDebugLoc ( I ) , get ( Opcode ) , DstReg ) ; if ( Src1Reg ) { MIB . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; } MIB . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addReg ( Src0Reg ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( - <NUM_LIT> ) ; if ( Src1Reg


<s> MachineInstr * MOVA = buildDefaultInstruction ( * MBB , I , <STR_LIT> , <STR_LIT> , OffsetReg ) ; setImmOperand ( MOVA , <STR_LIT> :: write , <NUM_LIT> ) ; MachineInstrBuilder Mov = buildDefaultInstruction ( * MBB , I , <STR_LIT> , AddrReg , ValueReg ) . addReg (

<STR_LIT> , RegState :: Implicit | RegState :: Kill ) ;


<s> MachineInstrBuilder <ISA_LIT>InstrInfo :: buildIndirectWrite ( MachineBasicBlock * MBB , MachineBasicBlock :: iterator I , unsigned ValueReg , unsigned Address , unsigned OffsetReg , unsigned AddrChan ) const { unsigned AddrReg ; switch ( AddrChan ) { default : llvm_unreachable ( <STR_LIT> ) ; case <NUM_LIT> : AddrReg = <STR_LIT> . getRegister ( Address ) ; break ; case <NUM_LIT> : AddrReg = <STR_LIT> . getRegister ( Address ) ; break ; case <NUM_LIT> : AddrReg = <STR_LIT> . getRegister ( Address ) ; break ; case <NUM_LIT> : AddrReg = <STR_LIT> . getRegister ( Address


<s> unsigned <ISA_LIT>InstrInfo :: calculateIndirectAddress ( unsigned RegIndex , unsigned Channel ) const { assert ( Channel == <NUM_LIT> ) ; return RegIndex


<s> bool <ISA_LIT>InstrInfo :: canBeConsideredALU ( const MachineInstr * MI ) const { if ( isALUInstr ( MI -> getOpcode ( ) ) ) return true ; if ( isVector ( * MI ) || isCubeOp ( MI -> getOpcode ( ) ) ) return true ; switch ( MI -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case


<s> } if ( VectorComponents > <NUM_LIT> ) { for ( unsigned I = <NUM_LIT> ; I < VectorComponents ; I ++ ) { unsigned SubRegIndex = RI . getSubRegFromChannel ( I ) ; buildDefaultInstruction ( MBB , MI , <STR_LIT> , RI . getSubReg ( DestReg , SubRegIndex ) , RI . getSubReg ( SrcReg , SubRegIndex ) ) . addReg ( DestReg , RegState :: Define | RegState :: Implicit ) ; } } else { MachineInstr * NewMI = buildDefaultInstruction ( MBB , MI , <STR_LIT> , DestReg ,


<s> bool <ISA_LIT>InstrInfo :: expandPostRAPseudo ( MachineBasicBlock :: iterator MI ) const { switch ( MI -> getOpcode ( ) ) { default : return <ISA_LIT>InstrInfo :: expandPostRAPseudo ( MI ) ; case <STR_LIT> : case <STR_LIT> : buildIndirectRead ( MI -> getParent ( ) , MI , MI -> getOperand ( <NUM_LIT> ) . getReg ( ) , RI . getHWRegIndex ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) , MI -> getOperand ( <NUM_LIT> ) . getReg ( ) , RI . getHWRegChan ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) ; break ; case <STR_LIT> : case <STR_LIT>


<s> static MachineInstr * findFirstPredicateSetterFrom ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator I ) { while ( I != MBB . begin ( ) ) { -- I ; MachineInstr * MI


<s> It != E ; ++ It ) { if ( It -> getOpcode ( ) == <STR_LIT> || It -> getOpcode ( ) == <STR_LIT> ) return


<s> bool <ISA_LIT>InstrInfo :: FindSwizzleForVectorSlot ( const std :: vector < std :: vector < std :: pair < int , unsigned > > > & IGSrcs , std :: vector < <ISA_LIT>InstrInfo :: BankSwizzle > & SwzCandidate , const std ::

vector < std :: pair < int , unsigned > > & TransSrcs , <ISA_LIT>InstrInfo :: BankSwizzle TransSwz ) const {


<s> bool <ISA_LIT>InstrInfo :: FindSwizzleForVectorSlot ( const std :: vector < std :: vector < std :: pair < int , unsigned > > > & IGSrcs , std :: vector < <ISA_LIT>InstrInfo :: BankSwizzle > & SwzCandidate , const std :: vector < std :: pair < int , unsigned > > & TransSrcs , <ISA_LIT>InstrInfo :: BankSwizzle TransSwz ) const { unsigned ValidUpTo = <NUM_LIT> ; do { ValidUpTo = isLegalUpTo ( IGSrcs , SwzCandidate , TransSrcs , TransSwz ) ; if ( ValidUpTo == IGSrcs . size ( ) ) return true ; } while ( NextPossibleSolution ( SwzCandidate , ValidUpTo ) ) ; return false


<s> const TargetRegisterClass * <ISA_LIT>InstrInfo :: getIndirectAddrRegClass ( ) const { return & <STR_LIT>


<s> case <ISA_LIT>InstrInfo :: ALU_VEC_120_SCL_212 : { unsigned Cycles [ <NUM_LIT> ] = { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; return Cycles [ Op ] ; } case <ISA_LIT>InstrInfo :: ALU_VEC_102_SCL_221 : { unsigned Cycles [ <NUM_LIT> ] = { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; return Cycles [ Op ] ; } default : llvm_unreachable ( <STR_LIT>


<s> static unsigned getTransSwizzle ( <ISA_LIT>InstrInfo :: BankSwizzle Swz , unsigned Op ) { switch ( Swz ) { case <ISA_LIT>InstrInfo :: ALU_VEC_012_SCL_210 : { unsigned Cycles [ <NUM_LIT> ] = { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; return Cycles [ Op ] ; } case <ISA_LIT>InstrInfo :: ALU_VEC_021_SCL_122


<s> bool <ISA_LIT>InstrInfo :: hasInstrModifiers ( unsigned Opcode ) const { unsigned TargetFlags = get ( Opcode ) .


<s> assert ( PredSet && <STR_LIT> ) ; addFlag ( PredSet , <NUM_LIT> , MO_FLAG_PUSH ) ; PredSet -> getOperand ( <NUM_LIT> ) . setImm ( Cond [ <NUM_LIT> ] . getImm ( ) ) ; BuildMI ( & MBB , DL , get ( <STR_LIT> ) ) . addMBB ( TBB ) . addReg ( <STR_LIT> , RegState :: Kill ) ; MachineBasicBlock :: iterator CfAlu = FindLastAluClause ( MBB ) ; if ( CfAlu == MBB . end ( ) ) return <NUM_LIT> ; assert ( CfAlu -> getOpcode ( ) == <STR_LIT> ) ; CfAlu -> setDesc ( get ( <STR_LIT> ) ) ; return <NUM_LIT> ; } } else { MachineInstr * PredSet = findFirstPredicateSetterFrom ( MBB ,


<s> bool <ISA_LIT>InstrInfo :: isALUInstr ( unsigned Opcode )


<s> return Opcode == <STR_LIT> || Opcode == <STR_LIT> ||


<s> for ( unsigned i = <NUM_LIT> , e = TransOps . size ( ) ; i < e ; ++ i ) { const std :: pair < int , unsigned > & Src = TransOps [ i ] ; unsigned Cycle = getTransSwizzle ( TransSwz , i ) ; if ( Src . first < <NUM_LIT> ) continue ; if ( ConstCount > <NUM_LIT> && Cycle ==


<s> bool <ISA_LIT>InstrInfo :: isCubeOp ( unsigned Opcode ) const { switch ( Opcode ) { default : return false ; case <STR_LIT> : case


<s> bool <ISA_LIT>InstrInfo :: isExport ( unsigned


<s> static bool isJump ( unsigned Opcode ) { return Opcode == <STR_LIT> ||


<s> bool <ISA_LIT>InstrInfo :: isLDSNoRetInstr ( unsigned


<s> return isLD<ISA_LIT>nstr ( Opcode ) && getOperandIdx ( Opcode , <STR_LIT>


<s> for ( MachineInstr :: const_mop_iterator I = MBBI -> operands_begin ( ) , E =

MBBI -> operands_end ( ) ; I != E ; ++ I ) {


<s> continue ; } if ( Vector [ Src . second ] [ j ] < <NUM_LIT> ) Vector [ Src . second ] [ j ] = Src . first ; if ( Vector [ Src . second ] [ j ] != Src . first ) return i ; } } for ( unsigned i = <NUM_LIT> , e = TransSrcs . size ( ) ; i < e ; ++ i ) { const std :: pair < int , unsigned > & Src = TransSrcs [ i ] ; unsigned Cycle = getTransSwizzle ( TransSwz , i ) ; if ( Src . first < <NUM_LIT> ) continue ; if ( Src . first == <NUM_LIT> ) continue ; if ( Vector [ Src . second ] [ Cycle ] < <NUM_LIT> ) Vector [ Src . second ] [


<s> for ( unsigned i = <NUM_LIT> , e = TransSrcs . size ( ) ; i < e ; ++ i ) { const std :: pair < int , unsigned > & Src = TransSrcs [ i ] ; unsigned Cycle = getTransSwizzle ( TransSwz , i ) ; if ( Src . first < <NUM_LIT> ) continue ; if ( Src . first == <NUM_LIT> ) continue ; if ( Vector [ Src . second ] [ Cycle ] < <NUM_LIT> ) Vector [ Src . second ] [ Cycle ] = Src . first ; if ( Vector [ Src . second ] [ Cycle ] != Src . first ) return IGSrcs . size ( ) - <NUM_LIT> ; } return IGSrcs . size (


<s> bool <ISA_LIT>InstrInfo :: isMov ( unsigned Opcode ) const { switch ( Opcode ) { default : return


<s> bool <ISA_LIT>InstrInfo :: isPlaceHolderOpcode (


<s> return false ; } else if ( MI -> getOpcode ( ) == <STR_LIT> ) { if ( MI -> getParent ( ) -> begin ( ) != MachineBasicBlock :: iterator ( MI ) ) return false ; if ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> || MI -> getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ) return false ; return


<s> bool <ISA_LIT>InstrInfo :: isProfitableToDupForIfCvt ( MachineBasicBlock & MBB , unsigned NumCyles

, BranchProbability Probability ) const {


<s> bool <ISA_LIT>InstrInfo :: isProfitableToUnpredicate ( MachineBasicBlock & TMBB ,


<s> bool isRegisterStore ( const MachineInstr & MI ) const { return get ( MI . getOpcode ( ) ) . TSFlags


<s> bool isRegisterStore ( const MachineInstr & MI )


<s> bool <ISA_LIT>InstrInfo :: isTrig ( const MachineInstr & MI ) const { return get ( MI . getOpcode (


<s> bool <ISA_LIT>InstrInfo :: isVector ( const MachineInstr


<s> return get ( MI . getOpcode ( ) ) . TSFlags


<s> bool <ISA_LIT>InstrInfo :: isVectorOnly ( const MachineInstr


<s> bool <ISA_LIT>InstrInfo :: mustBeLastInClause ( unsigned Opcode ) const { switch ( Opcode ) { case <STR_LIT> : case <STR_LIT> : return


<s> for ( unsigned i = ResetIdx + <NUM_LIT> , e = SwzCandidate . size ( ) ; i < e ; i ++ ) { SwzCandidate [ i ] = <ISA_LIT>InstrInfo :: ALU_VEC_012_SCL_210 ; } if ( ResetIdx == - <NUM_LIT> ) return false ; int NextSwizzle = SwzCandidate [ ResetIdx ] + <NUM_LIT> ; SwzCandidate [ ResetIdx ] = ( <ISA_LIT>InstrInfo ::


<s> MI -> getOperand ( getOperandIdx ( * MI , <STR_LIT> :: pred_sel_W ) ) . setReg ( Pred [ <NUM_LIT> ] . getReg ( ) ) ; MachineInstrBuilder MIB ( * MI -> getParent ( ) -> getParent ( ) , MI ) ; MIB . addReg ( <STR_LIT> , RegState :: Implicit ) ; return true ; } if ( PIdx != - <NUM_LIT> ) { MachineOperand & PMO = MI -> getOperand ( PIdx ) ; PMO . setReg ( Pred [ <NUM_LIT> ] . getReg ( ) ) ; MachineInstrBuilder MIB ( * MI -> getParent ( ) ->


<s> bool <ISA_LIT>InstrInfo :: readsLDSSrcReg ( const MachineInstr * MI ) const { if ( ! isALUInstr ( MI -> getOpcode ( ) ) ) { return false ; } for ( MachineInstr :: const_mop_iterator I = MI -> operands_begin ( ) , E = MI -> operands_end ( ) ; I != E


<s> unsigned StackWidth = TFL -> getStackWidth ( MF ) ; int End = getIndirectIndexEnd ( MF ) ; if ( End == - <NUM_LIT> ) return ; for ( int Index = getIndirectIndexBegin ( MF ) ; Index <= End ; ++ Index ) { unsigned SuperReg = <STR_LIT> . getRegister ( Index ) ; Reserved . set ( SuperReg ) ; for ( unsigned Chan = <NUM_LIT> ; Chan < StackWidth ; ++ Chan ) { unsigned Reg = <STR_LIT> . getRegister ( ( <NUM_LIT> * Index ) + Chan ) ; Reserved . set (


<s> int End = getIndirectIndexEnd ( MF ) ; if ( End == - <NUM_LIT> ) return ; for ( int Index = getIndirectIndexBegin ( MF ) ; Index <= End ; ++ Index ) { unsigned SuperReg = <STR_LIT> . getRegister ( Index ) ; Reserved . set ( SuperReg ) ; for ( unsigned Chan = <NUM_LIT> ; Chan < StackWidth ; ++ Chan ) { unsigned Reg = <STR_LIT> . getRegister ( ( <NUM_LIT> * Index ) + Chan ) ; Reserved . set ( Reg )


<s> MO . setImm ( OPCODE_IS_NOT_ZERO_INT ) ; break ; case OPCODE_IS_NOT_ZERO_INT : MO . setImm ( OPCODE_IS_ZERO_INT ) ; break ; case OPCODE_IS_ZERO : MO . setImm ( OPCODE_IS_NOT_ZERO ) ; break ; case OPCODE_IS_NOT_ZERO : MO . setImm ( OPCODE_IS_ZERO ) ; break ; default : return true ; } MachineOperand & MO2 = Cond [ <NUM_LIT> ] ; switch ( MO2 . getReg ( ) ) { case <STR_LIT> : MO2 . setReg ( <STR_LIT> ) ; break ; case <STR_LIT> : MO2 . setReg ( <STR_LIT> ) ; break ; default : return true


<s> bool <ISA_LIT>InstrInfo :: SubsumesPredicate ( ArrayRef < MachineOperand > Pred1 , ArrayRef < MachineOperand > Pred2 ) const { return


<s> bool <ISA_LIT>InstrInfo :: usesAddressRegister (


<s> bool <ISA_LIT>InstrInfo :: usesAddressRegister ( MachineInstr * MI ) const { return MI -> findRegisterUseOperandIdx ( <STR_LIT> ) !=


<s> bool <ISA_LIT>InstrInfo :: usesVertexCache ( const MachineInstr * MI ) const { const MachineFunction * MF = MI -> getParent ( ) -> getParent ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF ->

getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;


<s> const MachineFunction * MF = MI -> getParent ( ) -> getParent ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return MFI -> getShaderType ( ) != ShaderType :: COMPUTE && usesVertexCache ( MI -> getOpcode


<s> default : llvm_unreachable ( <STR_LIT> ) ; case <NUM_LIT> : AddrReg = <STR_LIT> . getRegister ( Address ) ; break ; case <NUM_LIT> : AddrReg = <STR_LIT> . getRegister ( Address ) ; break ; case <NUM_LIT> : AddrReg = <STR_LIT> . getRegister ( Address ) ; break ; case <NUM_LIT> : AddrReg = <STR_LIT> . getRegister ( Address ) ; break ; } MachineInstr * MOVA = buildDefaultInstruction ( * MBB , I , <STR_LIT> , <STR_LIT> , OffsetReg ) ; setImmOperand ( * MOVA , <STR_LIT> :: write , <NUM_LIT> ) ; MachineInstrBuilder Mov = buildDefaultInstruction ( * MBB , I , <STR_LIT> , ValueReg , AddrReg ) . addReg ( <STR_LIT> , RegState :: Implicit | RegState :: Kill ) ; setImmOperand ( * Mov , <STR_LIT> :: src0_rel , <NUM_LIT> ) ; return Mov


<s> case <NUM_LIT> : AddrReg = <STR_LIT> . getRegister ( Address ) ; break ; } MachineInstr * MOVA = buildDefaultInstruction ( * MBB , I , <STR_LIT> , <STR_LIT> , OffsetReg ) ; setImmOperand ( * MOVA , <STR_LIT> :: write , <NUM_LIT> ) ; MachineInstrBuilder Mov = buildDefaultInstruction ( * MBB , I , <STR_LIT> , AddrReg , ValueReg ) . addReg ( <STR_LIT> , RegState :: Implicit |


<s> } MachineInstr * MOVA = buildDefaultInstruction ( * MBB , I , <STR_LIT> , <STR_LIT> , OffsetReg ) ; setImmOperand ( * MOVA , <STR_LIT> :: write , <NUM_LIT> ) ; MachineInstrBuilder Mov = buildDefaultInstruction ( * MBB , I , <STR_LIT> , AddrReg , ValueReg ) . addReg ( <STR_LIT> , RegState :: Implicit | RegState :: Kill ) ; setImmOperand ( * Mov , <STR_LIT> :: dst_rel


<s> VectorComponents = <NUM_LIT> ; } else if ( ( <STR_LIT> . contains ( DestReg ) || <STR_LIT> . contains ( DestReg ) ) && ( <STR_LIT> . contains ( SrcReg ) || <STR_LIT> . contains ( SrcReg ) ) ) { VectorComponents = <NUM_LIT> ; } if ( VectorComponents > <NUM_LIT> ) { for ( unsigned I = <NUM_LIT> ; I < VectorComponents ; I ++ ) { unsigned SubRegIndex = RI . getSubRegFromChannel ( I ) ; buildDefaultInstruction ( MBB , MI , <STR_LIT> , RI . getSubReg ( DestReg , SubRegIndex ) , RI . getSubReg ( SrcReg , SubRegIndex ) ) . addReg ( DestReg , RegState :: Define | RegState :: Implicit ) ; } }


<s> return MI . findRegisterDefOperandIdx ( <STR_LIT> )


<s> bool <ISA_LIT>InstrInfo :: definesAddressRegister ( MachineInstr


<s> buildMovInstr ( MBB , MI , getIndirectAddrRegClass ( ) -> getRegister ( Address ) , MI . getOperand ( ValOpIdx ) . getReg ( ) ) ; } else { buildIndirectWrite ( MBB , MI , MI . getOperand ( ValOpIdx ) . getReg ( ) , calculateIndirectAddress ( RegIndex , Channel ) , OffsetReg ) ; } } else { return false ; } MBB -> erase ( MI ) ; return true ; } case <STR_LIT> : case <STR_LIT> : buildIndirectRead ( MI . getParent ( ) , MI , MI . getOperand ( <NUM_LIT> ) . getReg ( ) , RI . getHWRegIndex ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) , MI . getOperand ( <NUM_LIT> ) . getReg ( ) , RI . getHWRegChan ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) ; break ; case <STR_LIT> : case <STR_LIT> : buildIndirectWrite ( MI . getParent ( ) , MI , MI . getOperand ( <NUM_LIT> ) . getReg ( ) , RI . getHWRegIndex ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) , MI . getOperand ( <NUM_LIT> ) . getReg ( ) , RI


<s> MachineInstr & MI = * MIs [ i ] ; if ( ! isALUInstr ( MI . getOpcode ( ) ) ) continue ; ArrayRef < std :: pair < MachineOperand * , int64_t >> Srcs = getSrcs ( MI ) ; for ( const auto & Src : Srcs ) { if ( Src . first -> getReg ( ) == <STR_LIT> ) Literals . insert ( Src . second ) ; if ( Literals . size ( ) > <NUM_LIT> ) return false ; if ( Src . first -> getReg ( ) == <STR_LIT> ) Consts . push_back ( Src . second ) ; if ( <STR_LIT> . contains ( Src . first -> getReg ( ) ) || <STR_LIT> . contains ( Src . first -> getReg ( ) ) ) { unsigned Index = RI . getEncodingValue ( Src . first -> getReg ( ) ) & <NUM_LIT> ; unsigned Chan = RI . getHWRegChan ( Src . first -> getReg ( ) ) ; Consts . push_back ( ( Index << <NUM_LIT> ) | Chan ) ; } } } return fitsConstReadLimitations ( Consts


<s> ValidSwizzle . clear ( ) ; unsigned ConstCount ; BankSwizzle TransBS = ALU_VEC_012_SCL_210 ; for ( unsigned i = <NUM_LIT> , e = IG . size ( ) ; i < e ; ++ i ) { IGSrcs . push_back ( ExtractSrcs ( * IG [ i ] , PV , ConstCount ) ) ; unsigned Op = getOperandIdx ( IG [ i ] -> getOpcode ( ) , <STR_LIT> :: bank_swizzle ) ; ValidSwizzle . push_back ( ( <ISA_LIT>InstrInfo :: BankSwizzle ) IG [ i ] -> getOperand ( Op ) . getImm ( ) ) ; } std :: vector < std :: pair < int , unsigned > > TransOps ; if ( ! isLastAluTrans ) return FindSwizzleForVectorSlot ( IGSrcs , ValidSwizzle , TransOps , TransBS ) ; TransOps = std :: move ( IGSrcs . back ( ) ) ; IGSrcs . pop_back ( ) ; ValidSwizzle . pop_back (


<s> if ( MI . getOpcode ( ) == <STR_LIT> ) { static const unsigned OpTable [ <NUM_LIT> ] [ <NUM_LIT> ] = { { <STR_LIT> :: src0_X , <STR_LIT> :: src0_sel_X } , { <STR_LIT> :: src0_Y , <STR_LIT> :: src0_sel_Y } , { <STR_LIT> :: src0_Z , <STR_LIT> :: src0_sel_Z } , { <STR_LIT> :: src0_W , <STR_LIT> :: src0_sel_W } , { <STR_LIT> :: src1_X , <STR_LIT> :: src1_sel_X } , { <STR_LIT> :: src1_Y , <STR_LIT> :: src1_sel_Y } , { <STR_LIT> :: src1_Z , <STR_LIT> :: src1_sel_Z } , { <STR_LIT> :: src1_W , <STR_LIT> :: src1_sel_W } , } ; for ( unsigned j = <NUM_LIT> ; j < <NUM_LIT> ; j ++ ) { MachineOperand & MO = MI . getOperand ( getOperandIdx ( MI . getOpcode ( ) , OpTable [ j ] [ <NUM_LIT> ] ) ) ; unsigned Reg = MO . getReg ( ) ; if ( Reg == <STR_LIT> ) { MachineOperand & Sel = MI . getOperand ( getOperandIdx ( MI . getOpcode ( ) , OpTable [ j ] [ <NUM_LIT> ] ) ) ; Result . push_back ( std :: make_pair ( & MO , Sel . getImm ( ) ) ) ; continue ; } } return Result ; } static const unsigned OpTable [ <NUM_LIT> ] [ <NUM_LIT> ] = { { <STR_LIT> :: src0 , <STR_LIT> :: src0_sel } , { <STR_LIT> :: src1 , <STR_LIT> :: src1_sel } , { <STR_LIT> :: src2 , <STR_LIT> :: src2_sel } , } ; for ( unsigned j = <NUM_LIT> ; j < <NUM_LIT> ; j ++ ) { int SrcIdx = getOperandIdx ( MI . getOpcode ( ) , OpTable [ j ] [ <NUM_LIT> ] ) ; if ( SrcIdx < <NUM_LIT> ) break ; MachineOperand & MO = MI . getOperand ( SrcIdx ) ; unsigned Reg = MO . getReg ( ) ; if ( Reg == <STR_LIT>


<s> MachineInstr * PredSet = findFirstPredicateSetterFrom ( MBB , MBB . end ( ) ) ; assert ( PredSet && <STR_LIT> ) ; addFlag ( * PredSet , <NUM_LIT> , MO_FLAG_PUSH ) ; PredSet -> getOperand ( <NUM_LIT> ) . setImm ( Cond [ <NUM_LIT> ] . getImm ( ) ) ; BuildMI ( & MBB , DL , get ( <STR_LIT> ) ) . addMBB ( TBB ) . addReg ( <STR_LIT> , RegState :: Kill ) ; BuildMI ( & MBB , DL , get ( <STR_LIT> ) ) . addMBB ( FBB ) ; MachineBasicBlock :: iterator CfAlu = FindLastAluClause ( MBB ) ; if ( CfAlu == MBB .


<s> bool <ISA_LIT>InstrInfo :: isPredicable ( MachineInstr & MI ) const { if ( MI . getOpcode ( ) == <STR_LIT> ) { return false ; } else if ( MI . getOpcode ( ) == <STR_LIT> ) { if ( MI . getParent ( ) -> begin ( ) != MachineBasicBlock :: iterator ( MI ) ) return false ; return MI . getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> && MI . getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> ; } else if ( isVector ( MI )


<s> if ( idx < <NUM_LIT> ) return false ; unsigned Reg = MI . getOperand ( idx ) .


<s> bool <ISA_LIT>InstrInfo :: isPredicated ( const MachineInstr & MI ) const { int idx = MI . findFirstPredOperandIdx ( ) ; if ( idx < <NUM_LIT> ) return false ; unsigned Reg = MI . getOperand ( idx ) . getReg ( ) ; switch ( Reg ) { default : return false ; case <STR_LIT> : case <STR_LIT> : case


<s> if ( MI . getOpcode ( ) == <STR_LIT> ) { MI . getOperand ( <NUM_LIT> ) . setImm ( <NUM_LIT> ) ; return true ; } if ( MI . getOpcode ( ) == <STR_LIT> ) { MI . getOperand ( getOperandIdx ( MI , <STR_LIT> :: pred_sel_X ) ) . setReg ( Pred [ <NUM_LIT> ] . getReg ( ) ) ; MI . getOperand ( getOperandIdx ( MI , <STR_LIT> :: pred_sel_Y ) ) . setReg ( Pred [ <NUM_LIT> ] . getReg ( ) ) ; MI . getOperand ( getOperandIdx ( MI , <STR_LIT> :: pred_sel_Z ) ) . setReg ( Pred [ <NUM_LIT> ] . getReg ( ) ) ; MI . getOperand ( getOperandIdx ( MI , <STR_LIT> :: pred_sel_W ) ) . setReg ( Pred [ <NUM_LIT> ] . getReg ( ) ) ; MachineInstrBuilder MIB ( * MI . getParent ( ) -> getParent ( ) , MI ) ; MIB . addReg ( <STR_LIT> , RegState :: Implicit ) ; return true ; } if ( PIdx != - <NUM_LIT> ) { MachineOperand & PMO = MI . getOperand ( PIdx ) ; PMO . setReg ( Pred [ <NUM_LIT> ] . getReg ( ) ) ; MachineInstrBuilder MIB ( * MI . getParent ( ) -> getParent ( ) , MI ) ; MIB . addReg ( <STR_LIT> , RegState :: Implicit ) ; return true


<s> bool <ISA_LIT>InstrInfo :: usesAddressRegister ( MachineInstr & MI ) const { return MI . findRegisterUseOperandIdx ( <STR_LIT> )


<s> bool <ISA_LIT>InstrInfo :: usesTextureCache ( const MachineInstr & MI ) const { const MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; return ( <STR_LIT> ( MF -> getFunction ( ) -> getCallingConv ( ) ) && usesVertexCache ( MI . getOpcode ( ) ) ) || usesTextureCache ( MI . getOpcode


<s> if ( VectorComponents > <NUM_LIT> ) { for ( unsigned I = <NUM_LIT> ; I < VectorComponents ; I ++ ) { unsigned SubRegIndex = <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( I ) ; buildDefaultInstruction ( MBB , MI , <STR_LIT> , RI . getSubReg ( DestReg , SubRegIndex ) , RI . getSubReg ( SrcReg , SubRegIndex ) ) . addReg ( DestReg , RegState :: Define | RegState :: Implicit ) ; } } else { MachineInstr * NewMI = buildDefaultInstruction ( MBB , MI , <STR_LIT> , DestReg , SrcReg ) ; NewMI -> getOperand ( getOperandIdx ( * NewMI , <STR_LIT>


<s> BankSwizzle TransBS = ALU_VEC_012_SCL_210 ; for ( unsigned i = <NUM_LIT> , e = IG . size ( ) ; i < e ; ++ i ) { IGSrcs . push_back ( ExtractSrcs ( * IG [ i ] , PV , ConstCount ) ) ; unsigned Op = getOperandIdx ( IG [ i ] -> getOpcode ( ) , <STR_LIT> :: bank_swizzle ) ; ValidSwizzle . push_back ( ( <ISA_LIT>InstrInfo :: BankSwizzle ) IG [ i ] -> getOperand ( Op ) . getImm ( ) ) ; } std :: vector < std :: pair < int , unsigned >> TransOps ; if ( ! isLastAluTrans ) return FindSwizzleForVectorSlot ( IGSrcs , ValidSwizzle , TransOps , TransBS ) ; TransOps = std :: move ( IGSrcs . back ( ) ) ; IGSrcs . pop_back ( ) ; ValidSwizzle . pop_back ( ) ; static const <ISA_LIT>InstrInfo :: BankSwizzle TransSwz [ ] = { ALU_VEC_012_SCL_210 , ALU_VEC_021_SCL_122 , ALU_VEC_120_SCL_212 , ALU_VEC_102_SCL_221 } ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT>


<s> if ( Register :: isVirtualRegister ( Reg ) || ! IndirectRC -> contains ( Reg ) ) continue ; unsigned RegIndex ; unsigned RegEnd ; for ( RegIndex = <NUM_LIT> , RegEnd = IndirectRC -> getNumRegs ( ) ; RegIndex != RegEnd ; ++ RegIndex ) { if ( IndirectRC -> getRegister ( RegIndex ) == Reg ) break ; } Offset = std :: max ( Offset , ( int