<s> if ( MI -> getIterator ( ) != RegionEnd ) { BB -> remove ( MI ) ; BB -> insert ( RegionEnd , MI ) ; LIS -> handleMove ( * MI , true ) ; } for ( auto & Op : MI -> operands ( ) ) if ( Op . isReg ( ) && Op . isDef ( ) ) Op . setIsUndef ( false ) ; RegisterOperands RegOpers ; RegOpers . collect ( * MI , * TRI , MRI , ShouldTrackLaneMasks , false ) ; if ( ShouldTrackLaneMasks ) { SlotIndex SlotIdx = LIS -> getInstructionIndex ( * MI ) . getRegSlot ( ) ; RegOpers . adjustLaneLiveness ( * LIS , MRI , SlotIdx , MI ) ; } else { RegOpers . detectDeadDefs ( * MI , * LIS ) ; } RegionEnd = MI -> getIterator ( ) ; ++ RegionEnd ;

DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ;


<s> int NewRegs = countPopulation ( NewMask . getAsInteger ( ) ) - countPopulation ( PrevMask . getAsInteger ( ) ) ;

if ( SRI -> isSGPRReg ( MRI , Reg ) ) SGPRs += NewRegs ;


<s> static inline void setMask ( const MachineRegisterInfo & MRI , const <ISA_LIT>RegisterInfo * SRI , unsigned Reg , LaneBitmask & PrevMask , LaneBitmask NewMask , unsigned & SGPRs , unsigned & VGPRs ) { int NewRegs = countPopulation ( NewMask . getAsInteger ( ) ) - countPopulation ( PrevMask . getAsInteger ( ) ) ; if ( SRI -> isSGPRReg ( MRI , Reg ) ) SGPRs += NewRegs ; if ( SRI -> isVGPR ( MRI , Reg ) ) VGPRs += NewRegs ; assert ( ( int ) SGPRs >= <NUM_LIT> && ( int ) VGPRs >= <NUM_LIT> ) ; PrevMask = NewMask ;


<s> void GCNScheduleDAGMILive :: finalizeSchedule ( ) { GCNMaxOccupancySchedStrategy & S = ( GCNMaxOccupancySchedStrategy & ) * SchedImpl ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; LiveIns . resize ( Regions . size ( ) ) ; Pressure . resize ( Regions . size ( ) ) ; do { Stage ++ ; RegionIdx = <NUM_LIT> ; MachineBasicBlock * MBB = nullptr ; if ( Stage > <NUM_LIT> ) { if ( ! LIS || StartingOccupancy <= MinOccupancy ) break ; DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT> ) ; S . setTargetOccupancy ( MinOccupancy ) ; } for ( auto Region : Regions ) { RegionBegin = Region . first ; RegionEnd = Region . second ; if ( RegionBegin -> getParent ( ) != MBB ) {


<s> do { Stage ++ ; RegionIdx = <NUM_LIT> ; MachineBasicBlock * MBB = nullptr ; if ( Stage > <NUM_LIT> ) { if ( ! LIS || StartingOccupancy <= MinOccupancy ) break ; DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT> ) ; S . setTargetOccupancy ( MinOccupancy ) ; } for ( auto Region : Regions ) { RegionBegin = Region . first ; RegionEnd = Region . second ; if ( RegionBegin -> getParent ( ) != MBB ) { if ( MBB ) finishBlock ( ) ; MBB = RegionBegin -> getParent ( ) ; startBlock ( MBB ) ; if ( Stage == <NUM_LIT> ) computeBlockPressure ( MBB ) ; } unsigned NumRegionInstrs = std :: distance ( begin ( ) , end ( ) ) ; enterRegion ( MBB , begin ( ) , end ( ) , NumRegionInstrs ) ; if ( begin ( ) == end ( ) || begin ( ) == std :: prev ( end ( ) ) ) { exitRegion ( ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> ) ;

DEBUG ( dbgs ( ) << MF . getName ( ) << <STR_LIT> << printMBBReference ( * MBB ) << <STR_LIT> << MBB -> getName ( ) << <STR_LIT> << * begin ( ) << <STR_LIT> ;


<s> const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;

unsigned MinRegOccupancy = std :: min ( ST . getOccupancyWithNumSGPRs ( SGPRs ) , ST . getOccupancyWithNumVGPRs ( VGPRs ) ) ;


<s> } ScheduleDAGMILive :: schedule ( ) ; Regions [ RegionIdx ] = std :: make_pair ( RegionBegin , RegionEnd ) ; if ( ! LIS ) return ; GCNMaxOccupancySchedStrategy & S = ( GCNMaxOccupancySchedStrategy & ) * SchedImpl ; auto PressureAfter = getRealRegPressure ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; PressureAfter . print ( dbgs ( ) ) ) ; if ( PressureAfter . getSGPRNum ( ) <= S . SGPRCriticalLimit && PressureAfter . getVGPRNum ( ) <= S . VGPRCriticalLimit ) { Pressure [ RegionIdx ] = PressureAfter ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; return ; } unsigned WavesAfter = getMaxWaves ( PressureAfter . getSGPRNum ( ) , PressureAfter . getVGPRNum ( ) , MF ) ; unsigned WavesBefore = getMaxWaves ( PressureBefore . getSGPRNum ( ) , PressureBefore . getVGPRNum ( ) , MF ) ; DEBUG ( dbgs ( ) << <STR_LIT> << WavesBefore << <STR_LIT> << WavesAfter << <STR_LIT> ) ; unsigned NewOccupancy = std :: max ( WavesAfter , WavesBefore ) ; if ( NewOccupancy < MinOccupancy ) { MinOccupancy = NewOccupancy ;

DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT> ) ;


<s> ScheduleDAGMILive :: schedule ( ) ; Regions [ RegionIdx ] = std :: make_pair ( RegionBegin , RegionEnd ) ; if ( ! LIS ) return ; GCNMaxOccupancySchedStrategy & S = ( GCNMaxOccupancySchedStrategy & ) * SchedImpl ; auto PressureAfter = getRealRegPressure ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; PressureAfter . print ( dbgs ( ) ) ) ; if ( PressureAfter . getSGPRNum ( ) <= S . SGPRCriticalLimit && PressureAfter . getVGPRNum ( ) <= S . VGPRCriticalLimit ) { Pressure [ RegionIdx ] = PressureAfter ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; return ; } unsigned WavesAfter = getMaxWaves ( PressureAfter . getSGPRNum ( ) , PressureAfter . getVGPRNum ( ) , MF ) ; unsigned WavesBefore = getMaxWaves ( PressureBefore . getSGPRNum ( ) , PressureBefore . getVGPRNum ( ) , MF ) ; DEBUG ( dbgs ( ) << <STR_LIT> << WavesBefore << <STR_LIT> << WavesAfter << <STR_LIT> ) ; unsigned NewOccupancy = std :: max ( WavesAfter , WavesBefore ) ; if ( NewOccupancy < MinOccupancy ) { MinOccupancy = NewOccupancy ;


<s> unsigned WavesBefore = std :: min ( Occ , PressureBefore . getOccupancy ( ST ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << WavesBefore << <STR_LIT> << WavesAfter << <STR_LIT> ) ; unsigned NewOccupancy = std :: max ( WavesAfter , WavesBefore ) ; if ( WavesAfter < WavesBefore && WavesAfter < MinOccupancy && WavesAfter >= MFI . getMinAllowedOccupancy ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MFI . getMinAllowedOccupancy ( ) << <STR_LIT> ) ; NewOccupancy = WavesAfter ; } if ( NewOccupancy < MinOccupancy ) { MinOccupancy = NewOccupancy ; MFI . limitOccupancy ( MinOccupancy ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT> ) ; } if ( WavesAfter >= MinOccupancy ) { unsigned TotalVGPRs = <STR_LIT> :: getAddressableNumVGPRs ( & ST ) ; unsigned TotalSGPRs = <STR_LIT> :: getAddressableNumSGPRs ( & ST ) ; if ( WavesAfter > MFI . getMinWavesPerEU ( ) || PressureAfter . less ( ST , PressureBefore ) || ( TotalVGPRs >= PressureAfter . getVGPRNum ( ) && TotalSGPRs >= PressureAfter . getSGPRNum ( ) ) ) { Pressure [ RegionIdx ] = PressureAfter ; return ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; RegionEnd = RegionBegin ; for ( MachineInstr * MI : Unsched ) { if ( MI -> isDebugInstr ( ) ) continue ; if ( MI -> getIterator ( ) != RegionEnd ) { BB -> remove ( MI ) ; BB -> insert ( RegionEnd , MI ) ; if ( ! MI -> isDebugInstr ( ) ) LIS -> handleMove ( * MI , true ) ; } for ( auto & Op : MI -> operands ( ) ) if ( Op . isReg ( ) && Op . isDef ( ) ) Op . setIsUndef ( false ) ; RegisterOperands RegOpers ; RegOpers . collect ( * MI , * TRI , MRI , ShouldTrackLaneMasks , false ) ; if ( ! MI -> isDebugInstr ( ) ) { if ( ShouldTrackLaneMasks ) { SlotIndex SlotIdx = LIS -> getInstructionIndex ( * MI ) . getRegSlot ( ) ; RegOpers . adjustLaneLiveness ( * LIS , MRI , SlotIdx , MI ) ; } else { RegOpers . detectDeadDefs ( * MI , * LIS ) ; } } RegionEnd = MI -> getIterator ( ) ;


<s> if ( RescheduleRegions . none ( ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } if ( Stage == ClusteredLowOccupancyReschedule ) { if ( StartingOccupancy <= MinOccupancy ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT> ) ; } } if ( Stage == UnclusteredReschedule ) SavedMutations . swap ( Mutations ) ; for ( auto Region : Regions ) { if ( ( Stage == UnclusteredReschedule && ! RescheduleRegions [ RegionIdx ] ) || ( Stage == ClusteredLowOccupancyReschedule && ! RegionsWithClusters [ RegionIdx ] && ! RegionsWithHighRP [ RegionIdx ] ) ) { ++ RegionIdx ; continue ; } RegionBegin = Region . first ; RegionEnd = Region . second ; if ( RegionBegin -> getParent ( ) != MBB ) { if ( MBB ) finishBlock ( ) ; MBB = RegionBegin -> getParent ( ) ; startBlock ( MBB ) ; if ( Stage == InitialSchedule ) computeBlockPressure ( MBB ) ;


<s> RegionEnd = Region . second ; if ( RegionBegin -> getParent ( ) != MBB ) { if ( MBB ) finishBlock ( ) ; MBB = RegionBegin -> getParent ( ) ; startBlock ( MBB ) ; if ( Stage == InitialSchedule ) computeBlockPressure ( MBB ) ; } unsigned NumRegionInstrs = std :: distance ( begin ( ) , end ( ) ) ; enterRegion ( MBB , begin ( ) , end ( ) , NumRegionInstrs ) ; if ( begin ( ) == end ( ) || begin ( ) == std :: prev ( end ( ) ) ) { exitRegion ( ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << MF . getName ( ) << <STR_LIT> << printMBBReference ( * MBB ) << <STR_LIT> << MBB -> getName ( ) << <STR_LIT> << * begin ( ) << <STR_LIT> ; if ( RegionEnd != MBB -> end ( ) ) dbgs ( ) << * RegionEnd ; else dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> << NumRegionInstrs << '\n' ) ; schedule ( ) ; exitRegion ( ) ; ++ RegionIdx ; }


<s> GCNScheduleDAGMILive :: GCNScheduleDAGMILive ( MachineSchedContext * C , std :: unique_ptr < MachineSchedStrategy > S ) : ScheduleDAGMILive ( C , std :: move ( S ) ) , ST ( MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ) , MFI ( * MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ) , StartingOccupancy ( std :: min ( ST . getOccupancyWithLocalMemSize ( MFI . getLDSSize ( ) , MF . getFunction ( ) ) , MFI . getMaxWavesPerEU ( ) ) ) , MinOccupancy ( StartingOccupancy ) , Stage ( <NUM_LIT> ) , RegionIdx ( <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << StartingOccupancy << <STR_LIT> ) ;


<s> } ScheduleDAGMILive :: schedule ( ) ; Regions [ RegionIdx ] = std :: make_pair ( RegionBegin , RegionEnd ) ; if ( ! LIS ) return ; GCNMaxOccupancySchedStrategy & S = ( GCNMaxOccupancySchedStrategy & ) * SchedImpl ; auto PressureAfter = getRealRegPressure ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; PressureAfter . print ( dbgs ( ) ) ) ; if ( PressureAfter . getSGPRNum ( ) <= S . SGPRCriticalLimit && PressureAfter . getVGPRNum ( ) <= S . VGPRCriticalLimit ) { Pressure [ RegionIdx ] = PressureAfter ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return ; } unsigned WavesAfter = getMaxWaves ( PressureAfter . getSGPRNum ( ) , PressureAfter . getVGPRNum ( ) , MF ) ; unsigned WavesBefore = getMaxWaves ( PressureBefore . getSGPRNum ( ) , PressureBefore . getVGPRNum ( ) , MF ) ; WavesAfter = std :: min ( WavesAfter , MFI . getMaxWavesPerEU ( ) ) ; WavesBefore = std :: min ( WavesBefore , MFI . getMaxWavesPerEU ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << WavesBefore << <STR_LIT> << WavesAfter << <STR_LIT> ) ; unsigned NewOccupancy = std :: max ( WavesAfter , WavesBefore ) ; if ( NewOccupancy < MinOccupancy ) { MinOccupancy = NewOccupancy ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT> ) ; } if ( WavesAfter >= WavesBefore ) { Pressure [ RegionIdx ] = PressureAfter ; return ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; RegionEnd = RegionBegin ; for ( MachineInstr * MI : Unsched ) { if ( MI -> isDebugInstr ( ) ) continue ; if ( MI -> getIterator ( ) != RegionEnd ) { BB -> remove ( MI ) ; BB -> insert ( RegionEnd , MI ) ; if ( ! MI -> isDebugInstr ( ) ) LIS -> handleMove ( * MI , true ) ; } for ( auto & Op : MI -> operands ( ) ) if ( Op . isReg ( ) && Op . isDef ( ) ) Op . setIsUndef ( false ) ; RegisterOperands RegOpers ; RegOpers . collect ( * MI , * TRI , MRI , ShouldTrackLaneMasks , false ) ; if ( ! MI -> isDebugInstr ( ) ) { if ( ShouldTrackLaneMasks ) { SlotIndex SlotIdx = LIS -> getInstructionIndex ( * MI ) . getRegSlot ( ) ; RegOpers . adjustLaneLiveness ( * LIS , MRI , SlotIdx , MI ) ; } else {

RegOpers . detectDeadDefs ( * MI , * LIS ) ;


<s> TempTracker . getUpwardPressure ( SU -> getInstr ( ) , Pressure , MaxPressure ) ; } int NewSGPRPressure = Pressure [ SRI -> getSGPRPressureSet ( ) ] ; int NewVGPRPressure = Pressure [ SRI -> getVGPRPressureSet ( ) ] ; const int MaxVGPRPressureInc = <NUM_LIT> ; bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >= VGPRExcessLimit ; bool ShouldTrackSGPRs = ! ShouldTrackVGPRs && SGPRPressure >= SGPRExcessLimit ; const int ErrorMargin = <NUM_LIT> ; VGPRExcessLimit -= ErrorMargin ; SGPRExcessLimit -= ErrorMargin ; if ( ShouldTrackVGPRs && NewVGPRPressure >= VGPRExcessLimit ) { Cand . RPDelta . Excess = PressureChange ( SRI -> getVGPRPressureSet ( ) ) ; Cand . RPDelta . Excess . setUnitInc ( NewVGPRPressure - VGPRExcessLimit ) ; }

if ( ShouldTrackSGPRs && NewSGPRPressure >= SGPRExcessLimit ) {


<s> Register Reg = Register :: index2VirtReg ( I ) ; if ( ! LIS -> hasInterval ( Reg ) ) continue ; if ( ! SRI -> isVGPRClass ( MRI . getRegClass ( Reg ) ) || ! MRI . hasOneDef ( Reg ) || ! MRI . hasOneUse ( Reg ) ) continue ; if ( HighRPLiveIns . find ( Reg ) == HighRPLiveIns . end ( ) ) continue ; MachineInstr * Def = MRI . getOneDef ( Reg ) -> getParent ( ) ; if ( ! Def || Def -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != <NUM_LIT> || ! isTriviallyReMaterializable ( * Def , AA ) ) continue ; MachineInstr * UseI = & * MRI . use_instr_begin ( Reg ) ; if ( Def -> getParent ( ) == UseI -> getParent ( ) ) continue ;

RematerializableInsts . push_back ( std :: make_pair ( Def , UseI ) ) ;


<s> void GCNScheduleDAGMILive :: collectRematerializableInstructions ( unsigned HighRPIdx ) { const <ISA_LIT>RegisterInfo * SRI = static_cast < const <ISA_LIT>RegisterInfo * > ( TRI ) ; const GCNRPTracker :: LiveRegSet & HighRPLiveIns = LiveIns [ HighRPIdx ] ; for ( unsigned I = <NUM_LIT> , E = MRI . getNumVirtRegs ( ) ; I != E ; ++ I ) { Register Reg = Register :: index2VirtReg ( I ) ; if ( ! LIS -> hasInterval ( Reg ) ) continue ; if ( ! SRI -> isVGPRClass ( MRI . getRegClass ( Reg ) ) || ! MRI . hasOneDef ( Reg ) || ! MRI . hasOneUse ( Reg ) ) continue ; if ( HighRPLiveIns . find ( Reg ) == HighRPLiveIns . end ( ) ) continue ; MachineInstr * Def = MRI . getOneDef ( Reg ) -> getParent ( ) ; if ( ! Def || Def -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != <NUM_LIT> || ! isTriviallyReMaterializable ( * Def , AA ) ) continue ; MachineInstr * UseI = & * MRI . use_instr_begin ( Reg ) ; if ( Def -> getParent ( ) == UseI -> getParent ( ) ) continue ; RematerializableInsts . push_back ( std :: make_pair ( Def , UseI ) ) ; }


<s> if ( RegionsWithMinOcc . count ( ) != <NUM_LIT> || Regions . size ( ) == <NUM_LIT> ) break ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const TargetInstrInfo * TII = MF . getSubtarget ( ) . getInstrInfo ( ) ; if ( ST . computeOccupancy ( MF . getFunction ( ) , MFI . getLDSSize ( ) ) == MinOccupancy ) break ; static_assert ( LastStage == PreRARematerialize , <STR_LIT> ) ; unsigned HighRPIdx = RegionsWithMinOcc . find_first ( ) ; collectRematerializableInstructions ( HighRPIdx ) ; if ( RematerializableInsts . empty ( ) || ! sinkTriviallyRematInsts ( ST , TII , HighRPIdx ) ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT> ) ; } } if ( Stage == UnclusteredReschedule ) SavedMutations . swap ( Mutations ) ; for ( auto Region : Regions ) { if ( ( ( Stage == UnclusteredReschedule || Stage == PreRARematerialize ) && ! RescheduleRegions [ RegionIdx ] ) || ( Stage == ClusteredLowOccupancyReschedule && ! RegionsWithClusters [ RegionIdx ] && ! RegionsWithHighRP [ RegionIdx ] ) ) { ++ RegionIdx ; continue ; } RegionBegin = Region . first ; RegionEnd = Region . second ; if ( RegionBegin -> getParent ( ) != MBB ) { if ( MBB ) finishBlock ( ) ; MBB = RegionBegin -> getParent ( ) ; startBlock ( MBB ) ; if ( Stage == InitialSchedule ) computeBlockPressure ( MBB ) ; } unsigned NumRegionInstrs = std :: distance ( begin ( ) , end ( ) ) ; enterRegion ( MBB , begin ( ) , end ( ) , NumRegionInstrs ) ; if ( begin ( ) == end ( ) || begin ( ) == std :: prev ( end ( ) ) ) { exitRegion ( ) ; ++ RegionIdx ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << MF . getName ( ) << <STR_LIT> << printMBBReference ( * MBB ) << <STR_LIT> << MBB -> getName ( ) << <STR_LIT> << * begin ( ) << <STR_LIT> ; if ( RegionEnd != MBB -> end ( ) ) dbgs ( ) << * RegionEnd ;


<s> GCNRegPressure RPAfterSinking = RPT . moveMaxPressure ( ) ; ImproveOccupancy = RPAfterSinking . getOccupancy ( ST ) ; if ( ImproveOccupancy > MinOccupancy ) break ; } if ( ImproveOccupancy <= MinOccupancy ) { for ( auto & Entry : InsertedMIToOldDef ) { MachineInstr * MI = Entry . first ; MachineInstr * OldMI = Entry . second ; Register Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; LIS -> RemoveMachineInstrFromMaps ( * MI ) ; MI -> eraseFromParent ( ) ; OldMI -> clearRegisterDeads ( Reg ) ; LIS -> removeInterval ( Reg ) ; LIS -> createAndComputeVirtRegInterval ( Reg ) ; } return false ; } for ( auto & Entry : InsertedMIToOldDef ) { MachineInstr * MI = Entry . first ; MachineInstr * OldMI = Entry . second ; updateRegionBoundaries ( OldMI , nullptr , true ) ; BBLiveInMap . erase ( OldMI ) ; Register Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; LIS -> RemoveMachineInstrFromMaps ( * OldMI ) ; OldMI -> eraseFromParent ( ) ; LIS -> removeInterval ( Reg ) ; LIS -> createAndComputeVirtRegInterval ( Reg ) ; MachineBasicBlock :: iterator InsertPos = std :: next ( MachineBasicBlock :: iterator ( MI ) ) ; updateRegionBoundaries ( InsertPos , MI ) ; } LiveIns [ HighRPIdx ] . copyFrom ( NewLiveIns ) ; MBBLiveIns . erase ( Regions [ HighRPIdx ] . first -> getParent ( ) ) ; GCNDownwardRPTracker RPTracker ( * LIS ) ; RPTracker . advance ( Regions [ HighRPIdx ] . first , Regions [ HighRPIdx ] . second , & LiveIns [ HighRPIdx ] ) ; Pressure [ HighRPIdx ] = RPTracker . moveMaxPressure ( ) ; <ISA_LIT>MachineFunctionInfo & MFI = * MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;

MFI . increaseOccupancy ( MF , ++ MinOccupancy ) ;


<s> if ( MI -> getParent ( ) != Regions [ I ] . first -> getParent ( ) ) return ; if ( Removing && MI == Regions [ I ] . first && MI == Regions [ I ] . second ) { Regions [ I ] = std :: make_pair ( MI -> getParent ( ) -> end ( ) , MI -> getParent ( ) -> end ( ) ) ; return ; } if ( MI == Regions [ I ] . first ) { if ( Removing ) Regions [ I ] = std :: make_pair ( std :: next ( MI ) , Regions [ I ] . second ) ; else Regions [ I ] = std :: make_pair ( MachineBasicBlock :: iterator ( NewMI ) , Regions [ I ] . second ) ; return ; } if ( Removing && MI == Regions [ I ] . second ) { Regions [ I ] = std :: make_pair ( Regions [ I ] . first , std :: prev ( MI ) ) ; return ; }


<s> -- CurRegion ; auto I = MBB -> begin ( ) ; auto LiveInIt = MBBLiveIns . find ( MBB ) ; if ( LiveInIt != MBBLiveIns . end ( ) ) { auto LiveIn = std :: move ( LiveInIt -> second ) ; RPTracker . reset ( * MBB -> begin ( ) , & LiveIn ) ; MBBLiveIns . erase ( LiveInIt ) ; } else { auto & Rgn = Regions [ CurRegion ] ; I = Rgn . first ; auto * NonDbgMI = & * skipDebugInstructionsForward ( Rgn . first , Rgn . second ) ; auto LRS = BBLiveInMap . lookup ( NonDbgMI ) ; assert ( isEqual ( getLiveRegsBefore ( * NonDbgMI , * LIS ) , LRS ) ) ;


<s> void GCNScheduleDAGMILive :: computeBlockPressure ( const MachineBasicBlock * MBB ) { GCNDownwardRPTracker RPTracker ( * LIS ) ; const MachineBasicBlock * OnlySucc = nullptr ; if ( MBB -> succ_size ( ) == <NUM_LIT> && ! ( * MBB -> succ_begin ( ) ) -> empty ( ) ) { SlotIndexes * Ind = LIS -> getSlotIndexes ( ) ; if ( Ind -> getMBBStartIdx ( MBB ) < Ind -> getMBBStartIdx ( * MBB -> succ_begin ( ) ) ) OnlySucc = * MBB -> succ_begin ( ) ; } size_t CurRegion = RegionIdx ; for ( size_t E = Regions . size ( ) ; CurRegion != E ; ++ CurRegion ) if ( Regions [ CurRegion ] . first -> getParent ( ) != MBB ) break ; -- CurRegion ; auto I = MBB -> begin ( ) ; auto LiveInIt = MBBLiveIns . find ( MBB ) ; if ( LiveInIt != MBBLiveIns . end ( ) ) { auto LiveIn = std :: move ( LiveInIt -> second ) ; RPTracker . reset ( * MBB -> begin ( ) , & LiveIn ) ; MBBLiveIns . erase ( LiveInIt ) ; } else { auto & Rgn = Regions [ CurRegion ] ; I = Rgn . first ; auto * NonDbgMI = & * skipDebugInstructionsForward ( Rgn . first , Rgn . second ) ; auto LRS = BBLiveInMap . lookup ( NonDbgMI ) ; assert ( isEqual ( getLiveRegsBefore ( * NonDbgMI , * LIS ) , LRS ) ) ; RPTracker . reset ( * I , & LRS ) ; } for ( ; ; ) { I = RPTracker . getNext ( ) ; if ( Regions [ CurRegion ] . first == I ) { LiveIns [ CurRegion ] = RPTracker . getLiveRegs ( ) ; RPTracker . clearMaxPressure ( ) ; } if ( Regions [ CurRegion ] . second == I ) { Pressure [ CurRegion ] = RPTracker . moveMaxPressure ( ) ; if ( CurRegion -- == RegionIdx ) break ; } RPTracker . advanceToNext ( ) ; RPTracker . advanceBeforeNext ( ) ; } if ( OnlySucc ) { if ( I != MBB -> end ( ) ) { RPTracker . advanceToNext ( ) ; RPTracker . advance ( MBB -> end ( ) ) ; } RPTracker . reset ( * OnlySucc -> begin ( ) , & RPTracker . getLiveRegs ( ) ) ; RPTracker . advanceBeforeNext ( ) ;

MBBLiveIns [ OnlySucc ] = RPTracker . moveLiveRegs ( ) ;


<s> } } if ( Stage == UnclusteredReschedule ) SavedMutations . swap ( Mutations ) ; for ( auto Region : Regions ) { if ( Stage == UnclusteredReschedule && ! RescheduleRegions [ RegionIdx ] ) { ++ RegionIdx ; continue ; } RegionBegin = Region . first ; RegionEnd = Region . second ; if ( RegionBegin -> getParent ( ) != MBB ) { if ( MBB ) finishBlock ( ) ; MBB = RegionBegin -> getParent ( ) ; startBlock ( MBB ) ; if ( Stage == InitialSchedule ) computeBlockPressure ( MBB ) ; } unsigned NumRegionInstrs = std :: distance ( begin ( ) , end ( ) ) ; enterRegion ( MBB , begin ( ) , end ( ) , NumRegionInstrs ) ; if ( begin ( ) == end ( ) || begin ( ) == std :: prev ( end ( ) ) ) { exitRegion ( ) ; continue ; }


<s> MachineBasicBlock * MBB = nullptr ; if ( Stage > InitialSchedule ) { if ( ! LIS ) break ; if ( Stage == UnclusteredReschedule ) { if ( RescheduleRegions . none ( ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } if ( Stage == ClusteredLowOccupancyReschedule ) { if ( StartingOccupancy <= MinOccupancy ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT> ) ; S . setTargetOccupancy ( MinOccupancy ) ; } } if ( Stage == UnclusteredReschedule ) SavedMutations . swap ( Mutations ) ; for ( auto Region : Regions ) { if ( Stage == UnclusteredReschedule && ! RescheduleRegions [ RegionIdx ] ) { ++ RegionIdx ; continue ; } RegionBegin = Region . first ; RegionEnd = Region . second ; if ( RegionBegin -> getParent ( ) != MBB ) { if ( MBB ) finishBlock ( ) ; MBB = RegionBegin -> getParent ( ) ; startBlock ( MBB ) ; if ( Stage == InitialSchedule ) computeBlockPressure ( MBB ) ; } unsigned NumRegionInstrs = std :: distance ( begin ( ) , end ( ) ) ; enterRegion ( MBB , begin ( ) , end ( ) , NumRegionInstrs ) ; if ( begin ( ) == end ( ) || begin ( ) == std :: prev ( end ( ) ) ) { exitRegion ( ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << MF . getName ( ) << <STR_LIT> << printMBBReference ( * MBB ) << <STR_LIT> << MBB -> getName ( ) << <STR_LIT> << * begin ( ) << <STR_LIT> ; if ( RegionEnd != MBB -> end ( ) ) dbgs ( ) << * RegionEnd ;


<s> GenericScheduler ( C ) , TargetOccupancy ( <NUM_LIT> ) , MF ( nullptr ) {


<s> GCNMaxOccupancySchedStrategy :: GCNMaxOccupancySchedStrategy ( const MachineSchedContext * C ) : GenericScheduler ( C ) , TargetOccupancy ( <NUM_LIT> ) , MF ( nullptr ) {


<s> GCNScheduleDAGMILive :: GCNScheduleDAGMILive ( MachineSchedContext * C , std :: unique_ptr < MachineSchedStrategy > S ) : ScheduleDAGMILive ( C , std :: move ( S ) ) , ST ( MF . getSubtarget < GCNSubtarget > ( ) ) , MFI ( * MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ) , StartingOccupancy ( MFI . getOccupancy ( ) ) , MinOccupancy ( StartingOccupancy ) , Stage ( Collect ) , RegionIdx ( <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << StartingOccupancy << <STR_LIT> ) ;


<s> ScheduleDAGMILive ( C , std :: move ( S ) ) , ST ( MF . getSubtarget < GCNSubtarget > ( ) ) , MFI ( * MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ) , StartingOccupancy ( MFI . getOccupancy ( ) ) , MinOccupancy ( StartingOccupancy ) , Stage ( Collect ) , RegionIdx ( <NUM_LIT> ) {

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << StartingOccupancy << <STR_LIT> ) ;


<s> GCNDownwardRPTracker RPTracker ( * LIS ) ; RPTracker . advance ( begin ( ) , end ( ) , & LiveIns [ RegionIdx ] ) ; return RPTracker . moveMaxPressure ( ) ;


<s> unsigned NewSGPRPressure = Pressure [ <STR_LIT> :: SReg_32 ] ; unsigned NewVGPRPressure = Pressure [ <STR_LIT> :: VGPR_32 ] ; const unsigned MaxVGPRPressureInc = <NUM_LIT> ; bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >= VGPRExcessLimit ; bool ShouldTrackSGPRs = ! ShouldTrackVGPRs && SGPRPressure >= SGPRExcessLimit ; if ( ShouldTrackVGPRs && NewVGPRPressure >= VGPRExcessLimit ) { Cand . RPDelta . Excess = PressureChange ( <STR_LIT> :: VGPR_32 ) ; Cand . RPDelta . Excess . setUnitInc ( NewVGPRPressure - VGPRExcessLimit ) ; } if ( ShouldTrackSGPRs && NewSGPRPressure >= SGPRExcessLimit ) { Cand . RPDelta . Excess = PressureChange ( <STR_LIT> :: SReg_32 ) ; Cand . RPDelta . Excess . setUnitInc ( NewSGPRPressure - SGPRExcessLimit ) ; } int SGPRDelta = NewSGPRPressure - SGPRCriticalLimit ;

int VGPRDelta = NewVGPRPressure - VGPRCriticalLimit ;


<s> SGPRCriticalLimit = ST . getMaxNumSGPRs ( TargetOccupancy , true ) ; VGPRCriticalLimit = ST . getMaxNumVGPRs ( TargetOccupancy ) ; } else { SGPRCriticalLimit = SRI -> getRegPressureSetLimit ( DAG -> MF , <STR_LIT> :: SReg_32 ) ; VGPRCriticalLimit = SRI -> getRegPressureSetLimit ( DAG -> MF , <STR_LIT> :: VGPR_32 ) ; }


<s> SUnit * SU ; do { if ( RegionPolicy . OnlyTopDown ) { SU = Top . pickOnlyChoice ( ) ; if ( ! SU ) { CandPolicy NoPolicy ; TopCand . reset ( NoPolicy ) ; pickNodeFromQueue ( Top , NoPolicy , DAG -> getTopRPTracker ( ) , TopCand ) ; assert ( TopCand . Reason != NoCand && <STR_LIT> ) ; SU = TopCand . SU ; } IsTopNode = true ; } else if ( RegionPolicy . OnlyBottomUp ) { SU = Bot . pickOnlyChoice ( ) ; if ( ! SU ) { CandPolicy NoPolicy ; BotCand . reset ( NoPolicy ) ; pickNodeFromQueue ( Bot , NoPolicy , DAG -> getBotRPTracker ( ) , BotCand ) ; assert ( BotCand . Reason != NoCand && <STR_LIT> ) ; SU = BotCand . SU ; } IsTopNode = false ; } else { SU = pickNodeBidirectional ( IsTopNode ) ;


<s> SU = BotCand . SU ; } IsTopNode = false ; } else { SU = pickNodeBidirectional ( IsTopNode ) ; } } while ( SU -> isScheduled ) ; if ( SU -> isTopReady ( ) ) Top . removeReady ( SU ) ; if ( SU -> isBottomReady ( ) ) Bot . removeReady ( SU ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << SU -> NodeNum << <STR_LIT> << * SU -> getInstr ( ) ) ; return SU ;


<s> ReadyQueue & Q = Zone . Available ; for ( SUnit * SU : Q ) { SchedCandidate TryCand ( ZonePolicy ) ; initCandidate ( TryCand , SU , Zone . isTop ( ) , RPTracker , SRI , SGPRPressure , VGPRPressure ) ; SchedBoundary * ZoneArg = Cand . AtTop == TryCand . AtTop ? & Zone : nullptr ;

GenericScheduler :: tryCandidate ( Cand , TryCand , ZoneArg ) ;


<s> for ( SUnit * SU : Q ) { SchedCandidate TryCand ( ZonePolicy ) ; initCandidate ( TryCand , SU , Zone . isTop ( ) , RPTracker , SRI , SGPRPressure , VGPRPressure ) ; SchedBoundary * ZoneArg = Cand . AtTop == TryCand . AtTop ? & Zone : nullptr ; GenericScheduler :: tryCandidate ( Cand , TryCand , ZoneArg ) ; if ( TryCand . Reason != NoCand ) { if ( TryCand . ResDelta == SchedResourceDelta ( ) ) TryCand . initResourceDelta ( Zone . DAG , SchedModel ) ; Cand . setBest ( TryCand ) ; LLVM_DEBUG ( traceCandidate ( Cand ) ) ; } }


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; PressureAfter . print ( dbgs ( ) ) ) ; if ( PressureAfter . getSGPRNum ( ) <= S . SGPRCriticalLimit && PressureAfter . getVGPRNum ( ) <= S . VGPRCriticalLimit ) { Pressure [ RegionIdx ] = PressureAfter ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return ; } unsigned Occ = MFI . getOccupancy ( ) ; unsigned WavesAfter = std :: min ( Occ , PressureAfter . getOccupancy ( ST ) ) ; unsigned WavesBefore = std :: min ( Occ , PressureBefore . getOccupancy ( ST ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << WavesBefore << <STR_LIT> << WavesAfter << <STR_LIT> ) ; unsigned NewOccupancy = std :: max ( WavesAfter , WavesBefore ) ; if ( WavesAfter < WavesBefore && WavesAfter < MinOccupancy && WavesAfter >= MFI . getMinAllowedOccupancy ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MFI . getMinAllowedOccupancy ( ) << <STR_LIT> ) ; NewOccupancy = WavesAfter ; } if ( NewOccupancy < MinOccupancy ) { MinOccupancy = NewOccupancy ; MFI . limitOccupancy ( MinOccupancy ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT> ) ; } unsigned MaxVGPRs = ST . getMaxNumVGPRs ( MF ) ; unsigned MaxSGPRs = ST . getMaxNumSGPRs ( MF ) ; if ( PressureAfter . getVGPRNum ( ) > MaxVGPRs || PressureAfter . getSGPRNum ( ) > MaxSGPRs ) RescheduleRegions [ RegionIdx ] = true ; if ( WavesAfter >= MinOccupancy ) { if ( Stage == UnclusteredReschedule && ! PressureAfter . less ( ST , PressureBefore ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } else if ( WavesAfter > MFI . getMinWavesPerEU ( ) || PressureAfter . less ( ST , PressureBefore ) || ! RescheduleRegions [ RegionIdx ] ) { Pressure [ RegionIdx ] = PressureAfter ; return ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ;


<s> bool hasFmaakFmamkF32Insts ( ) const { return getGeneration ( ) >= GFX10 ;


<s> bool hasDstSelForwardingHazard ( ) const { return GFX940Insts ;


<s> bool hasVDecCoExecHazard ( ) const { return GFX940Insts ;


return HasAtomicFaddRtnInsts || HasAtomicFaddNoRtnInsts ;


<s> bool hasAtomicFaddNoRtnInsts ( ) const { return HasAtomicFaddNoRtnInsts ;


<s> bool hasAtomicFaddRtnInsts ( ) const { return HasAtomicFaddRtnInsts ;


<s> bool hasAtomicPkFaddNoRtnInsts ( ) const { return HasAtomicPkFaddNoRtnInsts ;


<s> bool hasCompressedExport ( ) const {


<s> bool hasFlatScratchSVSMode ( ) const {


<s> bool hasLegacyGeometry ( ) const { return getGeneration ( ) < GFX11 ;


<s> bool hasNullExportTarget ( ) const {


<s> bool hasPermLane64 ( ) const { return getGeneration ( ) >= GFX11 ;


<s> return getGeneration ( ) == <ISA_LIT>Subtarget :: GFX9 ;


<s> bool hasSPackHL ( ) const { return GFX11Insts ;


<s> bool hasVALUPartialForwardingHazard ( ) const {


<s> bool hasVALUTransUseHazard ( ) const {


<s> bool d16PreservesUnusedBits ( ) const {

return hasD16LoadStore ( ) && ! TargetID . isSramEccOnOrAny ( ) ;


<s> return flatScratchIsArchitected ( ) || ( EnableFlatScratch && hasFlatScratchInsts ( ) ) ;


<s> bool flatScratchIsArchitected ( ) const {


<s> TrapHandlerAbi getTrapHandlerAbi ( ) const {

return isAmdHsaOS ( ) ? TrapHandlerAbi :: AMDHSA : TrapHandlerAbi :: NONE ;


<s> bool hasDOTOpSelHazard ( ) const { return GFX940Insts ;


<s> bool hasExtendedImageInsts ( ) const {


<s> bool hasFlatScratchSTMode ( ) const {

return hasFlatScratchInsts ( ) && ( hasGFX10_3Insts ( ) || hasGFX940Insts ( ) ) ;


<s> return hasFlatScratchInsts ( ) && ( hasGFX10_3Insts ( ) || hasGFX940Insts ( ) ) ;


<s> return getGeneration ( ) >= GFX10 || hasGFX940Insts ( ) ;


<s> bool hasNegativeScratchOffsetBug ( ) const {


<s> bool hasNegativeUnalignedScratchOffsetBug ( ) const {


<s> bool hasNSAClauseBug ( ) const { return HasNSAClauseBug ;


<s> bool supportsBackOffBarrier ( ) const { return BackOffBarrier ;


<s> case CallingConv :: <ISA_LIT>_CS : LLVM_FALLTHROUGH ; case CallingConv :: <ISA_LIT>_VS : RsrcReg = R_028868_SQ_PGM_RESOURCES_VS ; break ; case CallingConv :: <ISA_LIT>_PS : RsrcReg = R_028850_SQ_PGM_RESOURCES_PS ; break ; } } OutStreamer -> emitIntValue ( RsrcReg , <NUM_LIT> ) ; OutStreamer -> emitIntValue ( S_NUM_GPRS ( MaxGPR + <NUM_LIT> ) | S_STACK_<ISA_LIT>ZE ( MFI -> CFStackSize ) , <NUM_LIT> ) ; OutStreamer -> emitIntValue ( R_02880C_DB_SHADER_CONTROL , <NUM_LIT> ) ; OutStreamer -> emitIntValue ( S_02880C_KILL_ENABLE ( killPixel ) , <NUM_LIT> ) ; if ( <STR_LIT> ( MF . getFunction ( ) . getCallingConv ( ) ) ) { OutStreamer -> emitIntValue ( R_0288E8_SQ_LDS_ALLOC , <NUM_LIT> ) ; OutStreamer -> emitIntValue ( alignTo ( MFI -> getLDSSize ( ) , <NUM_LIT> ) >> <NUM_LIT> , <NUM_LIT> ) ;


<s> default : LLVM_FALLTHROUGH ; case CallingConv :: <ISA_LIT>_GS : LLVM_FALLTHROUGH ; case CallingConv :: <ISA_LIT>_CS : LLVM_FALLTHROUGH ; case CallingConv :: <ISA_LIT>_VS : RsrcReg = R_028868_SQ_PGM_RESOURCES_VS ; break ; case CallingConv :: <ISA_LIT>_PS : RsrcReg = R_028850_SQ_PGM_RESOURCES_PS ; break ; } } OutStreamer -> emitIntValue ( RsrcReg , <NUM_LIT> ) ; OutStreamer -> emitIntValue ( S_NUM_GPRS ( MaxGPR + <NUM_LIT> ) | S_STACK_<ISA_LIT>ZE ( MFI -> CFStackSize ) , <NUM_LIT> ) ; OutStreamer -> emitIntValue ( R_02880C_DB_SHADER_CONTROL , <NUM_LIT> ) ; OutStreamer -> emitIntValue ( S_02880C_KILL_ENABLE ( killPixel ) , <NUM_LIT> ) ; if ( <STR_LIT> ( MF . getFunction ( ) . getCallingConv ( ) ) ) { OutStreamer -> emitIntValue ( R_0288E8_SQ_LDS_ALLOC , <NUM_LIT> ) ;

OutStreamer -> emitIntValue ( alignTo ( MFI -> getLDSSize ( ) , <NUM_LIT> ) >> <NUM_LIT> , <NUM_LIT> ) ;


<s> EmitProgramInfo<ISA_LIT> ( MF ) ; emitFunctionBody ( ) ; if ( isVerbose ( ) ) { MCSectionELF * CommentSection = Context . getELFSection ( <STR_LIT> , ELF :: SHT_PROGBITS , <NUM_LIT> ) ; OutStreamer -> SwitchSection ( CommentSection ) ;

<ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;


<s> SetupMachineFunction ( MF ) ; MCContext & Context = getObjFileLowering ( ) . getContext ( ) ; MCSectionELF * ConfigSection = Context . getELFSection ( <STR_LIT> , ELF :: SHT_PROGBITS , <NUM_LIT> ) ; OutStreamer -> SwitchSection ( ConfigSection ) ; EmitProgramInfo<ISA_LIT> ( MF ) ; EmitFunctionBody ( ) ; if ( isVerbose ( ) ) { MCSectionELF * CommentSection = Context . getELFSection ( <STR_LIT> , ELF :: SHT_PROGBITS , <NUM_LIT> ) ; OutStreamer -> SwitchSection ( CommentSection ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; OutStreamer -> emitRawComment ( Twine ( <STR_LIT> + Twine ( MFI -> CFStackSize ) ) ) ;


<s> MCSectionELF * ConfigSection = Context . getELFSection ( <STR_LIT> , ELF :: SHT_PROGBITS , <NUM_LIT> ) ; OutStreamer -> SwitchSection ( ConfigSection ) ; EmitProgramInfo<ISA_LIT> ( MF ) ; EmitFunctionBody ( ) ;


<s> } } } unsigned RsrcReg ; if ( STM . getGeneration ( ) >= <ISA_LIT>Subtarget :: EVERGREEN ) { switch ( MF . getFunction ( ) . getCallingConv ( ) ) { default : LLVM_FALLTHROUGH ; case CallingConv :: <ISA_LIT>_CS : RsrcReg = R_0288D4_SQ_PGM_RESOURCES_LS ; break ; case CallingConv :: <ISA_LIT>_GS : RsrcReg = R_028878_SQ_PGM_RESOURCES_GS ; break ; case CallingConv :: <ISA_LIT>_PS : RsrcReg = R_028844_SQ_PGM_RESOURCES_PS ; break ; case CallingConv :: <ISA_LIT>_VS : RsrcReg = R_028860_SQ_PGM_RESOURCES_VS ; break ; } } else { switch ( MF . getFunction ( ) . getCallingConv ( ) ) { default : LLVM_FALLTHROUGH ; case CallingConv :: <ISA_LIT>_GS : LLVM_FALLTHROUGH ; case CallingConv :: <ISA_LIT>_CS : LLVM_FALLTHROUGH ; case CallingConv :: <ISA_LIT>_VS : RsrcReg = R_028868_SQ_PGM_RESOURCES_VS ; break ; case CallingConv :: <ISA_LIT>_PS : RsrcReg = R_028850_SQ_PGM_RESOURCES_PS ; break ;


<s> AsmPrinter * llvm :: create<ISA_LIT>AsmPrinterPass ( TargetMachine & TM , std :: unique_ptr < MCStreamer > && Streamer ) { return new <ISA_LIT>AsmPrinter ( TM , std :: move ( Streamer ) ) ;


<s> const <ISA_LIT>Subtarget & STM = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const <ISA_LIT>RegisterInfo * RI = STM . getRegisterInfo ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; for ( const MachineBasicBlock & MBB : MF ) { for ( const MachineInstr & MI : MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) killPixel = true ; unsigned numOperands = MI . getNumOperands ( ) ; for ( unsigned op_idx = <NUM_LIT> ; op_idx < numOperands ; op_idx ++ ) { const MachineOperand & MO = MI . getOperand ( op_idx ) ; if ( ! MO . isReg ( ) ) continue ; unsigned HWReg = RI -> getHWRegIndex ( MO . getReg ( ) ) ; if ( HWReg > <NUM_LIT> ) continue ; MaxGPR = std :: max ( MaxGPR , HWReg ) ; } } } unsigned RsrcReg ; if ( STM . getGeneration ( ) >= <ISA_LIT>Subtarget :: EVERGREEN ) { switch ( MF . getFunction ( ) . getCallingConv ( ) ) { default : LLVM_FALLTHROUGH ; case CallingConv :: <ISA_LIT>_CS : RsrcReg = R_0288D4_SQ_PGM_RESOURCES_LS ; break ; case CallingConv :: <ISA_LIT>_GS : RsrcReg = R_028878_SQ_PGM_RESOURCES_GS ; break ;

case CallingConv :: <ISA_LIT>_PS : RsrcReg = R_028844_SQ_PGM_RESOURCES_PS ; break ;


<s> StringRef <ISA_LIT>AsmPrinter :: getPassName ( ) const {


<s> <ISA_LIT>AsmPrinter :: <ISA_LIT>AsmPrinter ( TargetMachine & TM , std :: unique_ptr < MCStreamer > Streamer ) : AsmPrinter ( TM , std :: move ( Streamer ) ) {


<s> MF . ensureAlignment ( <NUM_LIT> ) ; SetupMachineFunction ( MF ) ; MCContext & Context = getObjFileLowering ( ) . getContext ( ) ; MCSectionELF * ConfigSection = Context . getELFSection ( <STR_LIT> , ELF :: SHT_PROGBITS , <NUM_LIT> ) ; OutStreamer -> SwitchSection ( ConfigSection ) ; EmitProgramInfo<ISA_LIT> ( MF ) ; EmitFunctionBody ( ) ; if ( isVerbose ( ) ) { MCSectionELF * CommentSection = Context . getELFSection ( <STR_LIT> , ELF :: SHT_PROGBITS , <NUM_LIT> ) ; OutStreamer -> SwitchSection ( CommentSection ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; OutStreamer -> emitRawComment ( Twine ( <STR_LIT> + Twine ( MFI -> CFStackSize ) ) ) ; }


<s> OutStreamer -> SwitchSection ( ConfigSection ) ; EmitProgramInfo<ISA_LIT> ( MF ) ; EmitFunctionBody ( ) ; if ( isVerbose ( ) ) { MCSectionELF * CommentSection = Context . getELFSection ( <STR_LIT> , ELF :: SHT_PROGBITS , <NUM_LIT> ) ; OutStreamer -> SwitchSection ( CommentSection ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; OutStreamer -> emitRawComment ( Twine ( <STR_LIT> + Twine ( MFI -> CFStackSize ) ) ) ; }


<s> llvm :: FunctionPass * llvm :: create<ISA_LIT>ClauseMergePass ( TargetMachine & TM ) { return new <ISA_LIT>ClauseMergePass ( TM ) ;


<s> <ISA_LIT>ClauseMergePass ( TargetMachine & tm ) : MachineFunctionPass ( ID ) {


<s> bool <ISA_LIT>ClauseMergePass :: runOnMachineFunction ( MachineFunction & MF ) { if ( skipFunction ( * MF . getFunction ( ) ) ) return false ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = ST . getInstrInfo ( ) ; for ( MachineFunction :: iterator BB = MF . begin ( ) , BB_E = MF . end ( ) ; BB != BB_E ; ++ BB ) { MachineBasicBlock & MBB = * BB ; MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; MachineBasicBlock :: iterator LatestCFAlu = E ; while ( I != E ) { MachineInstr & MI = * I ++ ; if ( ( ! TII -> canBeConsideredALU ( MI ) && ! isCFAlu ( MI ) ) || TII -> mustBeLastInClause ( MI . getOpcode ( ) ) ) LatestCFAlu = E ; if ( ! isCFAlu ( MI ) ) continue ; cleanPotentialDisabledCFAlu ( MI ) ; if ( LatestCFAlu != E && mergeIfPossible ( * LatestCFAlu , MI ) ) { MI . eraseFromParent ( ) ; } else {

assert ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) && <STR_LIT> ) ;


<s> while ( I != E && ! isCFAlu ( I ) ) I ++ ; if ( I == E ) return ; MachineInstr * MI = I ++ ; if ( isCFAluEnabled ( MI ) ) break ; CFAlu -> getOperand ( CntIdx ) . setImm ( getCFAluSize ( CFAlu ) + getCFAluSize ( MI ) ) ; MI -> eraseFromParent ( ) ; } while ( I != E ) ;


<s> int CntIdx = TII -> getOperandIdx ( <STR_LIT> , <STR_LIT> :: COUNT ) ; MachineBasicBlock :: iterator I = CFAlu , E = CFAlu -> getParent ( ) -> end ( ) ; I ++ ; do { while ( I != E && ! isCFAlu ( I ) ) I ++ ;


<s> return MI -> getOperand ( TII -> getOperandIdx ( MI -> getOpcode ( ) , <STR_LIT> :: COUNT ) ) . getImm ( ) ;


<s> case <STR_LIT> : case <STR_LIT> : return true ; default :


<s> return MI -> getOperand ( TII -> getOperandIdx ( MI -> getOpcode ( ) , <STR_LIT> :: Enabled ) ) . getImm ( ) ;


<s> assert ( isCFAlu ( MI ) ) ; return MI -> getOperand ( TII -> getOperandIdx ( MI -> getOpcode ( ) , <STR_LIT> :: Enabled ) ) . getImm ( ) ;


<s> DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( RootCFAlu -> getOpcode ( ) == <STR_LIT> ) return false ; int Mode0Idx = TII -> getOperandIdx ( <STR_LIT> , <STR_LIT> :: KCACHE_MODE0 ) ; int KBank0Idx = TII -> getOperandIdx ( <STR_LIT> , <STR_LIT> :: KCACHE_BANK0 ) ; int KBank0LineIdx = TII -> getOperandIdx ( <STR_LIT> , <STR_LIT> :: KCACHE_ADDR0 ) ; if ( LatrCFAlu -> getOperand ( Mode0Idx ) . getImm ( ) && RootCFAlu -> getOperand ( Mode0Idx ) . getImm ( ) && ( LatrCFAlu -> getOperand ( KBank0Idx ) . getImm ( ) != RootCFAlu -> getOperand ( KBank0Idx ) . getImm ( ) || LatrCFAlu -> getOperand ( KBank0LineIdx ) . getImm ( ) != RootCFAlu -> getOperand ( KBank0LineIdx ) . getImm ( ) ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } int Mode1Idx = TII -> getOperandIdx ( <STR_LIT> , <STR_LIT> :: KCACHE_MODE1 ) ;

int KBank1Idx = TII -> getOperandIdx ( <STR_LIT> , <STR_LIT> :: KCACHE_BANK1 ) ;


<s> bool <ISA_LIT>ClauseMergePass :: runOnMachineFunction ( MachineFunction & MF ) { TII = static_cast < const <ISA_LIT>InstrInfo * > ( MF . getSubtarget ( ) . getInstrInfo ( ) ) ; for ( MachineFunction :: iterator BB = MF . begin ( ) , BB_E = MF . end ( ) ; BB != BB_E ; ++ BB ) { MachineBasicBlock & MBB = * BB ; MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; MachineBasicBlock :: iterator LatestCFAlu = E ; while ( I != E ) { MachineInstr * MI = I ++ ; if ( ( ! TII -> canBeConsideredALU ( MI ) && ! isCFAlu ( MI ) ) || TII -> mustBeLastInClause ( MI -> getOpcode ( ) ) ) LatestCFAlu = E ; if ( ! isCFAlu ( MI ) ) continue ; cleanPotentialDisabledCFAlu ( MI ) ; if ( LatestCFAlu != E && mergeIfPossible ( LatestCFAlu , MI ) ) { MI -> eraseFromParent ( ) ; } else { assert ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) && <STR_LIT> ) ; LatestCFAlu = MI ; } } } return false ;


<s> if ( isCFAluEnabled ( MI ) ) break ; CFAlu . getOperand ( CntIdx ) . setImm ( getCFAluSize ( CFAlu ) + getCFAluSize ( MI ) ) ; MI . eraseFromParent ( ) ; } while ( I != E ) ;


<s> unsigned <ISA_LIT>ClauseMergePass :: getCFAluSize ( const MachineInstr & MI ) const { assert ( isCFAlu ( MI ) ) ;

return MI . getOperand ( TII -> getOperandIdx ( MI . getOpcode ( ) , <STR_LIT> :: COUNT ) ) . getImm ( ) ;


<s> case <STR_LIT> : case <STR_LIT> : return true ; default : return false ;


<s> static bool isCFAlu ( const MachineInstr & MI ) { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : return true ; default : return false ;


<s> bool <ISA_LIT>ClauseMergePass :: isCFAluEnabled ( const MachineInstr & MI ) const {


<s> if ( LatrCFAlu . getOperand ( Mode0Idx ) . getImm ( ) && RootCFAlu . getOperand ( Mode0Idx ) . getImm ( ) && ( LatrCFAlu . getOperand ( KBank0Idx ) . getImm ( ) != RootCFAlu . getOperand ( KBank0Idx ) . getImm ( ) || LatrCFAlu . getOperand ( KBank0LineIdx ) . getImm ( ) != RootCFAlu . getOperand ( KBank0LineIdx ) . getImm ( ) ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } int Mode1Idx = TII -> getOperandIdx ( <STR_LIT> , <STR_LIT> :: KCACHE_MODE1 ) ; int KBank1Idx = TII -> getOperandIdx ( <STR_LIT> , <STR_LIT> :: KCACHE_BANK1 ) ; int KBank1LineIdx = TII -> getOperandIdx ( <STR_LIT> , <STR_LIT> :: KCACHE_ADDR1 ) ; if ( LatrCFAlu . getOperand ( Mode1Idx ) . getImm ( ) && RootCFAlu . getOperand ( Mode1Idx ) . getImm ( ) && ( LatrCFAlu . getOperand ( KBank1Idx ) . getImm ( ) != RootCFAlu . getOperand ( KBank1Idx ) . getImm ( ) || LatrCFAlu . getOperand ( KBank1LineIdx ) . getImm ( ) != RootCFAlu . getOperand ( KBank1LineIdx ) . getImm ( ) ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( LatrCFAlu . getOperand ( Mode0Idx ) . getImm ( ) ) { RootCFAlu . getOperand ( Mode0Idx ) . setImm ( LatrCFAlu . getOperand ( Mode0Idx ) . getImm ( ) ) ; RootCFAlu . getOperand ( KBank0Idx ) . setImm ( LatrCFAlu . getOperand ( KBank0Idx ) . getImm ( ) ) ; RootCFAlu . getOperand ( KBank0LineIdx ) . setImm ( LatrCFAlu . getOperand ( KBank0LineIdx ) . getImm ( ) ) ; } if ( LatrCFAlu . getOperand ( Mode1Idx ) . getImm ( ) ) { RootCFAlu . getOperand ( Mode1Idx ) . setImm ( LatrCFAlu . getOperand ( Mode1Idx ) . getImm ( ) ) ;

RootCFAlu . getOperand ( KBank1Idx ) . setImm ( LatrCFAlu . getOperand ( KBank1Idx ) . getImm ( ) ) ;


<s> if ( skipFunction ( * MF . getFunction ( ) ) ) return false ; TII = static_cast < const <ISA_LIT>InstrInfo * > ( MF . getSubtarget ( ) . getInstrInfo ( ) ) ; for ( MachineFunction :: iterator BB = MF . begin ( ) , BB_E = MF . end ( ) ; BB != BB_E ; ++ BB ) { MachineBasicBlock & MBB = * BB ; MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; MachineBasicBlock :: iterator LatestCFAlu = E ; while ( I != E ) { MachineInstr * MI = I ++ ;

if ( ( ! TII -> canBeConsideredALU ( MI ) && ! isCFAlu ( MI ) ) || TII -> mustBeLastInClause ( MI -> getOpcode ( ) ) ) LatestCFAlu = E ;


<s> MachineBasicBlock :: iterator LatestCFAlu = E ; while ( I != E ) { MachineInstr * MI = I ++ ; if ( ( ! TII -> canBeConsideredALU ( MI ) && ! isCFAlu ( MI ) ) || TII -> mustBeLastInClause ( MI -> getOpcode ( ) ) ) LatestCFAlu = E ; if ( ! isCFAlu ( MI ) ) continue ; cleanPotentialDisabledCFAlu ( MI ) ; if ( LatestCFAlu != E && mergeIfPossible ( LatestCFAlu , MI ) ) { MI -> eraseFromParent ( ) ;


<s> for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; MachineBasicBlock :: iterator LatestCFAlu = E ; while ( I != E ) { MachineInstr & MI = * I ++ ;

if ( ( ! TII -> canBeConsideredALU ( MI ) && ! isCFAlu ( MI ) ) || TII -> mustBeLastInClause ( MI . getOpcode ( ) ) ) LatestCFAlu = E ;


<s> MachineBasicBlock :: iterator I = CFAlu , E = CFAlu . getParent ( ) -> end ( ) ; I ++ ; do { while ( I != E && ! isCFAlu ( * I ) ) I ++ ; if ( I == E ) return ;


<s> llvm :: FunctionPass * llvm :: create<ISA_LIT>ClauseMergePass ( ) {

return new <ISA_LIT>ClauseMergePass ( ) ;


<s> StringRef <ISA_LIT>ClauseMergePass :: getPassName ( ) const { return <STR_LIT> ;


<s> static bool isCFAlu ( const MachineInstr & MI ) { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : return true ;


<s> static bool isCFAlu ( const MachineInstr & MI ) { switch ( MI . getOpcode ( ) ) {


<s> bool <ISA_LIT>ClauseMergePass :: isCFAluEnabled ( const MachineInstr & MI ) const { assert ( isCFAlu ( MI ) ) ;

return MI . getOperand ( TII -> getOperandIdx ( MI . getOpcode ( ) , <STR_LIT> :: Enabled ) ) . getImm ( ) ;


<s> int KBank0LineIdx = TII -> getOperandIdx ( <STR_LIT> , <STR_LIT> :: KCACHE_ADDR0 ) ; if ( LatrCFAlu . getOperand ( Mode0Idx ) . getImm ( ) && RootCFAlu . getOperand ( Mode0Idx ) . getImm ( ) && ( LatrCFAlu . getOperand ( KBank0Idx ) . getImm ( ) != RootCFAlu . getOperand ( KBank0Idx ) . getImm ( ) || LatrCFAlu . getOperand ( KBank0LineIdx ) . getImm ( ) != RootCFAlu . getOperand ( KBank0LineIdx ) . getImm ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } int Mode1Idx = TII -> getOperandIdx ( <STR_LIT> , <STR_LIT> :: KCACHE_MODE1 ) ; int KBank1Idx = TII -> getOperandIdx ( <STR_LIT> , <STR_LIT> :: KCACHE_BANK1 ) ; int KBank1LineIdx = TII -> getOperandIdx ( <STR_LIT> , <STR_LIT> :: KCACHE_ADDR1 ) ; if ( LatrCFAlu . getOperand ( Mode1Idx ) . getImm ( ) && RootCFAlu . getOperand ( Mode1Idx ) . getImm ( ) && ( LatrCFAlu . getOperand ( KBank1Idx ) . getImm ( ) != RootCFAlu . getOperand ( KBank1Idx ) . getImm ( ) || LatrCFAlu . getOperand ( KBank1LineIdx ) . getImm ( ) != RootCFAlu . getOperand ( KBank1LineIdx ) . getImm ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( LatrCFAlu . getOperand ( Mode0Idx ) . getImm ( ) ) { RootCFAlu . getOperand ( Mode0Idx ) . setImm ( LatrCFAlu . getOperand ( Mode0Idx ) . getImm ( ) ) ; RootCFAlu . getOperand ( KBank0Idx ) . setImm ( LatrCFAlu . getOperand ( KBank0Idx ) . getImm ( ) ) ; RootCFAlu . getOperand ( KBank0LineIdx ) . setImm ( LatrCFAlu . getOperand ( KBank0LineIdx ) . getImm ( ) ) ; } if ( LatrCFAlu . getOperand ( Mode1Idx ) . getImm ( ) ) { RootCFAlu . getOperand ( Mode1Idx ) . setImm ( LatrCFAlu . getOperand ( Mode1Idx ) . getImm ( ) ) ; RootCFAlu . getOperand ( KBank1Idx ) . setImm ( LatrCFAlu . getOperand ( KBank1Idx ) . getImm ( ) ) ; RootCFAlu . getOperand ( KBank1LineIdx ) . setImm ( LatrCFAlu . getOperand ( KBank1LineIdx ) . getImm ( ) ) ;


<s> <ISA_LIT>ClauseMergePass ( ) : MachineFunctionPass ( ID ) {


<s> bool <ISA_LIT>ClauseMergePass :: runOnMachineFunction ( MachineFunction & MF ) { if ( skipFunction ( MF . getFunction ( ) ) ) return false ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = ST . getInstrInfo ( ) ; for ( MachineFunction :: iterator BB = MF . begin ( ) , BB_E = MF . end ( ) ; BB != BB_E ; ++ BB ) { MachineBasicBlock & MBB = * BB ; MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; MachineBasicBlock :: iterator LatestCFAlu = E ; while ( I != E ) { MachineInstr & MI = * I ++ ; if ( ( ! TII -> canBeConsideredALU ( MI ) && ! isCFAlu ( MI ) ) || TII -> mustBeLastInClause ( MI . getOpcode ( ) ) ) LatestCFAlu = E ; if ( ! isCFAlu ( MI ) ) continue ;


<s> CFStack ( const <ISA_LIT>Subtarget * st , CallingConv :: ID cc ) : ST ( st ) , MaxStackSize ( cc == CallingConv :: <ISA_LIT>_VS ? <NUM_LIT> : <NUM_LIT> ) {


<s> void CounterPropagateAddr ( const std :: set < MachineInstr * > & MIs , unsigned Addr ) const {


<s> FunctionPass * llvm :: create<ISA_LIT>ControlFlowFinalizer ( ) {

return new <ISA_LIT>ControlFlowFinalizer ( ) ;


<s> BuildMI ( BB , DL , TII -> get ( <STR_LIT> ) ) . addImm ( CfCount ) ; for ( unsigned i = <NUM_LIT> , e = Clause . second . size ( ) ; i < e ; ++ i ) { BB -> splice ( InsertPos , BB , Clause . second [ i ] ) ;


<s> for ( unsigned i = <NUM_LIT> , e = Clause . second . size ( ) ; i < e ; ++ i ) { BB -> splice ( InsertPos , BB , Clause . second [ i ] ) ; } CfCount += Clause . second . size ( ) ;


<s> CounterPropagateAddr ( * Clause . first , CfCount ) ; MachineBasicBlock * BB = Clause . first -> getParent ( ) ;

BuildMI ( BB , DL , TII -> get ( <STR_LIT> ) ) . addImm ( CfCount ) ;


<s> static const unsigned LiteralRegs [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; const SmallVector < std :: pair < MachineOperand * , int64_t > , <NUM_LIT> > Srcs = TII -> getSrcs ( MI ) ; for ( const auto & Src : Srcs ) { if ( Src . first -> getReg ( ) != <STR_LIT> ) continue ; int64_t Imm = Src . second ; std :: vector < MachineOperand * > :: iterator It = llvm :: find_if ( Lits , [ & ] ( MachineOperand * val ) {

return val -> isImm ( ) && ( val -> getImm ( ) == Imm ) ;


<s> static const unsigned LiteralRegs [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; const SmallVector < std :: pair < MachineOperand * , int64_t > , <NUM_LIT> > Srcs = TII -> getSrcs ( MI ) ; for ( const auto & Src : Srcs ) { if ( Src . first -> getReg ( ) != <STR_LIT> ) continue ; int64_t Imm = Src . second ; std :: vector < MachineOperand * > :: iterator It = llvm :: find_if ( Lits , [ & ] ( MachineOperand * val ) { return val -> isImm ( ) && ( val -> getImm ( ) == Imm ) ; } ) ; MachineOperand & Operand = MI . getOperand ( TII -> getOperandIdx ( MI . getOpcode ( ) , <STR_LIT> :: literal ) ) ; if ( It != Lits . end ( ) ) {

unsigned Index = It - Lits . begin ( ) ;


<s> unsigned LiteralPair0 = Literals [ i ] ; unsigned LiteralPair1 = ( i + <NUM_LIT> < e ) ? Literals [ i + <NUM_LIT> ] : <NUM_LIT> ; InsertPos = BuildMI ( MBB , InsertPos -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( LiteralPair0 ) . addImm ( LiteralPair1 ) ; }


<s> if ( MO . isDef ( ) ) { unsigned Reg = MO . getReg ( ) ; if ( <STR_LIT> . contains ( Reg ) ) DstMI = Reg ; else DstMI = TRI -> getMatchingSuperReg ( Reg , <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ; } if ( MO . isUse ( ) ) { unsigned Reg = MO . getReg ( ) ; if ( <STR_LIT> . contains ( Reg ) ) SrcMI = Reg ; else SrcMI = TRI -> getMatchingSuperReg ( Reg , <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ; } } if ( ( DstRegs . find ( SrcMI ) == DstRegs . end ( ) ) ) { DstRegs . insert ( DstMI ) ; return true ; } else return false ;


<s> bool IsTrivialInst ( MachineInstr & MI ) const { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : return true ; default :


<s> BI -> unbundleFromPred ( ) ; for ( MachineOperand & MO : BI -> operands ( ) ) { if ( MO . isReg ( ) && MO . isInternalRead ( ) ) MO . setIsInternalRead ( false ) ; } getLiteral ( * BI , Literals ) ; ClauseContent . push_back ( & * BI ) ; } I = BI ; DeleteMI . eraseFromParent ( ) ; } else { getLiteral ( * I , Literals ) ; ClauseContent . push_back ( & * I ) ; I ++ ; } for ( unsigned i = <NUM_LIT> , e = Literals . size ( ) ; i < e ; i += <NUM_LIT> ) { MachineInstrBuilder MILit = BuildMI ( MBB , I , I -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; if ( Literals [ i ] -> isImm ( ) ) { MILit . addImm ( Literals [ i ] -> getImm ( ) ) ; } else { MILit . addGlobalAddress ( Literals [ i ] -> getGlobal ( ) , Literals [ i ] -> getOffset ( ) ) ; } if ( i + <NUM_LIT> < e ) { if ( Literals [ i + <NUM_LIT> ] -> isImm ( ) ) { MILit . addImm ( Literals [ i + <NUM_LIT> ] -> getImm ( ) ) ; } else { MILit . addGlobalAddress ( Literals [ i + <NUM_LIT> ] -> getGlobal ( ) , Literals [ i + <NUM_LIT> ] -> getOffset ( ) ) ; } } else MILit . addImm ( <NUM_LIT> ) ; ClauseContent . push_back ( MILit ) ; } }

assert ( ClauseContent . size ( ) < <NUM_LIT> && <STR_LIT> ) ;


<s> if ( IsTrivialInst ( * I ) ) continue ; if ( AluInstCount >= MaxFetchInst ) break ; if ( ( IsTex && ! TII -> usesTextureCache ( * I ) ) || ( ! IsTex && ! TII -> usesVertexCache ( * I ) ) ) break ; if ( ! isCompatibleWithClause ( * I , DstRegs ) ) break ; AluInstCount ++ ; ClauseContent . push_back ( & * I ) ; } MachineInstr * MIb = BuildMI ( MBB , ClauseHead , MBB . findDebugLoc ( ClauseHead ) , getHWInstrDesc ( IsTex ? CF_TC : CF_VC ) ) . addImm ( <NUM_LIT> ) . addImm ( AluInstCount - <NUM_LIT> ) ; return ClauseFile ( MIb , std :: move ( ClauseContent ) ) ;


<s> if ( ! isWQM ) { if ( ! ST -> hasCaymanISA ( ) && ! branchStackContains ( CFStack :: FIRST_NON_WQM_PUSH ) ) Item = CFStack :: FIRST_NON_WQM_PUSH ; else if ( CurrentEntries > <NUM_LIT> && ST -> getGeneration ( ) > <ISA_LIT>Subtarget :: EVERGREEN && ! ST -> hasCaymanISA ( ) && ! branchStackContains ( CFStack :: FIRST_NON_WQM_PUSH_W_FULL_ENTRY ) ) Item = CFStack :: FIRST_NON_WQM_PUSH_W_FULL_ENTRY ; else Item = CFStack :: SUB_ENTRY ; } else Item = CFStack :: ENTRY ; break ; }


<s> case <STR_LIT> : case <STR_LIT> : if ( ! isWQM ) { if ( ! ST -> hasCaymanISA ( ) && ! branchStackContains ( CFStack :: FIRST_NON_WQM_PUSH ) ) Item = CFStack :: FIRST_NON_WQM_PUSH ; else if ( CurrentEntries > <NUM_LIT> && ST -> getGeneration ( ) > <ISA_LIT>Subtarget :: EVERGREEN && ! ST -> hasCaymanISA ( ) && ! branchStackContains ( CFStack :: FIRST_NON_WQM_PUSH_W_FULL_ENTRY ) ) Item = CFStack :: FIRST_NON_WQM_PUSH_W_FULL_ENTRY ; else Item = CFStack :: SUB_ENTRY ; } else Item = CFStack :: ENTRY ; break ; } BranchStack . push_back ( Item ) ; if ( Item == CFStack :: ENTRY ) CurrentEntries ++ ;

else CurrentSubEntries += getSubEntrySize ( Item ) ;


<s> <ISA_LIT>ControlFlowFinalizer ( ) : MachineFunctionPass ( ID ) {


<s> if ( Opcode == <STR_LIT> && ST -> hasCaymanISA ( ) && getLoopDepth ( ) > <NUM_LIT> ) return true ; if ( ! ST -> hasCFAluBug ( ) ) return false ; switch ( Opcode ) { default : return false ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( CurrentSubEntries == <NUM_LIT> ) return false ; if ( ST -> getWavefrontSize ( ) == <NUM_LIT> ) {


<s> void CFStack :: updateMaxStackSize ( ) { unsigned CurrentStackSize = CurrentEntries + ( alignTo ( CurrentSubEntries , <NUM_LIT> ) / <NUM_LIT> ) ;

MaxStackSize = std :: max ( CurrentStackSize , MaxStackSize ) ;


<s> CFStack ( const <ISA_LIT>Subtarget * st , unsigned ShaderType ) : ST ( st ) , MaxStackSize ( ShaderType == ShaderType :: VERTEX ? <NUM_LIT> : <NUM_LIT> ) , CurrentEntries ( <NUM_LIT> ) , CurrentSubEntries ( <NUM_LIT> ) {


<s> for ( MachineInstr * MI : MIs ) { CounterPropagateAddr ( MI , Addr ) ; }


<s> void CounterPropagateAddr ( const std :: set < MachineInstr * > & MIs , unsigned Addr ) const { for ( MachineInstr * MI : MIs ) { CounterPropagateAddr ( MI , Addr ) ; }


<s> llvm :: FunctionPass * llvm :: create<ISA_LIT>ControlFlowFinalizer ( TargetMachine & TM ) { return new <ISA_LIT>ControlFlowFinalizer ( TM ) ;


<s> void EmitALUClause ( MachineBasicBlock :: iterator InsertPos , ClauseFile & Clause , unsigned & CfCount ) { Clause . first -> getOperand ( <NUM_LIT> ) . setImm ( <NUM_LIT> ) ; CounterPropagateAddr ( Clause . first , CfCount ) ; MachineBasicBlock * BB = Clause . first -> getParent ( ) ; BuildMI ( BB , InsertPos -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( CfCount ) ; for ( unsigned i = <NUM_LIT> , e = Clause . second . size ( ) ; i < e ; ++ i ) { BB -> splice ( InsertPos , BB , Clause . second [ i ] ) ;


<s> Clause . first -> getOperand ( <NUM_LIT> ) . setImm ( <NUM_LIT> ) ; CounterPropagateAddr ( Clause . first , CfCount ) ; MachineBasicBlock * BB = Clause . first -> getParent ( ) ; BuildMI ( BB , InsertPos -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( CfCount ) ;

for ( unsigned i = <NUM_LIT> , e = Clause . second . size ( ) ; i < e ; ++ i ) {


<s> CounterPropagateAddr ( Clause . first , CfCount ) ; MachineBasicBlock * BB = Clause . first -> getParent ( ) ; BuildMI ( BB , InsertPos -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( CfCount ) ; for ( unsigned i = <NUM_LIT> , e = Clause . second . size ( ) ; i < e ; ++ i ) { BB -> splice ( InsertPos , BB , Clause . second [ i ] ) ;


<s> const MCInstrDesc & getHWInstrDesc ( ControlFlowInstruction CFI ) const { unsigned Opcode = <NUM_LIT> ; bool isEg = ( ST -> getGeneration ( ) >= <ISA_LIT>Subtarget :: EVERGREEN ) ; switch ( CFI ) { case CF_TC : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_VC : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_CALL_FS : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_WHILE_LOOP : Opcode = isEg ? <STR_LIT> : <STR_LIT> ;


<s> void getLiteral ( MachineInstr * MI , std :: vector < int64_t > & Lits ) const { static const unsigned LiteralRegs [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; const SmallVector < std :: pair < MachineOperand * , int64_t > , <NUM_LIT> > Srcs = TII -> getSrcs ( MI ) ; for ( unsigned i = <NUM_LIT> , e = Srcs . size ( ) ; i < e ; ++ i ) { if ( Srcs [ i ] . first -> getReg ( ) != <STR_LIT> ) continue ; int64_t Imm = Srcs [ i ] . second ; std :: vector < int64_t > :: iterator It = std :: find ( Lits . begin ( ) , Lits . end ( ) , Imm ) ; if ( It != Lits . end ( ) ) { unsigned Index = It - Lits . begin ( ) ; Srcs [ i ] . first -> setReg ( LiteralRegs [ Index ] ) ; } else { assert ( Lits . size ( ) < <NUM_LIT> && <STR_LIT> ) ;

Srcs [ i ] . first -> setReg ( LiteralRegs [ Lits . size ( ) ] ) ;


<s> } else { return <NUM_LIT> ; } case CFStack :: FIRST_NON_WQM_PUSH_W_FULL_ENTRY : assert ( ST -> getGeneration ( ) >= <ISA_LIT>Subtarget :: EVERGREEN ) ;


<s> unsigned LiteralPair0 = Literals [ i ] ; unsigned LiteralPair1 = ( i + <NUM_LIT> < e ) ? Literals [ i + <NUM_LIT> ] : <NUM_LIT> ;

InsertPos = BuildMI ( MBB , InsertPos -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( LiteralPair0 ) . addImm ( LiteralPair1 ) ;


<s> if ( ! MO . isReg ( ) ) continue ; if ( MO . isDef ( ) ) { unsigned Reg = MO . getReg ( ) ; if ( <STR_LIT> . contains ( Reg ) ) DstMI = Reg ; else DstMI = TRI -> getMatchingSuperReg ( Reg , TRI -> getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ; } if ( MO . isUse ( ) ) {


<s> else DstMI = TRI -> getMatchingSuperReg ( Reg , TRI -> getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ; } if ( MO . isUse ( ) ) { unsigned Reg = MO . getReg ( ) ; if ( <STR_LIT> . contains ( Reg ) ) SrcMI = Reg ; else SrcMI = TRI -> getMatchingSuperReg ( Reg , TRI -> getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ; } } if ( ( DstRegs . find ( SrcMI ) == DstRegs . end ( ) ) ) {


<s> case <STR_LIT> : case <STR_LIT> : return true ; default :


<s> if ( MO . isReg ( ) && MO . isInternalRead ( ) ) MO . setIsInternalRead ( false ) ; } getLiteral ( & * BI , Literals ) ; ClauseContent . push_back ( & * BI ) ; } I = BI ; DeleteMI -> eraseFromParent ( ) ; } else { getLiteral ( I , Literals ) ; ClauseContent . push_back ( I ) ; I ++ ; } for ( unsigned i = <NUM_LIT> , e = Literals . size ( ) ; i < e ; i += <NUM_LIT> ) { unsigned literal0 = Literals [ i ] ; unsigned literal2 = ( i + <NUM_LIT> < e ) ? Literals [ i + <NUM_LIT> ] : <NUM_LIT> ; MachineInstr * MILit = BuildMI ( MBB , I , I -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( literal0 ) . addImm ( literal2 ) ;


<s> ClauseFile MakeFetchClause ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator & I ) const { MachineBasicBlock :: iterator ClauseHead = I ; std :: vector < MachineInstr * > ClauseContent ; unsigned AluInstCount = <NUM_LIT> ; bool IsTex = TII -> usesTextureCache ( ClauseHead ) ; std :: set < unsigned > DstRegs ; for ( MachineBasicBlock :: iterator E = MBB . end ( ) ; I != E ; ++ I ) { if ( IsTrivialInst ( I ) ) continue ; if ( AluInstCount >= MaxFetchInst ) break ; if ( ( IsTex && ! TII -> usesTextureCache ( I ) ) || ( ! IsTex && ! TII -> usesVertexCache ( I ) ) ) break ; if ( ! isCompatibleWithClause ( I , DstRegs ) ) break ;


<s> CFStack :: StackItem Top = BranchStack . back ( ) ; if ( Top == CFStack :: ENTRY ) CurrentEntries -- ; else CurrentSubEntries -= getSubEntrySize ( Top ) ;


<s> if ( Top == CFStack :: ENTRY ) CurrentEntries -- ; else CurrentSubEntries -= getSubEntrySize ( Top ) ; BranchStack . pop_back ( ) ;


<s> if ( ! isWQM ) { if ( ! ST -> hasCaymanISA ( ) && ! branchStackContains ( CFStack :: FIRST_NON_WQM_PUSH ) ) Item = CFStack :: FIRST_NON_WQM_PUSH ; else if ( CurrentEntries > <NUM_LIT> && ST -> getGeneration ( ) > <ISA_LIT>Subtarget :: EVERGREEN && ! ST -> hasCaymanISA ( ) && ! branchStackContains ( CFStack :: FIRST_NON_WQM_PUSH_W_FULL_ENTRY ) ) Item = CFStack :: FIRST_NON_WQM_PUSH_W_FULL_ENTRY ; else Item = CFStack :: SUB_ENTRY ; } else Item = CFStack :: ENTRY ; break ; } BranchStack . push_back ( Item ) ; if ( Item == CFStack :: ENTRY ) CurrentEntries ++ ; else CurrentSubEntries += getSubEntrySize ( Item ) ; updateMaxStackSize ( ) ;


LoopStack . push_back ( CFStack :: ENTRY ) ;


<s> <ISA_LIT>ControlFlowFinalizer ( TargetMachine & tm ) : MachineFunctionPass ( ID ) , TII ( nullptr ) , TRI ( nullptr ) , ST ( nullptr ) {


<s> if ( ! ST -> hasCFAluBug ( ) ) return false ; switch ( Opcode ) { default : return false ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :

if ( CurrentSubEntries == <NUM_LIT> ) return false ;


<s> void CFStack :: updateMaxStackSize ( ) { unsigned CurrentStackSize = CurrentEntries + ( RoundUpToAlignment ( CurrentSubEntries , <NUM_LIT> ) / <NUM_LIT> ) ;


<s> MachineBasicBlock * BB = Clause . first -> getParent ( ) ; BuildMI ( BB , DL , TII -> get ( <STR_LIT> ) ) . addImm ( CfCount ) ; for ( unsigned i = <NUM_LIT> , e = Clause . second . size ( ) ; i < e ; ++ i ) { BB -> splice ( InsertPos , BB , Clause . second [ i ] ) ; } CfCount += Clause . second . size ( ) ;


<s> void EmitALUClause ( MachineBasicBlock :: iterator InsertPos , const DebugLoc & DL , ClauseFile & Clause , unsigned & CfCount ) { Clause . first -> getOperand ( <NUM_LIT> ) . setImm ( <NUM_LIT> ) ; CounterPropagateAddr ( * Clause . first , CfCount ) ; MachineBasicBlock * BB = Clause . first -> getParent ( ) ; BuildMI ( BB , DL , TII -> get ( <STR_LIT> ) ) . addImm ( CfCount ) ; for ( unsigned i = <NUM_LIT> , e = Clause . second . size ( ) ; i < e ; ++ i ) { BB -> splice ( InsertPos , BB , Clause . second [ i ] ) ;


<s> CounterPropagateAddr ( * Clause . first , CfCount ) ; MachineBasicBlock * BB = Clause . first -> getParent ( ) ;


<s> int64_t Imm = Src . second ; std :: vector < MachineOperand * > :: iterator It = llvm :: find_if ( Lits , [ & ] ( MachineOperand * val ) { return val -> isImm ( ) && ( val -> getImm ( ) == Imm ) ; } ) ; MachineOperand & Operand = MI . getOperand ( TII -> getOperandIdx ( MI . getOpcode ( ) , <STR_LIT> :: literal ) ) ; if ( It != Lits . end ( ) ) { unsigned Index = It - Lits . begin ( ) ; Src . first -> setReg ( LiteralRegs [ Index ] ) ; } else { assert ( Lits . size ( ) < <NUM_LIT> && <STR_LIT> ) ; Src . first -> setReg ( LiteralRegs [ Lits . size ( ) ] ) ; Lits . push_back ( & Operand ) ; } }


<s> std :: vector < MachineOperand * > :: iterator It = llvm :: find_if ( Lits , [ & ] ( MachineOperand * val ) { return val -> isImm ( ) && ( val -> getImm ( ) == Imm ) ; } ) ; MachineOperand & Operand = MI . getOperand ( TII -> getOperandIdx ( MI . getOpcode ( ) , <STR_LIT> :: literal ) ) ; if ( It != Lits . end ( ) ) { unsigned Index = It - Lits . begin ( ) ; Src . first -> setReg ( LiteralRegs [ Index ] ) ; } else { assert ( Lits . size ( ) < <NUM_LIT> && <STR_LIT> ) ; Src . first -> setReg ( LiteralRegs [ Lits . size ( ) ] ) ; Lits . push_back ( & Operand ) ; }


<s> for ( MachineInstr :: const_mop_iterator I = MI . operands_begin ( ) , E = MI . operands_end ( ) ; I != E ; ++ I ) { const MachineOperand & MO = * I ; if ( ! MO . isReg ( ) ) continue ; if ( MO . isDef ( ) ) { unsigned Reg = MO . getReg ( ) ; if ( <STR_LIT> . contains ( Reg ) ) DstMI = Reg ;

else DstMI = TRI -> getMatchingSuperReg ( Reg , <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ;


<s> bool CFStack :: branchStackContains ( CFStack :: StackItem Item ) {

return llvm :: is_contained ( BranchStack , Item ) ;


<s> bool CFStack :: branchStackContains ( CFStack :: StackItem Item ) { return llvm :: is_contained ( BranchStack , Item ) ;


<s> BuildMI ( BB , DL , TII -> get ( <STR_LIT> ) ) . addImm ( CfCount ) ; for ( MachineInstr * MI : Clause . second ) BB -> splice ( InsertPos , BB , MI ) ;


<s> BuildMI ( BB , DL , TII -> get ( <STR_LIT> ) ) . addImm ( CfCount ) ;

for ( MachineInstr * MI : Clause . second ) BB -> splice ( InsertPos , BB , MI ) ;


<s> MachineBasicBlock * BB = Clause . first -> getParent ( ) ; BuildMI ( BB , DL , TII -> get ( <STR_LIT> ) ) . addImm ( CfCount ) ;


<s> for ( MachineInstr * MI : Clause . second ) BB -> splice ( InsertPos , BB , MI ) ; CfCount += <NUM_LIT> * Clause . second . size ( ) ;


<s> Register Reg = MO . getReg ( ) ; if ( <STR_LIT> . contains ( Reg ) ) DstMI = Reg ; else DstMI = TRI -> getMatchingSuperReg ( Reg , <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ; } if ( MO . isUse ( ) ) { Register Reg = MO . getReg ( ) ; if ( <STR_LIT> . contains ( Reg ) ) SrcMI = Reg ; else SrcMI = TRI -> getMatchingSuperReg ( Reg , <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ; } }

if ( ( DstRegs . find ( SrcMI ) == DstRegs . end ( ) ) ) {


<s> void CFStack :: updateMaxStackSize ( ) { unsigned CurrentStackSize = CurrentEntries + divideCeil ( CurrentSubEntries , <NUM_LIT> ) ;


<s> CFStack ( const <ISA_LIT>Subtarget * st , CallingConv :: ID cc ) : ST ( st ) , MaxStackSize ( cc == CallingConv :: <ISA_LIT>_VS ? <NUM_LIT> : <NUM_LIT> ) , CurrentEntries ( <NUM_LIT> ) , CurrentSubEntries ( <NUM_LIT> ) {


<s> Clause . first -> getOperand ( <NUM_LIT> ) . setImm ( <NUM_LIT> ) ; CounterPropagateAddr ( * Clause . first , CfCount ) ; MachineBasicBlock * BB = Clause . first -> getParent ( ) ;

BuildMI ( BB , InsertPos -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( CfCount ) ;


<s> for ( unsigned i = <NUM_LIT> , e = Clause . second . size ( ) ; i < e ; ++ i ) { BB -> splice ( InsertPos , BB , Clause . second [ i ] ) ;


<s> CounterPropagateAddr ( * Clause . first , CfCount ) ; MachineBasicBlock * BB = Clause . first -> getParent ( ) ; BuildMI ( BB , InsertPos -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( CfCount ) ; for ( unsigned i = <NUM_LIT> , e = Clause . second . size ( ) ; i < e ; ++ i ) { BB -> splice ( InsertPos , BB , Clause . second [ i ] ) ;


<s> CounterPropagateAddr ( * Clause . first , CfCount ) ; MachineBasicBlock * BB = Clause . first -> getParent ( ) ; BuildMI ( BB , InsertPos -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( CfCount ) ; for ( unsigned i = <NUM_LIT> , e = Clause . second . size ( ) ; i < e ; ++ i ) { BB -> splice ( InsertPos , BB , Clause . second [ i ] ) ; }

CfCount += <NUM_LIT> * Clause . second . size ( ) ;


<s> break ; case CF_LOOP_CONTINUE : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_JUMP : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_ELSE : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_POP : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_END : if ( ST -> hasCaymanISA ( ) ) { Opcode = <STR_LIT> ; break ; }


<s> bool isEg = ( ST -> getGeneration ( ) >= <ISA_LIT>Subtarget :: EVERGREEN ) ; switch ( CFI ) { case CF_TC : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_VC : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_CALL_FS : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_WHILE_LOOP : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_END_LOOP : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_LOOP_BREAK : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_LOOP_CONTINUE :


<s> <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; const SmallVector < std :: pair < MachineOperand * , int64_t > , <NUM_LIT> > Srcs = TII -> getSrcs ( MI ) ; for ( const auto & Src : Srcs ) { if ( Src . first -> getReg ( ) != <STR_LIT> ) continue ; int64_t Imm = Src . second ; std :: vector < MachineOperand * > :: iterator It = std :: find_if ( Lits . begin ( ) , Lits . end ( ) , [ & ] ( MachineOperand * val ) { return val -> isImm ( ) && ( val -> getImm ( ) == Imm ) ; } ) ; MachineOperand & Operand = MI . getOperand ( TII -> getOperandIdx ( MI . getOpcode ( ) , <STR_LIT> :: literal ) ) ; if ( It != Lits . end ( ) ) { unsigned Index = It - Lits . begin ( ) ; Src . first -> setReg ( LiteralRegs [ Index ] ) ; } else { assert ( Lits . size ( ) < <NUM_LIT> && <STR_LIT> ) ; Src . first -> setReg ( LiteralRegs [ Lits . size ( ) ] ) ;


<s> int64_t Imm = Src . second ; std :: vector < MachineOperand * > :: iterator It = std :: find_if ( Lits . begin ( ) , Lits . end ( ) , [ & ] ( MachineOperand * val ) { return val -> isImm ( ) && ( val -> getImm ( ) == Imm ) ; } ) ;

MachineOperand & Operand = MI . getOperand ( TII -> getOperandIdx ( MI . getOpcode ( ) , <STR_LIT> :: literal ) ) ;


<s> switch ( Item ) { default : return <NUM_LIT> ; case CFStack :: FIRST_NON_WQM_PUSH : assert ( ! ST -> hasCaymanISA ( ) ) ;

if ( ST -> getGeneration ( ) <= <ISA_LIT>Subtarget :: R700 ) {


<s> if ( ST -> getGeneration ( ) <= <ISA_LIT>Subtarget :: R700 ) { return <NUM_LIT> ; } else { return <NUM_LIT> ; }

case CFStack :: FIRST_NON_WQM_PUSH_W_FULL_ENTRY :


<s> I != E ; ++ I ) { const MachineOperand & MO = * I ; if ( ! MO . isReg ( ) ) continue ; if ( MO . isDef ( ) ) { unsigned Reg = MO . getReg ( ) ; if ( <STR_LIT> . contains ( Reg ) ) DstMI = Reg ; else DstMI = TRI -> getMatchingSuperReg ( Reg , TRI -> getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ; } if ( MO . isUse ( ) ) { unsigned Reg = MO . getReg ( ) ; if ( <STR_LIT> . contains ( Reg ) ) SrcMI = Reg ; else SrcMI = TRI -> getMatchingSuperReg ( Reg , TRI -> getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ; }


<s> I != E ; ++ I ) { const MachineOperand & MO = * I ; if ( ! MO . isReg ( ) ) continue ; if ( MO . isDef ( ) ) { unsigned Reg = MO . getReg ( ) ; if ( <STR_LIT> . contains ( Reg ) ) DstMI = Reg ; else DstMI = TRI -> getMatchingSuperReg ( Reg , TRI -> getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ; } if ( MO . isUse ( ) ) { unsigned Reg = MO . getReg ( ) ; if ( <STR_LIT> . contains ( Reg ) ) SrcMI = Reg ; else SrcMI = TRI -> getMatchingSuperReg ( Reg , TRI -> getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ; } } if ( ( DstRegs . find ( SrcMI ) == DstRegs . end ( ) ) ) { DstRegs . insert ( DstMI ) ;


<s> case <STR_LIT> : return true ; default : return false ;


<s> bool IsTrivialInst ( MachineInstr & MI ) const { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : return true ;


<s> ++ I ; continue ; } if ( ! I -> isBundle ( ) && ! TII -> isALUInstr ( I -> getOpcode ( ) ) ) break ; std :: vector < MachineOperand * > Literals ; if ( I -> isBundle ( ) ) { MachineInstr & DeleteMI = * I ; MachineBasicBlock :: instr_iterator BI = I . getInstrIterator ( ) ; while ( ++ BI != E && BI -> isBundledWithPred ( ) ) { BI -> unbundleFromPred ( ) ; for ( MachineOperand & MO : BI -> operands ( ) ) { if ( MO . isReg ( ) && MO . isInternalRead ( ) ) MO . setIsInternalRead ( false ) ; } getLiteral ( * BI , Literals ) ; ClauseContent . push_back ( & * BI ) ; } I = BI ; DeleteMI . eraseFromParent ( ) ; } else { getLiteral ( * I , Literals ) ; ClauseContent . push_back ( & * I ) ; I ++ ; } for ( unsigned i = <NUM_LIT> , e = Literals . size ( ) ; i < e ; i += <NUM_LIT> ) { MachineInstrBuilder MILit = BuildMI ( MBB , I , I -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; if ( Literals [ i ] -> isImm ( ) ) { MILit . addImm ( Literals [ i ] -> getImm ( ) ) ; } else { MILit . addGlobalAddress ( Literals [ i ] -> getGlobal ( ) , Literals [ i ] -> getOffset ( ) ) ; } if ( i + <NUM_LIT> < e ) { if ( Literals [ i + <NUM_LIT> ] -> isImm ( ) ) { MILit . addImm ( Literals [ i + <NUM_LIT> ] -> getImm ( ) ) ; } else { MILit . addGlobalAddress ( Literals [ i + <NUM_LIT> ] -> getGlobal ( ) , Literals [ i + <NUM_LIT> ] -> getOffset ( ) ) ; } } else MILit . addImm ( <NUM_LIT> ) ;


<s> case <STR_LIT> : if ( ! isWQM ) { if ( ! ST -> hasCaymanISA ( ) && ! branchStackContains ( CFStack :: FIRST_NON_WQM_PUSH ) ) Item = CFStack :: FIRST_NON_WQM_PUSH ; else if ( CurrentEntries > <NUM_LIT> && ST -> getGeneration ( ) > <ISA_LIT>Subtarget :: EVERGREEN && ! ST -> hasCaymanISA ( ) && ! branchStackContains ( CFStack :: FIRST_NON_WQM_PUSH_W_FULL_ENTRY ) ) Item = CFStack :: FIRST_NON_WQM_PUSH_W_FULL_ENTRY ; else Item = CFStack :: SUB_ENTRY ; } else Item = CFStack :: ENTRY ;


<s> } case <STR_LIT> : { LastAlu . push_back ( nullptr ) ; MachineInstr * MIb = BuildMI ( MBB , MI , MBB . findDebugLoc ( MI ) , getHWInstrDesc ( CF_JUMP ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; IfThenElseStack . push_back ( MIb ) ; DEBUG ( dbgs ( ) << CfCount << <STR_LIT> ; MIb -> dump ( ) ; ) ; MI -> eraseFromParent ( ) ; CfCount ++ ; break ; } case <STR_LIT> : { MachineInstr * JumpInst = IfThenElseStack . back ( ) ; IfThenElseStack . pop_back ( ) ; CounterPropagateAddr ( * JumpInst , CfCount ) ; MachineInstr * MIb = BuildMI ( MBB , MI , MBB . findDebugLoc ( MI ) , getHWInstrDesc ( CF_ELSE ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; DEBUG ( dbgs ( ) << CfCount << <STR_LIT> ; MIb -> dump ( ) ; ) ; IfThenElseStack . push_back ( MIb ) ; MI -> eraseFromParent ( ) ; CfCount ++ ; break ; } case <STR_LIT> : { CFStack . popBranch ( ) ; if ( LastAlu . back ( ) ) { ToPopAfter . push_back ( LastAlu . back ( ) ) ; } else { MachineInstr * MIb = BuildMI ( MBB , MI , MBB . findDebugLoc ( MI ) , getHWInstrDesc ( CF_POP ) ) . addImm ( CfCount + <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; ( void ) MIb ; DEBUG ( dbgs ( ) << CfCount << <STR_LIT> ; MIb -> dump ( ) ; ) ; CfCount ++ ; } MachineInstr * IfOrElseInst = IfThenElseStack . back ( ) ; IfThenElseStack . pop_back ( ) ; CounterPropagateAddr ( * IfOrElseInst , CfCount ) ; IfOrElseInst -> getOperand ( <NUM_LIT> ) . setImm ( <NUM_LIT> ) ; LastAlu . pop_back ( ) ; MI -> eraseFromParent ( ) ; break ; } case <STR_LIT> : { CfCount ++ ; MachineInstr * MIb = BuildMI ( MBB , MI , MBB . findDebugLoc ( MI ) , getHWInstrDesc ( CF_LOOP_BREAK ) ) . addImm ( <NUM_LIT> ) ; LoopStack . back ( ) . second . insert ( MIb ) ; MI -> eraseFromParent ( ) ; break ; } case <STR_LIT> : { MachineInstr * MIb = BuildMI ( MBB , MI , MBB . findDebugLoc ( MI ) , getHWInstrDesc ( CF_LOOP_CONTINUE ) ) . addImm ( <NUM_LIT> ) ; LoopStack . back ( ) . second . insert ( MIb ) ; MI -> eraseFromParent ( ) ; CfCount ++ ; break ; } case <STR_LIT> : { DebugLoc DL = MBB . findDebugLoc ( MI ) ;

BuildMI ( MBB , MI , DL , getHWInstrDesc ( CF_END ) ) ;


<s> CFStack ( const <ISA_LIT>Subtarget * st , CallingConv :: ID cc ) : ST ( st ) , MaxStackSize ( cc == CallingConv :: <ISA_LIT>_VS ? <NUM_LIT> : <NUM_LIT> ) , CurrentEntries ( <NUM_LIT> ) , CurrentSubEntries ( <NUM_LIT> ) {


<s> const SmallVector < std :: pair < MachineOperand * , int64_t > , <NUM_LIT> > Srcs = TII -> getSrcs ( MI ) ; for ( const auto & Src : Srcs ) { if ( Src . first -> getReg ( ) != <STR_LIT> ) continue ; int64_t Imm = Src . second ; std :: vector < MachineOperand * > :: iterator It = std :: find_if ( Lits . begin ( ) , Lits . end ( ) , [ & ] ( MachineOperand * val ) { return val -> isImm ( ) && ( val -> getImm ( ) == Imm ) ; } ) ; MachineOperand & Operand = MI -> getOperand ( TII -> getOperandIdx ( MI -> getOpcode ( ) , <STR_LIT> :: literal ) ) ; if ( It != Lits . end ( ) ) { unsigned Index = It - Lits . begin ( ) ;

Src . first -> setReg ( LiteralRegs [ Index ] ) ;


<s> Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_VC : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_CALL_FS : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_WHILE_LOOP : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_END_LOOP : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_LOOP_BREAK : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_LOOP_CONTINUE : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ;


<s> case CF_TC : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_VC : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_CALL_FS : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_WHILE_LOOP : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_END_LOOP : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_LOOP_BREAK : Opcode = isEg ? <STR_LIT> : <STR_LIT> ; break ; case CF_LOOP_CONTINUE : Opcode = isEg ? <STR_LIT> : <STR_LIT> ;


<s> CFStack :: StackItem Item = CFStack :: ENTRY ; switch ( Opcode ) { case <STR_LIT> : case <STR_LIT> : if ( ! isWQM ) { if ( ! ST -> hasCaymanISA ( ) && ! branchStackContains ( CFStack :: FIRST_NON_WQM_PUSH ) ) Item = CFStack :: FIRST_NON_WQM_PUSH ; else if ( CurrentEntries > <NUM_LIT> && ST -> getGeneration ( ) > <ISA_LIT>Subtarget :: EVERGREEN && ! ST -> hasCaymanISA ( ) && ! branchStackContains ( CFStack :: FIRST_NON_WQM_PUSH_W_FULL_ENTRY ) ) Item = CFStack :: FIRST_NON_WQM_PUSH_W_FULL_ENTRY ; else Item = CFStack :: SUB_ENTRY ; } else Item = CFStack :: ENTRY ; break ; } BranchStack . push_back ( Item ) ; if ( Item == CFStack :: ENTRY ) CurrentEntries ++ ; else CurrentSubEntries += getSubEntrySize ( Item ) ;


<s> I = BI ; DeleteMI -> eraseFromParent ( ) ; } else { getLiteral ( I , Literals ) ; ClauseContent . push_back ( I ) ; I ++ ; } for ( unsigned i = <NUM_LIT> , e = Literals . size ( ) ; i < e ; i += <NUM_LIT> ) { unsigned literal0 = Literals [ i ] ; unsigned literal2 = ( i + <NUM_LIT> < e ) ? Literals [ i + <NUM_LIT> ] : <NUM_LIT> ; MachineInstr * MILit = BuildMI ( MBB , I , I -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( literal0 ) . addImm ( literal2 ) ; ClauseContent . push_back ( MILit ) ; } }

assert ( ClauseContent . size ( ) < <NUM_LIT> && <STR_LIT> ) ;


<s> <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; const SmallVector < std :: pair < MachineOperand * , int64_t > , <NUM_LIT> > Srcs = TII -> getSrcs ( MI ) ; for ( const auto & Src : Srcs ) { if ( Src . first -> getReg ( ) != <STR_LIT> ) continue ; int64_t Imm = Src . second ; std :: vector < MachineOperand * > :: iterator It = find_if ( Lits , [ & ] ( MachineOperand * val ) { return val -> isImm ( ) && ( val -> getImm ( ) == Imm ) ;


<s> Register Reg = MO . getReg ( ) ; if ( <STR_LIT> . contains ( Reg ) ) DstMI = Reg ; else DstMI = TRI -> getMatchingSuperReg ( Reg , <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ; } if ( MO . isUse ( ) ) { Register Reg = MO . getReg ( ) ; if ( <STR_LIT> . contains ( Reg ) ) SrcMI = Reg ; else SrcMI = TRI -> getMatchingSuperReg ( Reg , <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ; } } if ( ( DstRegs . find ( SrcMI ) == DstRegs . end ( ) ) ) { DstRegs . insert ( DstMI ) ;


<s> unsigned DstMI , SrcMI ; for ( MachineInstr :: const_mop_iterator I = MI . operands_begin ( ) , E = MI . operands_end ( ) ; I != E ; ++ I ) { const MachineOperand & MO = * I ; if ( ! MO . isReg ( ) ) continue ; if ( MO . isDef ( ) ) { Register Reg = MO . getReg ( ) ; if ( <STR_LIT> . contains ( Reg ) ) DstMI = Reg ; else DstMI = TRI -> getMatchingSuperReg ( Reg , <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( TRI -> getHWRegChan ( Reg ) ) , & <STR_LIT> ) ;


<s> FunctionPass * llvm :: create<ISA_LIT>ControlFlowFinalizer ( TargetMachine & TM ) {

return new <ISA_LIT>ControlFlowFinalizer ( TM ) ;


<s> FunctionPass * llvm :: create<ISA_LIT>ControlFlowFinalizer ( TargetMachine & TM ) { return new <ISA_LIT>ControlFlowFinalizer ( TM ) ;


<s> <ISA_LIT>ControlFlowFinalizer ( TargetMachine & tm ) : MachineFunctionPass ( ID ) {


<s> for ( MachineBasicBlock :: iterator UseI = Def ; UseI != BBEnd ; ++ UseI ) { AluInstCount += OccupiedDwords ( * UseI ) ; if ( ! SubstituteKCacheBank ( * UseI , KCacheBanks , false ) ) return false ; if ( AluInstCount >= TII -> getMaxAlusPerClause ( ) ) return false ;

if ( UseI -> readsRegister ( MOI -> getReg ( ) , & TRI ) ) LastUseCount = AluInstCount ;


<s> for ( MachineInstr :: const_mop_iterator MOI = Def -> operands_begin ( ) , MOE = Def -> operands_end ( ) ; MOI != MOE ; ++ MOI ) { if ( ! MOI -> isReg ( ) || ! MOI -> isDef ( ) || TRI . isPhysRegLiveAcrossClauses ( MOI -> getReg ( ) ) ) continue ; unsigned LastUseCount = <NUM_LIT> ;

for ( MachineBasicBlock :: iterator UseI = Def ; UseI != BBEnd ; ++ UseI ) {


<s> FunctionPass * llvm :: create<ISA_LIT>EmitClauseMarkers ( ) { return new <ISA_LIT>EmitClauseMarkers ( ) ;


<s> StringRef getPassName ( ) const override { return <STR_LIT> ;


<s> bool isALU ( const MachineInstr & MI ) const { if ( TII -> isALUInstr ( MI . getOpcode ( ) ) ) return true ;

if ( TII -> isVector ( MI ) || TII -> isCubeOp ( MI . getOpcode ( ) ) ) return true ;


<s> bool IsTrivialInst ( MachineInstr & MI ) const { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> :


<s> BuildMI ( MBB , ClauseHead , MBB . findDebugLoc ( ClauseHead ) , TII -> get ( Opcode ) ) . addImm ( Address ++ ) . addImm ( KCacheBanks . empty ( ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . first ) . addImm ( ( KCacheBanks . size ( ) < <NUM_LIT> ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . first ) . addImm ( KCacheBanks . empty ( ) ? <NUM_LIT> : <NUM_LIT> ) . addImm ( ( KCacheBanks . size ( ) < <NUM_LIT> ) ? <NUM_LIT> : <NUM_LIT> ) . addImm ( KCacheBanks . empty ( ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . second ) . addImm ( ( KCacheBanks . size ( ) < <NUM_LIT> ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . second ) . addImm ( AluInstCount ) . addImm ( <NUM_LIT> ) ;


<s> unsigned NumLiteral = <NUM_LIT> ; for ( MachineInstr :: mop_iterator It = MI . operands_begin ( ) , E = MI . operands_end ( ) ; It != E ; ++ It ) { MachineOperand & MO = * It ; if ( MO . isReg ( ) && MO . getReg ( ) == <STR_LIT> ) ++ NumLiteral ; }


<s> TII = ST . getInstrInfo ( ) ; for ( MachineFunction :: iterator BB = MF . begin ( ) , BB_E = MF . end ( ) ; BB != BB_E ; ++ BB ) { MachineBasicBlock & MBB = * BB ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; if ( I != MBB . end ( ) && I -> getOpcode ( ) == <STR_LIT> ) continue ; for ( MachineBasicBlock :: iterator E = MBB . end ( ) ; I != E ; ) { if ( isALU ( * I ) ) { auto next = MakeALUClause ( MBB , I ) ;


<s> const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = ST . getInstrInfo ( ) ; for ( MachineFunction :: iterator BB = MF . begin ( ) , BB_E = MF . end ( ) ; BB != BB_E ; ++ BB ) { MachineBasicBlock & MBB = * BB ;

MachineBasicBlock :: iterator I = MBB . begin ( ) ;


<s> bool SubstituteKCacheBank ( MachineInstr & MI , std :: vector < std :: pair < unsigned , unsigned >> & CachedConsts , bool UpdateInstr = true ) const { std :: vector < std :: pair < unsigned , unsigned >> UsedKCache ; if ( ! TII -> isALUInstr ( MI . getOpcode ( ) ) && MI . getOpcode ( ) != <STR_LIT> ) return true ; const SmallVectorImpl < std :: pair < MachineOperand * , int64_t >> & Consts = TII -> getSrcs ( MI ) ; assert ( ( TII -> isALUInstr ( MI . getOpcode ( ) ) || MI . getOpcode ( ) == <STR_LIT> ) && <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> , n = Consts . size ( ) ; i < n ; ++ i ) { if ( Consts [ i ] . first -> getReg ( ) != <STR_LIT> ) continue ; unsigned Sel = Consts [ i ] . second ; unsigned Chan = Sel & <NUM_LIT> , Index = ( ( Sel >> <NUM_LIT> ) - <NUM_LIT> ) & <NUM_LIT> ;

unsigned KCacheIndex = Index * <NUM_LIT> + Chan ;


<s> for ( MachineBasicBlock :: iterator UseI = Def ; UseI != BBEnd ; ++ UseI ) { AluInstCount += OccupiedDwords ( UseI ) ; if ( ! SubstituteKCacheBank ( UseI , KCacheBanks , false ) ) return false ; if ( AluInstCount >= TII -> getMaxAlusPerClause ( ) ) return false ; if ( UseI -> findRegisterUseOperandIdx ( MOI -> getReg ( ) ) ) LastUseCount = AluInstCount ; if ( UseI != Def && UseI -> findRegisterDefOperandIdx ( MOI -> getReg ( ) ) != - <NUM_LIT> ) break ;


<s> std :: pair < unsigned , unsigned > getAccessedBankLine ( unsigned Sel ) const { return std :: pair < unsigned , unsigned > ( ( ( Sel >> <NUM_LIT> ) - <NUM_LIT> ) >> <NUM_LIT> , ( ( ( ( Sel >> <NUM_LIT> ) - <NUM_LIT> ) & <NUM_LIT> ) >> <NUM_LIT> ) << <NUM_LIT> ) ;


<s> return std :: pair < unsigned , unsigned > ( ( ( Sel >> <NUM_LIT> ) - <NUM_LIT> ) >> <NUM_LIT> , ( ( ( ( Sel >> <NUM_LIT> ) - <NUM_LIT> ) & <NUM_LIT> ) >> <NUM_LIT> ) << <NUM_LIT> ) ;


<s> const char * getPassName ( ) const override { return <STR_LIT> ;


<s> if ( TII -> isALUInstr ( MI -> getOpcode ( ) ) ) return true ; if ( TII -> isVector ( * MI ) || TII -> isCubeOp ( MI -> getOpcode ( ) ) ) return true ; switch ( MI -> getOpcode ( ) ) { case <STR_LIT> :


<s> switch ( MI -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; default : return false ; }


<s> } if ( ! canClauseLocalKillFitInClause ( AluInstCount , KCacheBanks , I , E ) ) break ; if ( ! SubstituteKCacheBank ( I , KCacheBanks ) ) break ; AluInstCount += OccupiedDwords ( I ) ; } unsigned Opcode = PushBeforeModifier ? <STR_LIT> : <STR_LIT> ; BuildMI ( MBB , ClauseHead , MBB . findDebugLoc ( ClauseHead ) , TII -> get ( Opcode ) ) . addImm ( Address ++ ) . addImm ( KCacheBanks . empty ( ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . first ) . addImm ( ( KCacheBanks . size ( ) < <NUM_LIT> ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . first ) . addImm ( KCacheBanks . empty ( ) ? <NUM_LIT> : <NUM_LIT> ) . addImm ( ( KCacheBanks . size ( ) < <NUM_LIT> ) ? <NUM_LIT> : <NUM_LIT> ) . addImm ( KCacheBanks . empty ( ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . second ) . addImm ( ( KCacheBanks . size ( ) < <NUM_LIT> ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . second ) . addImm ( AluInstCount ) . addImm ( <NUM_LIT> ) ;


<s> unsigned NumLiteral = <NUM_LIT> ; for ( MachineInstr :: mop_iterator It = MI -> operands_begin ( ) , E = MI -> operands_end ( ) ; It != E ; ++ It ) { MachineOperand & MO = * It ;

if ( MO . isReg ( ) && MO . getReg ( ) == <STR_LIT> ) ++ NumLiteral ;


<s> bool runOnMachineFunction ( MachineFunction & MF ) override { TII = static_cast < const <ISA_LIT>InstrInfo * > ( MF . getSubtarget ( ) . getInstrInfo ( ) ) ; for ( MachineFunction :: iterator BB = MF . begin ( ) , BB_E = MF . end ( ) ; BB != BB_E ; ++ BB ) { MachineBasicBlock & MBB = * BB ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; if ( I -> getOpcode ( ) == <STR_LIT> ) continue ; for ( MachineBasicBlock :: iterator E = MBB . end ( ) ; I != E ; ) {

if ( isALU ( I ) ) I = MakeALUClause ( MBB , I ) ;


<s> UsedKCache . push_back ( std :: pair < unsigned , unsigned > ( <NUM_LIT> , KCacheIndex ) ) ; continue ; } if ( CachedConsts [ <NUM_LIT> ] == BankLine ) { UsedKCache . push_back ( std :: pair < unsigned , unsigned > ( <NUM_LIT> , KCacheIndex ) ) ; continue ; } if ( CachedConsts . size ( ) == <NUM_LIT> ) { CachedConsts . push_back ( BankLine ) ; UsedKCache . push_back ( std :: pair < unsigned , unsigned > ( <NUM_LIT> , KCacheIndex ) ) ; continue ; } if ( CachedConsts [ <NUM_LIT> ] == BankLine ) { UsedKCache . push_back ( std :: pair < unsigned , unsigned > ( <NUM_LIT> , KCacheIndex ) ) ; continue ; } return false ; } if ( ! UpdateInstr ) return true ; for ( unsigned i = <NUM_LIT> , j = <NUM_LIT> , n = Consts . size ( ) ; i < n ; ++ i ) { if ( Consts [ i ] . first -> getReg ( ) != <STR_LIT> ) continue ; switch ( UsedKCache [ j ] . first ) {


<s> AluInstCount += OccupiedDwords ( * UseI ) ; if ( ! SubstituteKCacheBank ( * UseI , KCacheBanks , false ) ) return false ; if ( AluInstCount >= TII -> getMaxAlusPerClause ( ) ) return false ; if ( UseI -> findRegisterUseOperandIdx ( MOI -> getReg ( ) ) ) LastUseCount = AluInstCount ; if ( UseI != Def && UseI -> findRegisterDefOperandIdx ( MOI -> getReg ( ) ) != - <NUM_LIT> ) break ; } if ( LastUseCount ) return LastUseCount <= TII -> getMaxAlusPerClause ( ) ; llvm_unreachable ( <STR_LIT> ) ;


<s> for ( MachineInstr :: const_mop_iterator MOI = Def -> operands_begin ( ) , MOE = Def -> operands_end ( ) ; MOI != MOE ; ++ MOI ) { if ( ! MOI -> isReg ( ) || ! MOI -> isDef ( ) || TRI . isPhysRegLiveAcrossClauses ( MOI -> getReg ( ) ) ) continue ; unsigned LastUseCount = <NUM_LIT> ; for ( MachineBasicBlock :: iterator UseI = Def ; UseI != BBEnd ; ++ UseI ) { AluInstCount += OccupiedDwords ( * UseI ) ;

if ( ! SubstituteKCacheBank ( * UseI , KCacheBanks , false ) ) return false ;


<s> if ( TII -> isALUInstr ( MI . getOpcode ( ) ) ) return true ; if ( TII -> isVector ( MI ) || TII -> isCubeOp ( MI . getOpcode ( ) ) ) return true ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; default :


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> } if ( TII -> mustBeLastInClause ( I -> getOpcode ( ) ) ) { I ++ ; break ; } if ( ! canClauseLocalKillFitInClause ( AluInstCount , KCacheBanks , I , E ) ) break ; if ( ! SubstituteKCacheBank ( * I , KCacheBanks ) ) break ; AluInstCount += OccupiedDwords ( * I ) ; } unsigned Opcode = PushBeforeModifier ? <STR_LIT> : <STR_LIT> ; BuildMI ( MBB , ClauseHead , MBB . findDebugLoc ( ClauseHead ) , TII -> get ( Opcode ) ) . addImm ( Address ++ ) . addImm ( KCacheBanks . empty ( ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . first ) . addImm ( ( KCacheBanks . size ( ) < <NUM_LIT> ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . first ) . addImm ( KCacheBanks . empty ( ) ? <NUM_LIT> : <NUM_LIT> ) . addImm ( ( KCacheBanks . size ( ) < <NUM_LIT> ) ? <NUM_LIT> : <NUM_LIT> ) . addImm ( KCacheBanks . empty ( ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . second ) . addImm ( ( KCacheBanks . size ( ) < <NUM_LIT> ) ? <NUM_LIT> : KCacheBanks [ <NUM_LIT> ] . second ) . addImm ( AluInstCount ) . addImm ( <NUM_LIT> ) ;


<s> unsigned NumLiteral = <NUM_LIT> ; for ( MachineInstr :: mop_iterator It = MI . operands_begin ( ) , E = MI . operands_end ( ) ; It != E ; ++ It ) { MachineOperand & MO = * It ; if ( MO . isReg ( ) && MO . getReg ( ) == <STR_LIT> ) ++ NumLiteral ; } return <NUM_LIT> + NumLiteral ;


<s> } if ( TII -> isLDSRetInstr ( MI . getOpcode ( ) ) ) return <NUM_LIT> ; if ( TII -> isVector ( MI ) || TII -> isCubeOp ( MI . getOpcode ( ) ) || TII -> isReductionOp ( MI . getOpcode ( ) ) ) return <NUM_LIT> ;


<s> BB != BB_E ; ++ BB ) { MachineBasicBlock & MBB = * BB ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; if ( I != MBB . end ( ) && I -> getOpcode ( ) == <STR_LIT> ) continue ; for ( MachineBasicBlock :: iterator E = MBB . end ( ) ; I != E ; ) { if ( isALU ( * I ) ) I = MakeALUClause ( MBB , I ) ;


<s> } if ( CachedConsts [ <NUM_LIT> ] == BankLine ) { UsedKCache . push_back ( std :: pair < unsigned , unsigned > ( <NUM_LIT> , KCacheIndex ) ) ; continue ; } return false ; } if ( ! UpdateInstr ) return true ; for ( unsigned i = <NUM_LIT> , j = <NUM_LIT> , n = Consts . size ( ) ; i < n ; ++ i ) { if ( Consts [ i ] . first -> getReg ( ) != <STR_LIT> ) continue ; switch ( UsedKCache [ j ] . first ) { case <NUM_LIT> : Consts [ i ] . first -> setReg ( <STR_LIT> . getRegister ( UsedKCache [ j ] . second ) ) ; break ; case <NUM_LIT> : Consts [ i ] . first -> setReg ( <STR_LIT> . getRegister ( UsedKCache [ j ] . second ) ) ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } j ++ ; }


<s> CachedConsts . push_back ( BankLine ) ; UsedKCache . push_back ( std :: pair < unsigned , unsigned > ( <NUM_LIT> , KCacheIndex ) ) ; continue ; } if ( CachedConsts [ <NUM_LIT> ] == BankLine ) { UsedKCache . push_back ( std :: pair < unsigned , unsigned > ( <NUM_LIT> , KCacheIndex ) ) ; continue ; } if ( CachedConsts . size ( ) == <NUM_LIT> ) { CachedConsts . push_back ( BankLine ) ; UsedKCache . push_back ( std :: pair < unsigned , unsigned > ( <NUM_LIT> , KCacheIndex ) ) ; continue ; } if ( CachedConsts [ <NUM_LIT> ] == BankLine ) { UsedKCache . push_back ( std :: pair < unsigned , unsigned > ( <NUM_LIT> , KCacheIndex ) ) ; continue ; } return false ; } if ( ! UpdateInstr ) return true ; for ( unsigned i = <NUM_LIT> , j = <NUM_LIT> , n = Consts . size ( ) ; i < n ; ++ i ) { if ( Consts [ i ] . first -> getReg ( ) != <STR_LIT> ) continue ; switch ( UsedKCache [ j ] . first ) { case <NUM_LIT> : Consts [ i ] . first -> setReg ( <STR_LIT> . getRegister ( UsedKCache [ j ] . second ) ) ; break ; case <NUM_LIT> : Consts [ i ] . first -> setReg ( <STR_LIT> . getRegister ( UsedKCache [ j ] . second ) ) ; break ; default : llvm_unreachable ( <STR_LIT> ) ;


<s> if ( I != MBB . end ( ) && I -> getOpcode ( ) == <STR_LIT> ) continue ; for ( MachineBasicBlock :: iterator E = MBB . end ( ) ; I != E ; ) { if ( isALU ( * I ) ) { auto next = MakeALUClause ( MBB , I ) ; assert ( next != I ) ; I = next ;


<s> TII = ST . getInstrInfo ( ) ; for ( MachineFunction :: iterator BB = MF . begin ( ) , BB_E = MF . end ( ) ; BB != BB_E ; ++ BB ) { MachineBasicBlock & MBB = * BB ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; if ( I != MBB . end ( ) && I -> getOpcode ( ) == <STR_LIT> ) continue ; for ( MachineBasicBlock :: iterator E = MBB . end ( ) ; I != E ; ) { if ( isALU ( * I ) ) { auto next = MakeALUClause ( MBB , I ) ; assert ( next != I ) ; I = next ;


<s> MachineBasicBlock & MBB = * BB ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; if ( I != MBB . end ( ) && I -> getOpcode ( ) == <STR_LIT> ) continue ; for ( MachineBasicBlock :: iterator E = MBB . end ( ) ; I != E ; ) { if ( isALU ( * I ) ) {


<s> const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; for ( MachineInstr :: const_mop_iterator MOI = Def -> operands_begin ( ) , MOE = Def -> operands_end ( ) ; MOI != MOE ; ++ MOI ) {

if ( ! MOI -> isReg ( ) || ! MOI -> isDef ( ) || TRI . isPhysRegLiveAcrossClauses ( MOI -> getReg ( ) ) ) continue ;


<s> for ( MachineInstr :: const_mop_iterator MOI = Def -> operands_begin ( ) , MOE = Def -> operands_end ( ) ; MOI != MOE ; ++ MOI ) { if ( ! MOI -> isReg ( ) || ! MOI -> isDef ( ) || TRI . isPhysRegLiveAcrossClauses ( MOI -> getReg ( ) ) ) continue ; unsigned LastUseCount = <NUM_LIT> ; for ( MachineBasicBlock :: iterator UseI = Def ; UseI != BBEnd ; ++ UseI ) { AluInstCount += OccupiedDwords ( * UseI ) ; if ( ! SubstituteKCacheBank ( * UseI , KCacheBanks , false ) ) return false ; if ( AluInstCount >= TII -> getMaxAlusPerClause ( ) ) return false ;

if ( UseI -> findRegisterUseOperandIdx ( MOI -> getReg ( ) ) ) LastUseCount = AluInstCount ;


<s> CachedConsts . push_back ( BankLine ) ; UsedKCache . push_back ( std :: pair < unsigned , unsigned > ( <NUM_LIT> , KCacheIndex ) ) ; continue ; } if ( CachedConsts [ <NUM_LIT> ] == BankLine ) { UsedKCache . push_back ( std :: pair < unsigned , unsigned > ( <NUM_LIT> , KCacheIndex ) ) ; continue ; } if ( CachedConsts . size ( ) == <NUM_LIT> ) { CachedConsts . push_back ( BankLine ) ; UsedKCache . push_back ( std :: pair < unsigned , unsigned > ( <NUM_LIT> , KCacheIndex ) ) ; continue ; } if ( CachedConsts [ <NUM_LIT> ] == BankLine ) { UsedKCache . push_back ( std :: pair < unsigned , unsigned > ( <NUM_LIT> , KCacheIndex ) ) ; continue ; } return false ; } if ( ! UpdateInstr ) return true ; for ( unsigned i = <NUM_LIT> , j = <NUM_LIT> , n = Consts . size ( ) ; i < n ; ++ i ) { if ( Consts [ i ] . first -> getReg ( ) != <STR_LIT> ) continue ; switch ( UsedKCache [ j ] . first ) { case <NUM_LIT> : Consts [ i ] . first -> setReg ( <STR_LIT> . getRegister ( UsedKCache [ j ] . second ) ) ; break ; case <NUM_LIT> : Consts [ i ] . first -> setReg ( <STR_LIT> . getRegister ( UsedKCache [ j ] . second ) ) ;


<s> if ( ! SubstituteKCacheBank ( * UseI , KCacheBanks , false ) ) return false ; if ( AluInstCount >= TII -> getMaxAlusPerClause ( ) ) return false ; if ( UseI -> readsRegister ( MOI -> getReg ( ) ) ) LastUseCount = AluInstCount ; if ( UseI != Def && UseI -> killsRegister ( MOI -> getReg ( ) ) ) break ; } if ( LastUseCount ) return LastUseCount <= TII -> getMaxAlusPerClause ( ) ; llvm_unreachable ( <STR_LIT> ) ; } return true ;


<s> for ( MachineFunction :: iterator BB = MF . begin ( ) , BB_E = MF . end ( ) ; BB != BB_E ; ++ BB ) { MachineBasicBlock & MBB = * BB ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; if ( I -> getOpcode ( ) == <STR_LIT> ) continue ; for ( MachineBasicBlock :: iterator E = MBB . end ( ) ; I != E ; ) {

if ( isALU ( * I ) ) I = MakeALUClause ( MBB , I ) ;


<s> bool runOnMachineFunction ( MachineFunction & MF ) override { const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = ST . getInstrInfo ( ) ; for ( MachineFunction :: iterator BB = MF . begin ( ) , BB_E = MF . end ( ) ; BB != BB_E ; ++ BB ) { MachineBasicBlock & MBB = * BB ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; if ( I -> getOpcode ( ) == <STR_LIT> ) continue ; for ( MachineBasicBlock :: iterator E = MBB . end ( ) ; I != E ; ) { if ( isALU ( * I ) ) I = MakeALUClause ( MBB , I ) ; else ++ I ; }


<s> return new <ISA_LIT>ExpandSpecialInstrsPass ( ) ;


<s> StringRef getPassName ( ) const override {


<s> <ISA_LIT>ExpandSpecialInstrsPass ( ) : MachineFunctionPass ( ID ) {