<s> MachineInstr * IncomingDef = MRI -> getUniqueVRegDef ( IncomingReg ) ; if ( IncomingDef -> getOpcode ( ) == <STR_LIT> ) { IncomingReg = IncomingDef -> getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( isLaneMaskReg ( IncomingReg ) || isVreg1 ( IncomingReg ) ) ; assert ( ! IncomingDef -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ) ; } else if ( IncomingDef -> getOpcode ( ) == <STR_LIT> ) { continue ; } else { assert ( IncomingDef -> isPHI ( ) || PhiRegisters . count ( IncomingReg ) ) ; } IncomingBlocks . push_back ( IncomingMBB ) ; IncomingRegs . push_back ( IncomingReg ) ; } PhiRegisters . insert ( DstReg ) ; std :: vector < MachineBasicBlock * > DomBlocks = { & MBB } ; for ( MachineInstr & Use : MRI -> use_instructions ( DstReg ) ) DomBlocks . push_back ( Use . getParent ( ) ) ; MachineBasicBlock * PostDomBound = PDT -> findNearestCommonDominator ( DomBlocks ) ; unsigned FoundLoopLevel = LF . findLoop ( PostDomBound ) ; SSAUpdater . Initialize ( DstReg ) ; if ( FoundLoopLevel ) { LF . addLoopEntries ( FoundLoopLevel , SSAUpdater , IncomingBlocks ) ; for ( unsigned i = <NUM_LIT> ; i < IncomingRegs . size ( ) ; ++ i ) { IncomingUpdated . push_back ( createLaneMaskReg ( * MF ) ) ; SSAUpdater . AddAvailableValue ( IncomingBlocks [ i ] , IncomingUpdated . back ( ) ) ; } for ( unsigned i = <NUM_LIT> ; i < IncomingRegs . size ( ) ; ++ i ) { MachineBasicBlock & IMBB = * IncomingBlocks [ i ] ; buildMergeLaneMasks ( IMBB , getSaluInsertionAtEnd ( IMBB ) , { } , IncomingUpdated [ i ] , SSAUpdater . GetValueInMiddleOfBlock ( & IMBB ) , IncomingRegs [ i ] ) ; } } else { PIA . analyze ( MBB , IncomingBlocks ) ; for ( MachineBasicBlock * MBB : PIA . predecessors ( ) ) SSAUpdater . AddAvailableValue ( MBB , insertUndefLaneMask ( * MBB ) ) ; for ( unsigned i = <NUM_LIT> ; i < IncomingRegs . size ( ) ; ++ i ) { MachineBasicBlock & IMBB = * IncomingBlocks [ i ] ; if ( PIA . isSource ( IMBB ) ) { IncomingUpdated . push_back ( <NUM_LIT> ) ; SSAUpdater . AddAvailableValue ( & IMBB , IncomingRegs [ i ] ) ; } else { IncomingUpdated . push_back ( createLaneMaskReg ( * MF ) ) ; SSAUpdater . AddAvailableValue ( & IMBB , IncomingUpdated . back ( ) ) ; } } for ( unsigned i = <NUM_LIT> ; i < IncomingRegs


<s> bool <ISA_LIT>LowerI1Copies :: runOnMachineFunction ( MachineFunction & TheMF ) { if ( TheMF . getProperties ( ) . hasProperty ( MachineFunctionProperties :: Property :: Selected ) ) return false ; MF = & TheMF ; MRI = & MF -> getRegInfo ( ) ; DT = & getAnalysis < MachineDominatorTree > ( ) ; PDT = & getAnalysis < MachinePostDominatorTree > ( ) ; ST = & MF -> getSubtarget < GCNSubtarget > ( ) ; TII = ST -> getInstrInfo ( ) ; IsWave32 = ST -> isWave32 ( ) ; if ( IsWave32 ) { ExecReg = <STR_LIT> ; MovOp = <STR_LIT> ; AndOp = <STR_LIT> ; OrOp = <STR_LIT> ; XorOp = <STR_LIT> ; AndN2Op = <STR_LIT> ; OrN2Op = <STR_LIT> ; } else { ExecReg = <STR_LIT>


<s> static unsigned createLaneMaskReg ( MachineFunction & MF ) { MachineRegisterInfo & MRI = MF


<s> if ( ! isLaneMaskReg ( Reg ) ) return false ; } if ( MI -> getOpcode ( ) != <STR_LIT> ) return false ; if ( ! MI -> getOperand ( <NUM_LIT> ) . isImm ( ) ) return false ; int64_t Imm = MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( Imm == <NUM_LIT> ) { Val = false ; return true


<s> SmallVector < MachineInstr * , <NUM_LIT> > DeadCopies ; for ( MachineBasicBlock & MBB : * MF ) { LF . initialize ( MBB ) ; for ( MachineInstr & MI : MBB ) { if ( MI . getOpcode ( ) != <STR_LIT> && MI . getOpcode ( ) != <STR_LIT> ) continue ; unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( DstReg ) || MRI -> getRegClass ( DstReg ) != & <STR_LIT> ) continue ; if ( MRI -> use_empty ( DstReg ) ) { DeadCopies . push_back ( & MI ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MRI -> setRegClass ( DstReg , & <STR_LIT> ) ; if ( MI . getOpcode ( ) == <STR_LIT> ) continue ; DebugLoc DL = MI . getDebugLoc ( ) ; unsigned SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( ! MI . getOperand ( <NUM_LIT> ) . getSubReg ( ) ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( SrcReg ) || ! isLaneMaskReg ( SrcReg ) ) { assert ( TII -> getRegisterInfo ( ) . getRegSizeInBits ( SrcReg , * MRI ) == <NUM_LIT> ) ; unsigned TmpReg = createLaneMaskReg ( * MF ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpReg ) . addReg ( SrcReg ) . addImm ( <NUM_LIT> ) ; MI . getOperand ( <NUM_LIT> ) . setReg ( TmpReg ) ; SrcReg = TmpReg ; } MachineBasicBlock * PostDomBound = & MBB ; for ( MachineInstr & Use : MRI -> use_instructions ( DstReg ) ) { PostDomBound = PDT -> findNearestCommonDominator ( PostDomBound , Use . getParent ( ) ) ; } unsigned FoundLoopLevel = LF . findLoop ( PostDomBound ) ; if ( FoundLoopLevel ) { SSAUpdater . Initialize ( DstReg ) ; SSAUpdater . AddAvailableValue ( & MBB , DstReg ) ; LF . addLoopEntries ( FoundLoopLevel , SSAUpdater ) ; buildMergeLaneMasks ( MBB , MI , DL , DstReg , SSAUpdater . GetValueInMiddleOfBlock ( & MBB ) ,


<s> LF . initialize ( MBB ) ; for ( MachineInstr & MI : MBB ) { if ( MI . getOpcode ( ) != <STR_LIT> && MI . getOpcode ( ) != <STR_LIT> ) continue ; unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( DstReg ) || MRI -> getRegClass ( DstReg ) != & <STR_LIT> ) continue ; if ( MRI -> use_empty ( DstReg ) ) { DeadCopies . push_back ( & MI ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MRI -> setRegClass ( DstReg , & <STR_LIT> ) ; if ( MI . getOpcode ( ) == <STR_LIT> ) continue ; DebugLoc DL = MI . getDebugLoc ( ) ; unsigned SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( ! MI . getOperand ( <NUM_LIT> ) . getSubReg ( ) ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( SrcReg ) || ! isLaneMaskReg ( SrcReg ) ) { assert ( TII -> getRegisterInfo ( ) . getRegSizeInBits ( SrcReg , * MRI ) == <NUM_LIT> ) ; unsigned TmpReg = createLaneMaskReg ( * MF ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpReg ) . addReg ( SrcReg ) . addImm ( <NUM_LIT> ) ; MI . getOperand ( <NUM_LIT> ) . setReg ( TmpReg ) ; SrcReg = TmpReg ; } MachineBasicBlock * PostDomBound = & MBB ; for ( MachineInstr & Use : MRI -> use_instructions ( DstReg ) ) { PostDomBound = PDT -> findNearestCommonDominator ( PostDomBound , Use . getParent ( ) ) ; } unsigned FoundLoopLevel = LF . findLoop


<s> } PhiRegisters . insert ( DstReg ) ; MachineBasicBlock * PostDomBound = & MBB ; for ( MachineInstr & Use : MRI -> use_instructions ( DstReg ) ) { PostDomBound = PDT -> findNearestCommonDominator ( PostDomBound , Use . getParent ( ) ) ; } unsigned FoundLoopLevel = LF . findLoop ( PostDomBound ) ; SSAUpdater . Initialize ( DstReg ) ; if ( FoundLoopLevel ) { LF . addLoopEntries ( FoundLoopLevel , SSAUpdater , IncomingBlocks ) ; for ( unsigned i = <NUM_LIT> ; i < IncomingRegs . size ( ) ; ++ i ) { IncomingUpdated . push_back ( createLaneMaskReg ( * MF ) ) ; SSAUpdater . AddAvailableValue ( IncomingBlocks [ i ] , IncomingUpdated . back ( ) ) ; } for ( unsigned i = <NUM_LIT> ; i < IncomingRegs . size ( ) ; ++ i ) { MachineBasicBlock & IMBB = * IncomingBlocks [ i ] ; buildMergeLaneMasks ( IMBB , getSaluInsertionAtEnd ( IMBB ) , { } , IncomingUpdated [ i ] , SSAUpdater . GetValueInMiddleOfBlock ( & IMBB ) , IncomingRegs [ i ] ) ; } } else { PIA . analyze ( MBB , IncomingBlocks ) ; for ( MachineBasicBlock * MBB : PIA . predecessors ( ) ) SSAUpdater . AddAvailableValue ( MBB , insertUndefLaneMask ( * MBB ) ) ; for ( unsigned i = <NUM_LIT> ; i < IncomingRegs . size ( ) ; ++ i ) { MachineBasicBlock & IMBB = * IncomingBlocks [ i ] ; if ( PIA . isSource ( IMBB ) ) { IncomingUpdated . push_back ( <NUM_LIT> ) ; SSAUpdater . AddAvailableValue ( & IMBB , IncomingRegs [ i ] ) ; } else { IncomingUpdated . push_back ( createLaneMaskReg ( * MF ) ) ; SSAUpdater . AddAvailableValue ( & IMBB , IncomingUpdated . back ( ) ) ; } } for ( unsigned i = <NUM_LIT> ; i < IncomingRegs . size ( ) ; ++ i ) { if ( ! IncomingUpdated [ i ] ) continue ; MachineBasicBlock & IMBB = * IncomingBlocks [ i ] ; buildMergeLaneMasks ( IMBB , getSaluInsertionAtEnd ( IMBB ) , { } , IncomingUpdated [ i ] , SSAUpdater . GetValueInMiddleOfBlock ( & IMBB ) , IncomingRegs [ i ] ) ; } } unsigned NewReg = SSAUpdater . GetValueInMiddleOfBlock ( & MBB ) ; if ( NewReg != DstReg ) { MRI -> replaceRegWith ( NewReg , DstReg ) ; MI . getOperand ( <NUM_LIT> ) . setReg ( NewReg ) ; DeadPhis . push_back ( & MI ) ; } IncomingBlocks . clear ( ) ; IncomingRegs . clear ( ) ; IncomingUpdated . clear ( ) ; } for ( MachineInstr * MI : DeadPhis


<s> MF = & TheMF ; MRI = & MF -> getRegInfo ( ) ; DT = & getAnalysis < MachineDominatorTree > ( ) ; PDT = & getAnalysis < MachinePostDominatorTree > ( ) ; ST = & MF -> getSubtarget < GCNSubtarget > ( ) ; TII = ST -> getInstrInfo ( ) ; lowerCopiesFromI1 ( ) ; lowerPhis ( ) ; lowerCopiesToI1 ( ) ; for ( unsigned Reg : ConstrainRegs ) MRI

-> constrainRegClass ( Reg , & <STR_LIT> ) ;


<s> SmallVector < MachineInstr * , <NUM_LIT> > DeadCopies ; for ( MachineBasicBlock & MBB : * MF ) { for ( MachineInstr & MI : MBB ) { if ( MI . getOpcode ( ) != <STR_LIT> ) continue ; unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( SrcReg ) || MRI -> getRegClass ( SrcReg ) != & <STR_LIT> ) continue ; if ( isLaneMaskReg ( DstReg ) || ( TargetRegisterInfo :: isVirtualRegister ( DstReg ) && MRI -> getRegClass ( DstReg ) == & <STR_LIT> ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; DebugLoc DL = MI . getDebugLoc ( ) ; assert ( TII -> getRegisterInfo ( ) . getRegSizeInBits ( DstReg , * MRI ) == <NUM_LIT> ) ; assert ( ! MI . getOperand ( <NUM_LIT> ) . getSubReg ( ) ) ; ConstrainRegs . insert ( SrcReg ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addImm ( <NUM_LIT> ) . addImm ( - <NUM_LIT> ) . addReg ( SrcReg ) ; DeadCopies . push_back ( & MI ) ; } for ( MachineInstr * MI : DeadCopies ) MI -> eraseFromParent ( ) ; DeadCopies . clear (


<s> SmallVector < unsigned , <NUM_LIT> > IncomingUpdated ; for ( MachineBasicBlock & MBB : * MF ) { LF . initialize ( MBB ) ; for ( MachineInstr & MI : MBB . phis ( ) ) { unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( MRI -> getRegClass ( DstReg ) != & <STR_LIT> ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MRI -> setRegClass ( DstReg , & <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> ; i < MI . getNumOperands ( ) ; i += <NUM_LIT> ) { assert ( i + <NUM_LIT> < MI . getNumOperands ( ) ) ; unsigned IncomingReg = MI . getOperand ( i ) . getReg ( ) ; MachineBasicBlock * IncomingMBB = MI . getOperand ( i + <NUM_LIT> ) . getMBB ( ) ; MachineInstr * IncomingDef = MRI -> getUniqueVRegDef ( IncomingReg ) ; if ( IncomingDef -> getOpcode ( ) == <STR_LIT> ) { IncomingReg = IncomingDef -> getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( isLaneMaskReg ( IncomingReg ) ) ; assert ( ! IncomingDef -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ) ; } else if ( IncomingDef -> getOpcode ( ) == <STR_LIT> ) { continue ; } else { assert ( IncomingDef -> isPHI ( ) ) ; } IncomingBlocks . push_back ( IncomingMBB ) ; IncomingRegs . push_back ( IncomingReg ) ; } MachineBasicBlock * PostDomBound = & MBB ; for ( MachineInstr & Use : MRI -> use_instructions ( DstReg ) ) { PostDomBound = PDT -> findNearestCommonDominator ( PostDomBound , Use . getParent ( ) ) ; } unsigned FoundLoopLevel = LF . findLoop ( PostDomBound ) ; SSAUpdater . Initialize ( DstReg ) ; if ( FoundLoopLevel ) { LF . addLoopEntries ( FoundLoopLevel , SSAUpdater , IncomingBlocks ) ; for ( unsigned i = <NUM_LIT> ; i < IncomingRegs . size ( ) ; ++ i ) { IncomingUpdated . push_back ( createLaneMaskReg ( * MF ) ) ; SSAUpdater . AddAvailableValue ( IncomingBlocks [ i ] , IncomingUpdated . back ( ) ) ; } for ( unsigned i = <NUM_LIT> ; i < IncomingRegs . size ( ) ; ++ i ) { MachineBasicBlock & IMBB = * IncomingBlocks [ i ] ; buildMergeLaneMasks ( IMBB , getSaluInsertionAtEnd ( IMBB ) , { } , IncomingUpdated [ i ] , SSAUpdater . GetValueInMiddleOfBlock ( & IMBB ) , IncomingRegs [ i ] ) ; } } else { PIA . analyze ( MBB , IncomingBlocks ) ; for ( MachineBasicBlock * MBB : PIA . predecessors ( ) ) SSAUpdater . AddAvailableValue (


<s> bool isVreg1 ( unsigned Reg ) const { return TargetRegisterInfo :: isVirtualRegister ( Reg ) && MRI -> getRegClass ( Reg )


<s> bool isVreg1 ( unsigned Reg ) const { return TargetRegisterInfo :: isVirtualRegister ( Reg ) && MRI ->


<s> unsigned SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( ! MI . getOperand ( <NUM_LIT> ) . getSubReg ( ) ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( SrcReg ) || ( ! isLaneMaskReg ( SrcReg ) && ! isVreg1 ( SrcReg ) ) ) { assert ( TII -> getRegisterInfo ( ) . getRegSizeInBits ( SrcReg , * MRI ) == <NUM_LIT> ) ; unsigned TmpReg = createLaneMaskReg ( * MF ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpReg ) . addReg ( SrcReg ) . addImm ( <NUM_LIT> ) ; MI . getOperand ( <NUM_LIT> ) . setReg ( TmpReg ) ; SrcReg = TmpReg ; } MachineBasicBlock * PostDomBound = & MBB ; for ( MachineInstr & Use : MRI -> use_instructions ( DstReg ) ) { PostDomBound = PDT -> findNearestCommonDominator ( PostDomBound , Use . getParent ( ) ) ; } unsigned FoundLoopLevel = LF . findLoop


<s> if ( isLaneMaskReg ( DstReg ) || isVreg1 ( DstReg ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; DebugLoc DL = MI . getDebugLoc ( ) ; assert ( TII -> getRegisterInfo ( ) . getRegSizeInBits ( DstReg , * MRI ) == <NUM_LIT> ) ; assert ( ! MI . getOperand ( <NUM_LIT> ) . getSubReg ( ) ) ; ConstrainRegs . insert ( SrcReg ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( - <NUM_LIT> ) . addReg ( SrcReg ) ; DeadCopies . push_back ( &


<s> MRI -> setRegClass ( DstReg , IsWave32 ? & <STR_LIT> : & <STR_LIT> ) ; if ( MI . getOpcode ( ) == <STR_LIT> ) continue ; DebugLoc DL = MI . getDebugLoc ( ) ; Register SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( ! MI . getOperand ( <NUM_LIT> ) . getSubReg ( ) ) ; if ( ! Register :: isVirtualRegister ( SrcReg ) || ( ! isLaneMaskReg ( SrcReg ) && ! isVreg1 ( SrcReg ) ) ) { assert ( TII -> getRegisterInfo ( ) . getRegSizeInBits ( SrcReg , * MRI ) == <NUM_LIT> ) ; unsigned TmpReg = createLaneMaskReg ( * MF ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpReg ) . addReg ( SrcReg ) . addImm ( <NUM_LIT> ) ; MI . getOperand ( <NUM_LIT> ) . setReg ( TmpReg ) ; SrcReg = TmpReg ; } MachineBasicBlock * PostDomBound = & MBB ; for ( MachineInstr & Use : MRI -> use_instructions ( DstReg ) ) { PostDomBound = PDT -> findNearestCommonDominator ( PostDomBound , Use . getParent ( ) ) ; } unsigned FoundLoopLevel = LF . findLoop ( PostDomBound ) ; if ( FoundLoopLevel ) { SSAUpdater . Initialize ( DstReg ) ; SSAUpdater . AddAvailableValue ( & MBB , DstReg ) ; LF . addLoopEntries ( FoundLoopLevel , SSAUpdater ) ; buildMergeLaneMasks ( MBB , MI , DL , DstReg ,

SSAUpdater . GetValueInMiddleOfBlock ( & MBB ) , SrcReg ) ;


<s> if ( ! isVreg1 ( DstReg ) ) continue ; if ( MRI -> use_empty ( DstReg ) ) { DeadCopies . push_back ( & MI ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MRI -> setRegClass ( DstReg , IsWave32 ? & <STR_LIT> : & <STR_LIT> ) ; if ( MI . getOpcode ( ) == <STR_LIT> ) continue ; DebugLoc DL = MI . getDebugLoc ( ) ; Register SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( ! MI . getOperand ( <NUM_LIT> ) . getSubReg ( ) ) ; if ( ! Register :: isVirtualRegister ( SrcReg ) || ( ! isLaneMaskReg ( SrcReg ) && ! isVreg1 ( SrcReg ) ) ) { assert ( TII -> getRegisterInfo ( ) . getRegSizeInBits ( SrcReg , * MRI ) == <NUM_LIT> ) ; unsigned TmpReg = createLaneMaskReg ( * MF ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpReg ) . addReg ( SrcReg ) . addImm ( <NUM_LIT> ) ; MI . getOperand ( <NUM_LIT> ) . setReg ( TmpReg ) ; SrcReg = TmpReg ; } MachineBasicBlock * PostDomBound = & MBB ; for ( MachineInstr & Use : MRI -> use_instructions ( DstReg ) ) { PostDomBound = PDT -> findNearestCommonDominator ( PostDomBound , Use . getParent ( ) ) ; } unsigned FoundLoopLevel = LF . findLoop ( PostDomBound ) ; if ( FoundLoopLevel ) { SSAUpdater . Initialize ( DstReg


<s> if ( MI . getOpcode ( ) == <STR_LIT> ) { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * RC = MRI . getRegClass ( Reg ) ; if ( RC == & <STR_LIT> ) MRI . setRegClass ( Reg , & <STR_LIT> ) ; continue ; } if ( MI . getOpcode ( ) != <STR_LIT> ) continue ; const MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; const MachineOperand & Src = MI . getOperand ( <NUM_LIT> ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( Src . getReg ( ) ) || ! TargetRegisterInfo :: isVirtualRegister ( Dst . getReg ( ) ) ) continue ; const TargetRegisterClass * DstRC = MRI . getRegClass ( Dst . getReg ( ) ) ; const TargetRegisterClass * SrcRC = MRI . getRegClass ( Src . getReg ( ) ) ; if ( DstRC == & <STR_LIT> && TRI -> getCommonSubClass ( SrcRC , & <STR_LIT> ) ) { I1Defs . push_back ( Dst . getReg ( ) ) ; DebugLoc DL = MI . getDebugLoc ( ) ; MachineInstr * DefInst = MRI . getUniqueVRegDef ( Src


<s> const TargetRegisterInfo * TRI = & TII -> getRegisterInfo ( ) ; std :: vector < unsigned > I1Defs ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ; MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; if ( MI . getOpcode ( ) == <STR_LIT> ) { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * RC = MRI . getRegClass ( Reg ) ; if ( RC == & <STR_LIT> ) MRI . setRegClass ( Reg , & <STR_LIT> ) ; continue ; } if ( MI . getOpcode ( ) != <STR_LIT> ) continue ; const MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; const MachineOperand & Src = MI . getOperand ( <NUM_LIT> ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( Src . getReg ( ) ) || ! TargetRegisterInfo :: isVirtualRegister ( Dst . getReg ( ) ) ) continue ; const TargetRegisterClass * DstRC = MRI . getRegClass ( Dst . getReg ( ) ) ; const TargetRegisterClass * SrcRC = MRI . getRegClass ( Src . getReg (


<s> if ( DefInst -> getOpcode ( ) == <STR_LIT> ) { if ( DefInst -> getOperand ( <NUM_LIT> ) . isImm ( ) ) { I1Defs . push_back ( Dst . getReg ( ) ) ; int64_t Val = DefInst -> getOperand ( <NUM_LIT> ) . getImm ( ) ; assert ( Val == <NUM_LIT> || Val == - <NUM_LIT> ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . add ( Dst ) . addImm ( Val ) ; MI . eraseFromParent ( ) ; continue ; } } unsigned int TmpSrc = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , TmpSrc ) . add ( Src ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . add ( Dst ) . addImm ( <NUM_LIT> ) . addImm ( - <NUM_LIT> ) . addReg ( TmpSrc ) ; MI . eraseFromParent ( ) ; } else if ( TRI -> getCommonSubClass ( DstRC , & <STR_LIT> ) && SrcRC == & <STR_LIT> ) { if ( DefInst -> getOpcode ( ) == <STR_LIT> && DefInst -> getOperand ( <NUM_LIT> ) . isImm ( ) && DefInst -> getOperand ( <NUM_LIT> ) . isImm ( ) && DefInst -> getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> && DefInst -> getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> && DefInst -> getOperand ( <NUM_LIT> ) . isReg ( ) && TargetRegisterInfo :: isVirtualRegister ( DefInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ) && TRI -> getCommonSubClass ( MRI . getRegClass ( DefInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ) , & <STR_LIT> ) && <STR_LIT> ( DefInst -> getParent ( ) , & MBB ) ) { BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . add ( Dst ) . addReg ( <STR_LIT> ) . add ( DefInst -> getOperand ( <NUM_LIT> ) ) ; }


<s> initialize<ISA_LIT>LowerI1CopiesPass ( * PassRegistry :: getPassRegistry


<s> <ISA_LIT>LowerI1Copies ( ) : MachineFunctionPass ( ID ) { initialize<ISA_LIT>LowerI1CopiesPass ( * PassRegistry :: getPassRegistry


<s> MCRegister Reg = CS . getReg ( ) ; MachineInstrSpan MIS ( I , & SaveBlock ) ; const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( Reg , <STR_LIT> ) ; const bool IsLiveIn = MRI . isLiveIn ( Reg ) ; TII . storeRegToStackSlot ( SaveBlock , I , Reg , ! IsLiveIn , CS . getFrameIdx ( ) , RC , TRI ) ; if ( LIS ) { assert ( std :: distance ( MIS . begin ( ) , I ) ==


<s> MachineInstr & MI = * I ; Next = std :: next ( I ) ; if ( ! TII -> isSGPRSpill ( MI ) ) continue ; int FI = TII -> getNamedOperand ( MI , <STR_LIT> :: addr ) -> getIndex ( ) ; assert ( MFI . getStackID ( FI ) == TargetStackID :: SGPRSpill ) ; if ( FuncInfo -> allocateSGPRSpillToVGPR ( MF , FI ) ) { NewReservedRegs = true ; bool Spilled = TRI -> eliminateSGPRToVGPRSpillFrameIndex ( MI , FI , nullptr , LIS ) ; ( void ) Spilled ; assert ( Spilled && <STR_LIT> ) ; SpillFIs . set ( FI ) ; } } } for ( MachineBasicBlock & MBB : MF ) { for ( auto SSpill : FuncInfo -> getSGPRSpillVGPRs ( ) ) MBB . addLiveIn ( SSpill . VGPR ) ; MBB . sortUniqueLiveIns ( ) ; for ( MachineInstr & MI : MBB ) { if ( MI . isDebugValue ( ) && MI . getOperand ( <NUM_LIT> ) . isFI ( ) && SpillFIs [ MI . getOperand ( <NUM_LIT> ) . getIndex ( ) ] ) { MI . getOperand ( <NUM_LIT> ) . ChangeToRegister ( Register ( ) , false ) ; MI . getOperand ( <NUM_LIT> ) . setIsDebug ( ) ; } } } MadeChange = true ; } else if ( FuncInfo -> VGPRReservedForSGPRSpill ) { FuncInfo -> removeVGPRForSGPRSpill ( FuncInfo -> VGPRReservedForSGPRSpill , MF ) ; } SaveBlocks . clear ( ) ; RestoreBlocks . clear ( ) ; if ( NewReservedRegs ) MRI . freezeReservedRegs


<s> static bool lowerShiftReservedVGPR ( MachineFunction & MF , const GCNSubtarget & ST ) { <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const Register PreReservedVGPR = FuncInfo -> VGPRReservedForSGPRSpill ; if ( ! PreReservedVGPR ) return false ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; Register LowestAvailableVGPR = TRI -> findUnusedRegister ( MF . getRegInfo ( ) , & <STR_LIT> , MF ) ; if ( ! LowestAvailableVGPR

) LowestAvailableVGPR = PreReservedVGPR ;


<s> TRI = & TII -> getRegisterInfo ( ) ; VRM = getAnalysisIfAvailable < VirtRegMap > ( ) ; assert ( SaveBlocks . empty ( ) && RestoreBlocks . empty ( ) ) ; calculateSaveRestoreBlocks ( MF ) ; bool HasCSRs = spillCalleeSavedRegs ( MF ) ; MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; if ( ! MFI . hasStackObjects ( ) && ! HasCSRs ) { SaveBlocks . clear ( ) ; RestoreBlocks . clear ( ) ; return false ; } MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const bool SpillVGPRToAGPR = ST . hasMAIInsts ( ) && FuncInfo -> hasSpilledVGPRs ( ) && EnableSpillVGPRToAGPR ; bool MadeChange = false ; const bool SpillToAGPR = EnableSpillVGPRToAGPR && ST . hasMAIInsts ( ) ; std :: unique_ptr < RegScavenger > RS ; if ( ( TRI -> spillSGPRToVGPR ( ) && ( HasCSRs || FuncInfo -> hasSpilledSGPRs ( ) ) ) || SpillVGPRToAGPR ) { lowerShiftReservedVGPR ( MF , ST ) ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator Next ; for ( auto I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; I = Next ) { MachineInstr & MI = * I ; Next = std :: next ( I ) ; if ( SpillToAGPR && TII -> isVGPRSpill ( MI ) ) { unsigned FIOp = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vaddr ) ; int FI = MI . getOperand ( FIOp ) . getIndex ( ) ; Register VReg = TII -> getNamedOperand ( MI , <STR_LIT> :: vdata ) -> getReg ( ) ; if ( FuncInfo -> allocateVGPRSpillToAGPR ( MF , FI , TRI -> isAGPR ( MRI , VReg ) ) ) { if ( ! RS ) RS . reset ( new RegScavenger ( ) ) ; RS -> enterBasicBlock ( MBB ) ; TRI -> eliminateFrameIndex ( MI , <NUM_LIT> , FIOp , RS . get ( ) ) ; continue ; } } if ( ! TII -> isSGPRSpill ( MI ) ) continue ; int FI = TII -> getNamedOperand ( MI , <STR_LIT> :: addr ) -> getIndex ( ) ; assert ( MFI . getStackID ( FI ) == TargetStackID :: SGPRSpill ) ; if ( FuncInfo -> allocateSGPRSpillToVGPR ( MF , FI ) ) { bool Spilled = TRI -> eliminateSGPRToVGPRSpillFrameIndex ( MI , FI , nullptr ) ; ( void ) Spilled ; assert ( Spilled && <STR_LIT> ) ; } } } for ( MachineBasicBlock &


<s> const MCPhysReg * CSRegs = MRI . getCalleeSavedRegs ( ) ; for ( unsigned I = <NUM_LIT> ; CSRegs [ I ] ; ++ I ) { unsigned Reg = CSRegs [ I ] ; if ( SavedRegs . test ( Reg ) ) { const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( Reg , <STR_LIT> ) ; int JunkFI = MFI . CreateStackObject ( TRI -> getSpillSize ( * RC ) , TRI -> getSpillAlign ( * RC ) , true ) ; C<ISA_LIT> . push_back ( CalleeSavedInfo ( Reg , JunkFI ) ) ; } } if ( ! C<ISA_LIT> . empty ( ) ) { for ( MachineBasicBlock * SaveBlock : SaveBlocks ) insertCSRSaves ( * SaveBlock ,


<s> TRI = & TII -> getRegisterInfo ( ) ; LIS = getAnalysisIfAvailable < LiveIntervals > ( ) ; assert ( SaveBlocks . empty ( ) && RestoreBlocks . empty ( ) ) ; calculateSaveRestoreBlocks ( MF ) ; bool HasCSRs = spillCalleeSavedRegs ( MF ) ; MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! MFI . hasStackObjects ( ) && ! HasCSRs ) { SaveBlocks . clear ( ) ; RestoreBlocks . clear ( ) ; if ( FuncInfo -> VGPRReservedForSGPRSpill ) { FuncInfo -> removeVGPRForSGPRSpill ( FuncInfo -> VGPRReservedForSGPRSpill , MF ) ; MRI . freezeReservedRegs ( MF ) ; } return false ; } bool MadeChange = false ; bool NewReservedRegs = false ; const bool HasSGPRSpillToVGPR = TRI -> spillSGPRToVGPR ( ) && ( HasCSRs || FuncInfo -> hasSpilledSGPRs ( ) ) ; if ( HasSGPRSpillToVGPR


<s> return makeArrayRef ( <STR_LIT> . begin ( ) , ST . getMaxNumVGPRs (


<s> LowestAvailableVGPR = Reg ; break ; } } if ( ! LowestAvailableVGPR ) return false ; ReservedVGPR = FuncInfo -> VGPRReservedForSGPRSpill ; const MCPhysReg * CSRegs = MF . getRegInfo ( ) . getCalleeSavedRegs ( ) ; int i = <NUM_LIT> ; for ( MachineBasicBlock & MBB : MF ) { for ( auto Reg : FuncInfo -> getSGPRSpillVGPRs ( ) ) { if ( Reg . VGPR == ReservedVGPR ) { MBB . removeLiveIn ( ReservedVGPR ) ; MBB . addLiveIn ( LowestAvailableVGPR ) ; Optional < int >


<s> Register LowestAvailableVGPR , ReservedVGPR ; ArrayRef < MCPhysReg > AllVGPR32s = getAllVGPR32 ( ST , MF ) ; for ( MCPhysReg Reg : AllVGPR32s ) { if ( MRI . isAllocatable ( Reg ) && ! MRI . isPhysRegUsed ( Reg ) ) { LowestAvailableVGPR = Reg ; break ; } } if ( ! LowestAvailableVGPR ) return false ; ReservedVGPR = FuncInfo -> VGPRReservedForSGPRSpill ; const MCPhysReg * CSRegs = MF . getRegInfo ( ) . getCalleeSavedRegs ( ) ; int i = <NUM_LIT> ; for ( MachineBasicBlock & MBB : MF ) { for ( auto Reg : FuncInfo -> getSGPRSpillVGPRs ( ) ) { if ( Reg . VGPR == ReservedVGPR ) { MBB . removeLiveIn ( ReservedVGPR ) ; MBB . addLiveIn ( LowestAvailableVGPR ) ; Optional < int >


<s> MachineFunction & MF = * RestoreBlock . getParent ( ) ; const TargetInstrInfo & TII = * MF . getSubtarget ( ) . getInstrInfo ( ) ; const TargetFrameLowering * TFI = MF . getSubtarget ( ) . getFrameLowering ( ) ; const TargetRegisterInfo * TRI = MF . getSubtarget ( ) . getRegisterInfo ( ) ; MachineBasicBlock :: iterator I = RestoreBlock . getFirstTerminator ( ) ; if ( ! TFI -> restoreCalleeSavedRegisters ( RestoreBlock , I , C<ISA_LIT> , TRI ) ) { for ( const CalleeSavedInfo & CI : reverse ( C<ISA_LIT> ) ) { unsigned Reg = CI . getReg ( ) ; const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( Reg ) ; TII . loadRegFromStackSlot ( RestoreBlock , I , Reg , CI . getFrameIdx (


<s> if ( ! TFI -> spillCalleeSavedRegisters ( SaveBlock , I , C<ISA_LIT> , TRI ) ) { for ( const CalleeSavedInfo & CS : C<ISA_LIT> ) { unsigned Reg = CS . getReg ( ) ; MachineInstrSpan MIS ( I , & SaveBlock ) ; const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( Reg ) ; TII . storeRegToStackSlot ( SaveBlock , I , Reg , true , CS . getFrameIdx ( ) , RC , TRI ) ; if ( LIS ) { assert ( std :: distance ( MIS . begin ( ) , I ) == <NUM_LIT> ) ; MachineInstr & Inst = * std :: prev ( I ) ; LIS -> InsertMachineInstrInMaps


<s> bool HasCSRs = spillCalleeSavedRegs ( MF ) ; MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! MFI . hasStackObjects ( ) && ! HasCSRs ) { SaveBlocks . clear ( ) ; RestoreBlocks . clear ( ) ; if ( FuncInfo -> VGPRReservedForSGPRSpill ) { FuncInfo -> removeVGPRForSGPRSpill ( FuncInfo -> VGPRReservedForSGPRSpill , MF ) ; MRI . freezeReservedRegs ( MF ) ; } return false ; } bool MadeChange = false ; bool NewReservedRegs = false ; const bool HasSGPRSpillToVGPR = TRI -> spillSGPRToVGPR ( ) && ( HasCSRs || FuncInfo -> hasSpilledSGPRs ( ) ) ; if ( HasSGPRSpillToVGPR ) { lowerShiftReservedVGPR ( MF , ST ) ; BitVector SpillFIs ( MFI . getObjectIndexEnd ( ) , false ) ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator Next ; for ( auto I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; I = Next ) { MachineInstr & MI = * I ; Next = std :: next ( I ) ; if ( ! TII -> isSGPRSpill ( MI ) ) continue ; int FI = TII -> getNamedOperand ( MI , <STR_LIT> :: addr ) -> getIndex ( ) ; assert ( MFI . getStackID ( FI ) == TargetStackID :: SGPRSpill ) ; if ( FuncInfo -> allocateSGPRSpillToVGPR ( MF , FI ) ) { NewReservedRegs = true ; bool Spilled = TRI -> eliminateSGPRToVGPRSpillFrameIndex ( MI , FI , nullptr , LIS ) ; ( void ) Spilled ; assert ( Spilled && <STR_LIT> ) ; SpillFIs . set ( FI ) ; } } } for ( MachineBasicBlock & MBB : MF ) { for ( auto SSpill : FuncInfo -> getSGPRSpillVGPRs ( ) ) MBB . addLiveIn ( SSpill . VGPR ) ; MBB . sortUniqueLiveIns ( ) ; for ( MachineInstr & MI : MBB ) { if ( MI . isDebugValue ( ) && MI . getOperand ( <NUM_LIT> ) . isFI ( ) && SpillFIs [ MI . getOperand ( <NUM_LIT> ) . getIndex ( ) ] ) { MI . getOperand ( <NUM_LIT> ) . ChangeToRegister ( Register ( ) , false ) ; } } } FuncInfo -> removeDeadFrameIndices ( MFI ) ; MadeChange = true


<s> const TargetFrameLowering * TFI = MF . getSubtarget ( ) . getFrameLowering ( ) ; const TargetRegisterInfo * TRI = MF . getSubtarget ( ) . getRegisterInfo ( ) ; MachineBasicBlock :: iterator I = SaveBlock . begin ( ) ; if ( ! TFI -> spillCalleeSavedRegisters ( SaveBlock , I , C<ISA_LIT> , TRI ) ) { for ( const CalleeSavedInfo & CS : C<ISA_LIT> ) { unsigned Reg = CS . getReg ( ) ; MachineInstrSpan MIS ( I , & SaveBlock ) ; const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( Reg , <STR_LIT> ) ; TII . storeRegToStackSlot ( SaveBlock , I , Reg , true , CS . getFrameIdx ( ) , RC , TRI ) ; if ( LIS )


<s> <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register LowestAvailableVGPR , ReservedVGPR ; ArrayRef < MCPhysReg > AllVGPR32s = ST . getRegisterInfo ( ) -> getAllVGPR32 ( MF ) ; for ( MCPhysReg Reg : AllVGPR32s ) { if ( MRI . isAllocatable ( Reg ) && ! MRI . isPhysRegUsed ( Reg ) ) { LowestAvailableVGPR = Reg ; break ; } } if ( ! LowestAvailableVGPR ) return false ; ReservedVGPR = FuncInfo -> VGPRReservedForSGPRSpill ; const MCPhysReg * CSRegs = MF . getRegInfo ( ) . getCalleeSavedRegs ( ) ; int i = <NUM_LIT> ; for ( MachineBasicBlock & MBB : MF ) { for ( auto Reg : FuncInfo -> getSGPRSpillVGPRs ( ) ) { if ( Reg . VGPR == ReservedVGPR ) { MBB . removeLiveIn ( ReservedVGPR


<s> MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register LowestAvailableVGPR , ReservedVGPR ; ArrayRef < MCPhysReg > AllVGPR32s = ST . getRegisterInfo ( ) -> getAllVGPR32 ( MF ) ; for ( MCPhysReg Reg : AllVGPR32s ) { if ( MRI . isAllocatable ( Reg ) && ! MRI . isPhysRegUsed ( Reg ) ) { LowestAvailableVGPR = Reg ; break ; } } if ( ! LowestAvailableVGPR ) return false ; ReservedVGPR = FuncInfo -> VGPRReservedForSGPRSpill ; const MCPhysReg * CSRegs = MF . getRegInfo ( ) . getCalleeSavedRegs ( ) ; int i = <NUM_LIT> ; for ( MachineBasicBlock & MBB : MF ) { for ( auto Reg : FuncInfo -> getSGPRSpillVGPRs ( ) ) { if ( Reg . VGPR == ReservedVGPR ) { MBB . removeLiveIn ( ReservedVGPR ) ; MBB . addLiveIn ( LowestAvailableVGPR ) ; Optional < int > FI ; if ( FuncInfo -> isCalleeSavedReg ( CSRegs , LowestAvailableVGPR ) ) FI = FrameInfo . CreateSpillStackObject ( <NUM_LIT> , Align ( <NUM_LIT> ) ) ; FuncInfo -> setSGPRSpillVGPRs ( LowestAvailableVGPR , FI , i ) ; } ++ i ; } MBB . sortUniqueLiveIns (


<s> VRM = getAnalysisIfAvailable < VirtRegMap > ( ) ; assert ( SaveBlocks . empty ( ) && RestoreBlocks . empty ( ) ) ; calculateSaveRestoreBlocks ( MF ) ; bool HasCSRs = spillCalleeSavedRegs ( MF ) ; MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; if ( ! MFI . hasStackObjects ( ) && ! HasCSRs ) { SaveBlocks . clear ( ) ; RestoreBlocks . clear ( ) ; return false ; } MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const bool SpillVGPRToAGPR = ST . hasMAIInsts ( ) && FuncInfo -> hasSpilledVGPRs ( ) && EnableSpillVGPRToAGPR ; bool MadeChange = false ; const bool SpillToAGPR = EnableSpillVGPRToAGPR && ST . hasMAIInsts ( ) ; if ( ( TRI -> spillSGPRToVGPR ( ) && ( HasCSRs || FuncInfo -> hasSpilledSGPRs ( ) ) ) || SpillVGPRToAGPR ) { lowerShiftReservedVGPR ( MF , ST ) ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator Next ; for ( auto I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; I = Next ) { MachineInstr & MI = * I ; Next = std :: next ( I ) ; if ( SpillToAGPR && TII -> isVGPRSpill ( MI ) ) { unsigned FIOp = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vaddr ) ; int FI = MI . getOperand ( FIOp ) . getIndex ( ) ; Register VReg = TII -> getNamedOperand ( MI , <STR_LIT> :: vdata ) -> getReg ( ) ; if ( FuncInfo -> allocateVGPRSpillToAGPR ( MF , FI , TRI -> isAGPR ( MRI , VReg ) ) ) { TRI -> eliminateFrameIndex ( MI , <NUM_LIT> , FIOp , nullptr ) ; continue ; } } if ( ! TII -> isSGPRSpill ( MI ) ) continue ; int FI = TII -> getNamedOperand ( MI , <STR_LIT> :: addr ) -> getIndex ( ) ; assert ( MFI . getStackID ( FI ) == TargetStackID :: SGPRSpill ) ; if ( FuncInfo -> allocateSGPRSpillToVGPR ( MF , FI ) ) { bool Spilled = TRI -> eliminateSGPRToVGPRSpillFrameIndex ( MI , FI , nullptr ) ; ( void ) Spilled ; assert ( Spilled && <STR_LIT> ) ; } } } for ( MachineBasicBlock & MBB : MF ) { for ( auto SSpill : FuncInfo -> getSGPRSpillVGPRs ( ) ) MBB . addLiveIn ( SSpill . VGPR ) ; for ( MCPhysReg Reg : FuncInfo -> getVGPRSpillAGPRs ( ) ) MBB . addLiveIn ( Reg ) ; for ( MCPhysReg Reg : FuncInfo -> getAGPRSpillVGPRs ( ) ) MBB . addLiveIn ( Reg ) ; MBB . sortUniqueLiveIns ( ) ; } MadeChange = true ; } else if (

FuncInfo -> VGPRReservedForSGPRSpill ) {


<s> unsigned Reg = CSRegs [ I ] ; if ( SavedRegs . test ( Reg ) ) { const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( Reg , <STR_LIT> ) ; int JunkFI = MFI . CreateStackObject ( TRI -> getSpillSize ( * RC ) , TRI -> getSpillAlignment ( * RC ) , true ) ; C<ISA_LIT> . push_back ( CalleeSavedInfo ( Reg , JunkFI ) ) ; } } if ( ! C<ISA_LIT> . empty ( )


<s> <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! MFI . hasStackObjects ( ) && ! HasCSRs ) { SaveBlocks . clear ( ) ; RestoreBlocks . clear ( ) ; if ( FuncInfo -> VGPRReservedForSGPRSpill ) { FuncInfo -> removeVGPRForSGPRSpill ( FuncInfo -> VGPRReservedForSGPRSpill , MF ) ; MRI . freezeReservedRegs ( MF ) ; } return false ; } bool MadeChange = false ; bool NewReservedRegs = false ; const bool HasSGPRSpillToVGPR = TRI -> spillSGPRToVGPR ( ) && ( HasCSRs || FuncInfo -> hasSpilledSGPRs ( ) ) ; if ( HasSGPRSpillToVGPR ) { lowerShiftReservedVGPR ( MF , ST ) ; BitVector SpillFIs ( MFI . getObjectIndexEnd ( ) , false ) ; for ( MachineBasicBlock & MBB : MF ) { for ( MachineInstr & MI : llvm :: make_early_inc_range ( MBB ) ) { if ( ! TII -> isSGPRSpill ( MI ) ) continue ; int FI = TII -> getNamedOperand ( MI , <STR_LIT> :: addr ) -> getIndex ( ) ; assert ( MFI . getStackID ( FI ) == TargetStackID :: SGPRSpill ) ; if ( FuncInfo -> allocateSGPRSpillToVGPR ( MF , FI ) ) { NewReservedRegs =


<s> Optional < int > FI ; if ( FuncInfo -> isCalleeSavedReg ( CSRegs , LowestAvailableVGPR ) ) FI = FrameInfo . CreateSpillStackObject ( <NUM_LIT> , Align ( <NUM_LIT> ) ) ; const auto * ReservedVGPRInfoItr = std :: find_if ( FuncInfo -> getSGPRSpillVGPRs ( ) . begin ( ) , FuncInfo -> getSGPRSpillVGPRs ( ) . end ( ) , [ PreReservedVGPR ] ( const auto & SpillRegInfo ) { return SpillRegInfo . VGPR == PreReservedVGPR ; } ) ; assert ( ReservedVGPRInfoItr != FuncInfo -> getSGPRSpillVGPRs ( ) . end ( ) ) ; auto Index = std :: distance ( FuncInfo -> getSGPRSpillVGPRs ( ) . begin ( ) , ReservedVGPRInfoItr ) ; FuncInfo -> setSGPRSpillVGPRs ( LowestAvailableVGPR , FI , Index ) ; for ( MachineBasicBlock & MBB : MF ) { MBB . addLiveIn ( LowestAvailableVGPR ) ; MBB . sortUniqueLiveIns ( ) ; } return true


<s> const TargetInstrInfo & TII = * MF . getSubtarget ( ) . getInstrInfo ( ) ; const TargetFrameLowering * TFI = MF . getSubtarget ( ) . getFrameLowering ( ) ; const TargetRegisterInfo * TRI = MF . getSubtarget ( ) . getRegisterInfo ( ) ; MachineBasicBlock :: iterator I = SaveBlock . begin ( ) ; if ( ! TFI -> spillCalleeSavedRegisters ( SaveBlock , I , C<ISA_LIT> , TRI ) ) { for ( const CalleeSavedInfo & CS


<s> const TargetFrameLowering * TFI = MF . getSubtarget ( ) . getFrameLowering ( ) ; const TargetRegisterInfo * TRI = MF . getSubtarget ( ) . getRegisterInfo ( ) ; MachineBasicBlock :: iterator I = SaveBlock . begin ( ) ; if ( ! TFI -> spillCalleeSavedRegisters ( SaveBlock , I , C<ISA_LIT> , TRI ) ) { for ( const CalleeSavedInfo & CS : C<ISA_LIT> ) { MCRegister Reg = CS .


<s> return false ; } MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const bool SpillVGPRToAGPR = ST . hasMAIInsts ( ) && FuncInfo -> hasSpilledVGPRs ( ) && EnableSpillVGPRToAGPR ; bool MadeChange = false ; const bool SpillToAGPR = EnableSpillVGPRToAGPR && ST . hasMAIInsts ( ) ; std :: unique_ptr < RegScavenger > RS ; bool NewReservedRegs = false ; const bool HasSGPRSpillToVGPR = TRI -> spillSGPRToVGPR ( ) && ( HasCSRs || FuncInfo -> hasSpilledSGPRs ( ) ) ; if ( HasSGPRSpillToVGPR || SpillVGPRToAGPR ) { lowerShiftReservedVGPR ( MF , ST ) ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator Next ; for ( auto I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; I = Next ) { MachineInstr & MI = * I ; Next = std :: next ( I ) ; if ( SpillToAGPR && TII -> isVGPRSpill ( MI ) ) { unsigned FIOp = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> ::


<s> if ( TRI -> spillSGPRToVGPR ( ) && ( HasCSRs || FuncInfo -> hasSpilledSGPRs ( ) ) ) { for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator Next ; for ( auto I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; I = Next ) { MachineInstr & MI = * I ; Next = std :: next ( I ) ; if ( ! TII -> isSGPRSpill ( MI ) ) continue ; int FI = TII -> getNamedOperand ( MI , <STR_LIT> :: addr ) -> getIndex ( ) ; assert ( MFI . getStackID ( FI ) == TargetStackID :: SGPRSpill ) ; if ( FuncInfo -> allocateSGPRSpillToVGPR ( MF , FI ) ) { bool Spilled = TRI -> eliminateSGPRToVGPRSpillFrameIndex ( MI , FI , nullptr ) ; ( void ) Spilled ; assert ( Spilled


<s> const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( Reg , <STR_LIT> ) ; TII . loadRegFromStackSlot ( RestoreBlock , I , Reg , CI . getFrameIdx ( ) , RC , TRI ) ; assert ( I != RestoreBlock . begin ( ) && <STR_LIT> ) ; if ( LIS ) { MachineInstr & Inst = * std ::


<s> } } } for ( MachineBasicBlock & MBB : MF ) { for ( auto SSpill : FuncInfo -> getSGPRSpillVGPRs ( ) ) MBB . addLiveIn ( SSpill . VGPR ) ; MBB . sortUniqueLiveIns ( ) ; for ( MachineInstr & MI : MBB ) { if ( MI . isDebugValue ( ) && MI . getOperand ( <NUM_LIT> ) . isFI ( ) && SpillFIs [ MI . getOperand ( <NUM_LIT> ) . getIndex ( ) ] ) { MI . getOperand ( <NUM_LIT> ) . ChangeToRegister ( Register ( ) , false ) ; } } } FuncInfo -> removeDeadFrameIndices ( MFI ) ; MadeChange = true ; } SaveBlocks . clear ( ) ; RestoreBlocks . clear ( ) ; if ( NewReservedRegs ) MRI . freezeReservedRegs ( MF ) ; return MadeChange


<s> MCRegister Reg = CS . getReg ( ) ; MachineInstrSpan MIS ( I , & SaveBlock ) ; const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( Reg , Reg == RI -> getReturnAddressReg ( MF ) ? <STR_LIT> : <STR_LIT> ) ; const bool IsLiveIn = MRI . isLiveIn ( Reg ) ; TII . storeRegToStackSlot ( SaveBlock , I , Reg , ! IsLiveIn , CS . getFrameIdx ( ) , RC , TRI ) ; if ( LIS ) { assert ( std :: distance ( MIS . begin ( ) , I ) == <NUM_LIT> ) ; MachineInstr & Inst = * std ::


<s> MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! MFI . hasStackObjects ( ) && ! HasCSRs ) { SaveBlocks . clear ( ) ; RestoreBlocks . clear ( ) ; return false ; } bool MadeChange = false ; bool NewReservedRegs = false ; const bool HasSGPRSpillToVGPR = TRI -> spillSGPRToVGPR ( ) && ( HasCSRs || FuncInfo -> hasSpilledSGPRs ( ) ) ; if ( HasSGPRSpillToVGPR ) { BitVector SpillFIs ( MFI . getObjectIndexEnd ( ) , false ) ; for ( MachineBasicBlock & MBB : MF ) { for ( MachineInstr & MI : llvm :: make_early_inc_range ( MBB ) ) { if ( ! TII -> isSGPRSpill ( MI ) ) continue ; int FI = TII -> getNamedOperand ( MI , <STR_LIT> :: addr ) -> getIndex ( ) ; assert ( MFI . getStackID ( FI ) == TargetStackID :: SGPRSpill ) ; if ( FuncInfo -> allocateSGPRSpillToVGPR ( MF , FI ) ) { NewReservedRegs = true ; bool Spilled = TRI -> eliminateSGPRToVGPRSpillFrameIndex ( MI , FI , nullptr , LIS ) ; ( void ) Spilled ; assert ( Spilled && <STR_LIT> ) ; SpillFIs . set ( FI ) ; } } } for ( MachineBasicBlock & MBB : MF ) { for ( auto SSpill : FuncInfo -> getSGPRSpillVGPRs ( ) ) MBB . addLiveIn ( SSpill . VGPR ) ; MBB . sortUniqueLiveIns ( ) ; for ( MachineInstr & MI : MBB ) { if ( MI . isDebugValue ( ) && MI . getOperand ( <NUM_LIT> ) . isFI ( ) && SpillFIs [ MI . getOperand ( <NUM_LIT> ) . getIndex ( ) ] ) { MI . getOperand ( <NUM_LIT> ) . ChangeToRegister ( Register ( ) , false ) ; } } } FuncInfo -> removeDeadFrameIndices ( MFI , false ) ; MadeChange = true ; } SaveBlocks . clear ( ) ; RestoreBlocks . clear ( ) ; if ( NewReservedRegs ) MRI .


<s> SaveBlocks . push_back ( & MF . front ( ) ) ; for ( MachineBasicBlock & MBB : MF ) { if ( MBB . isEHFuncletEntry ( ) ) SaveBlocks . push_back ( & MBB ) ; if ( MBB . isReturnBlock ( ) ) RestoreBlocks . push_back ( & MBB


<s> AU . setPreservesAll ( ) ; MachineFunctionPass :: getAnalysisUsage ( AU


<s> } ) ; assert ( ReservedVGPRInfoItr != FuncInfo -> getSGPRSpillVGPRs ( ) . end ( ) ) ; auto Index = std :: distance ( FuncInfo -> getSGPRSpillVGPRs ( ) . begin ( ) , ReservedVGPRInfoItr ) ; FuncInfo -> setSGPRSpillVGPRs ( LowestAvailableVGPR , FI , Index ) ; for ( MachineBasicBlock & MBB : MF ) { assert ( LowestAvailableVGPR . isValid ( ) && <STR_LIT> ) ; MBB . addLiveIn ( LowestAvailableVGPR ) ; MBB . sortUniqueLiveIns (


<s> MachineBasicBlock :: iterator Next ; for ( auto I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; I = Next ) { MachineInstr & MI = * I ; Next = std :: next ( I ) ; if ( ! TII -> isSGPRSpill ( MI ) ) continue ; int FI = TII -> getNamedOperand ( MI , <STR_LIT> :: addr ) -> getIndex ( ) ; assert ( MFI . getStackID ( FI ) == TargetStackID :: SGPRSpill ) ; if ( FuncInfo -> allocateSGPRSpillToVGPR ( MF , FI ) ) { NewReservedRegs = true ; bool Spilled = TRI -> eliminateSGPRToVGPRSpillFrameIndex ( MI , FI , nullptr , LIS ) ; ( void ) Spilled ; assert ( Spilled && <STR_LIT> ) ; SpillFIs . set ( FI ) ; } } } for ( MachineBasicBlock & MBB : MF ) { for ( auto SSpill : FuncInfo -> getSGPRSpillVGPRs ( ) ) MBB . addLiveIn ( SSpill . VGPR ) ; MBB . sortUniqueLiveIns ( ) ; for ( MachineInstr & MI : MBB ) { if ( MI . isDebugValue ( ) && MI . getOperand ( <NUM_LIT> ) . isFI ( ) && SpillFIs [ MI . getOperand ( <NUM_LIT> ) . getIndex ( ) ] ) { MI . getOperand ( <NUM_LIT> ) . ChangeToRegister ( Register ( ) , false ) ; } } } FuncInfo -> removeDeadFrameIndices ( MF ) ; MadeChange = true ; } else if ( FuncInfo ->


<s> LIS = getAnalysisIfAvailable < LiveIntervals > ( ) ; assert ( SaveBlocks . empty ( ) && RestoreBlocks . empty ( ) ) ; calculateSaveRestoreBlocks ( MF ) ; bool HasCSRs = spillCalleeSavedRegs ( MF ) ; MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! MFI . hasStackObjects ( ) && ! HasCSRs ) { SaveBlocks . clear ( ) ; RestoreBlocks . clear ( ) ; if ( FuncInfo -> VGPRReservedForSGPRSpill ) { FuncInfo -> removeVGPRForSGPRSpill ( FuncInfo -> VGPRReservedForSGPRSpill , MF ) ; MRI . freezeReservedRegs ( MF ) ; } return false ; } bool MadeChange = false ; bool NewReservedRegs = false ; const bool HasSGPRSpillToVGPR = TRI -> spillSGPRToVGPR ( ) && ( HasCSRs || FuncInfo -> hasSpilledSGPRs ( ) ) ; if ( HasSGPRSpillToVGPR ) { lowerShiftReservedVGPR ( MF , ST ) ; BitVector SpillFIs ( MFI . getObjectIndexEnd ( ) , false ) ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator Next ; for ( auto I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; I = Next ) { MachineInstr & MI = * I ; Next = std :: next ( I ) ; if ( ! TII -> isSGPRSpill ( MI ) ) continue ; int FI = TII -> getNamedOperand ( MI , <STR_LIT> :: addr ) -> getIndex ( ) ; assert ( MFI . getStackID ( FI ) == TargetStackID :: SGPRSpill ) ; if ( FuncInfo -> allocateSGPRSpillToVGPR ( MF , FI ) ) { NewReservedRegs = true ; bool Spilled = TRI -> eliminateSGPRToVGPRSpillFrameIndex ( MI , FI , nullptr , LIS ) ; ( void )


<s> int JunkFI = MFI . CreateStackObject ( TRI -> getSpillSize ( * RC ) , TRI -> getSpillAlign ( * RC ) , true ) ; C<ISA_LIT> . push_back ( CalleeSavedInfo ( Reg , JunkFI ) ) ; } } if ( ! C<ISA_LIT> . empty ( ) ) { for ( MachineBasicBlock * SaveBlock : SaveBlocks ) insertCSRSaves ( ST , * SaveBlock , C<ISA_LIT> , LIS ) ; assert ( SaveBlocks . size ( ) == <NUM_LIT> && <STR_LIT> ) ; updateLiveness ( MF , C<ISA_LIT> ) ; for ( MachineBasicBlock * RestoreBlock : RestoreBlocks ) insertCSRRestores


<s> const MCPhysReg * CSRegs = MRI . getCalleeSavedRegs ( ) ; for ( unsigned I = <NUM_LIT> ; CSRegs [ I ] ; ++ I ) { MCRegister Reg = CSRegs [ I ] ; if ( SavedRegs . test ( Reg ) ) { const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( Reg , <STR_LIT> ) ; int JunkFI = MFI . CreateStackObject ( TRI -> getSpillSize ( * RC ) , TRI -> getSpillAlign ( * RC ) , true ) ; C<ISA_LIT> . push_back ( CalleeSavedInfo ( Reg , JunkFI ) ) ; } } if ( ! C<ISA_LIT> . empty ( ) ) { for ( MachineBasicBlock * SaveBlock : SaveBlocks ) insertCSRSaves ( ST , * SaveBlock , C<ISA_LIT> , LIS ) ; assert ( SaveBlocks . size ( ) == <NUM_LIT> && <STR_LIT> ) ; updateLiveness ( MF , C<ISA_LIT>


<s> static void updateLiveness ( MachineFunction & MF , ArrayRef < CalleeSavedInfo > C<ISA_LIT> ) { MachineBasicBlock & EntryBB = MF . front (


<s> for ( const CalleeSavedInfo & C<ISA_LIT>Reg : C<ISA_LIT> ) EntryBB . addLiveIn


<s> void addToSpilledVGPRs ( unsigned num


<s> const <ISA_LIT>BufferPseudoSourceValue * getBufferPSV (


<s> int getDebuggerWorkGroupIDStackObjectIndex ( unsigned Dim ) const { assert ( Dim < <NUM_LIT> ) ; return DebuggerWorkGroupIDStackObjectIndices [


<s> int getDebuggerWorkGroupIDStackObjectIndex ( unsigned Dim ) const { assert ( Dim < <NUM_LIT> ) ; return DebuggerWorkGroupIDStackObjectIndices [ Dim


<s> std :: pair < unsigned , unsigned > getFlatWorkGroupSizes ( ) const { return


<s> std :: pair < unsigned , unsigned > getFlatWorkGroupSizes ( )


<s> unsigned getFrameOffsetReg ( ) const { return FrameOffsetReg


<s> const <ISA_LIT>ImagePseudoSourceValue * getImagePSV


<s> const <ISA_LIT>ImagePseudoSourceValue * getImagePSV ( ) const { return & ImagePSV


<s> unsigned getImplicitBufferPtrUserSGPR ( )


<s> unsigned getLDSWaveSpillSize ( ) const { return


<s> unsigned getMinFlatWorkGroupSize ( ) const


<s> unsigned getNumSpilledSGPRs ( ) const { return NumSpilledSGPRs


<s> unsigned getNumSpilledVGPRs ( ) const { return


<s> unsigned getP<ISA_LIT>nputEnable ( ) const { return P<ISA_LIT>nputEnable


<s> std :: pair < unsigned , unsigned > getWavesPerEU ( ) const { return WavesPerEU


<s> case <NUM_LIT> : assert ( hasWorkItemIDX ( ) ) ; return <STR_LIT> ; case <NUM_LIT> : assert ( hasWorkItemIDY ( ) ) ; return <STR_LIT>


<s> unsigned getWorkItemIDVGPR ( unsigned Dim ) const { switch ( Dim ) { case <NUM_LIT> : assert ( hasWorkItemIDX ( ) ) ; return <STR_LIT> ; case <NUM_LIT> : assert ( hasWorkItemIDY ( ) ) ; return <STR_LIT> ; case <NUM_LIT> : assert ( hasWorkItemIDZ ( ) ) ; return <STR_LIT>


<s> bool hasImplicitBufferPtr ( ) const { return ImplicitBufferPtr


<s> bool hasNonSpillStackObjects ( ) const { return HasNonSpillStackObjects


<s> bool isAliased ( const MachineFrameInfo * )


<s> bool isConstant ( const MachineFrameInfo * )


<s> return P<ISA_LIT>nputAddr & ( <NUM_LIT> << Index


<s> return P<ISA_LIT>nputAddr & ( <NUM_LIT> <<


<s> bool mayAlias ( const MachineFrameInfo * ) const


<s> assert ( Dim < <NUM_LIT> ) ; DebuggerWorkGroupIDStackObjectIndices [ Dim ] =


<s> void setDebuggerWorkItemIDStackObjectIndex ( unsigned Dim , int ObjectIdx ) { assert ( Dim


<s> assert ( Dim < <NUM_LIT> ) ; DebuggerWorkItemIDStackObjectIndices [ Dim


<s> void setIfReturnsVoid ( bool Value ) { ReturnsVoid =


<s> unsigned <ISA_LIT>MachineFunctionInfo :: addDispatchPtr ( const <ISA_LIT>RegisterInfo & TRI ) { DispatchPtrUserSGPR = TRI . getMatchingSuperReg ( getNextUserSGPR ( ) , <STR_LIT> , & <STR_LIT> ) ; NumUserSGPRs +=


<s> PrivateSegmentBufferUserSGPR = TRI . getMatchingSuperReg ( getNextUserSGPR ( ) ,


<s> PrivateSegmentBufferUserSGPR = TRI . getMatchingSuperReg ( getNextUserSGPR ( ) , <STR_LIT> , & <STR_LIT> ) ; NumUserSGPRs += <NUM_LIT>


<s> Register addPrivateSegmentWaveByteOffset ( ) { ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( getNextSystemSGPR (


<s> Register addPrivateSegmentWaveByteOffset ( ) { ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( getNextSystemSGPR ( ) ) ; NumSystemSGPRs += <NUM_LIT> ; return ArgInfo . PrivateSegmentWaveByteOffset


<s> QueuePtrUserSGPR = TRI . getMatchingSuperReg ( getNextUserSGPR ( ) , <STR_LIT> , & <STR_LIT> ) ; NumUserSGPRs +=


<s> ArgInfo . WorkGroupIDZ = ArgDescriptor :: createRegister


<s> static bool classof ( const PseudoSourceValue * V ) { return V -> kind ( ) ==


<s> static inline <ISA_LIT>Argument createArgument ( bool IsReg ) { if ( IsReg ) return <ISA_LIT>Argument ( IsReg ) ; return <ISA_LIT>Argument (


<s> ArrayRef < MCPhysReg > getAGPRSpillVGPRs ( ) const { return


<s> const <ISA_LIT>FunctionArgInfo & getArgInfo ( ) const { return


<s> const <ISA_LIT>BufferPseudoSourceValue * getBufferPSV ( const <ISA_LIT>TargetMachine & TM ) { if ( ! BufferPSV ) BufferPSV = std :: make_unique < <ISA_LIT>BufferPseudoSourceValue > (


<s> unsigned getGITPtrHigh ( ) const { return GITPtrHigh


<s> Register getImplicitBufferPtrUserSGPR ( ) const { return ArgInfo . ImplicitBufferPtr . getRegister


<s> Register getImplicitBufferPtrUserSGPR (


<s> return getShaderType ( ) == ShaderType :: COMPUTE ? <NUM_LIT>


<s> if ( ! isMemoryBound ( ) && ! needsWaveLimiter ( ) ) return Occupancy ; return ( Occupancy < <NUM_LIT> ) ? Occupancy :


<s> Optional < int > getOptionalScavengeFI ( ) const { return ScavengeFI


<s> MCRegister getPreloadedReg ( <STR_LIT> Value ) const { auto Arg = std :: get < <NUM_LIT> > ( ArgInfo . getPreloadedValue


<s> std :: tuple < const ArgDescriptor * , const TargetRegisterClass * , LLT > getPreloadedValue ( <STR_LIT> Value ) const { return ArgInfo . getPreloadedValue ( Value


<s> Register getQueuePtrUserSGPR ( ) const { return ArgInfo . QueuePtr


<s> Register getStackPtrOffsetReg ( ) const { return StackPtrOffsetReg


<s> Register getVGPRForAGPRCopy ( ) const { return


<s> auto I = VGPRToAGPRSpills . find ( FrameIndex ) ; return ( I == VGPRToAGPRSpills . end ( ) ) ? ( MCPhysReg ) <STR_LIT> : I ->


<s> bool hasImplicitArgPtr ( ) const { return


<s> if ( Occupancy < Limit ) Occupancy = Limit ; limitOccupancy ( MF )


<s> void limitOccupancy ( unsigned Limit ) { if ( Occupancy > Limit ) Occupancy


<s> static void mapping ( IO & YamlIO , <ISA_LIT>MachineFunctionInfo & MFI ) { YamlIO . mapOptional ( <STR_LIT> , MFI . ExplicitKernArgSize , UINT64_C ( <NUM_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . MaxKernArgAlign , <NUM_LIT> ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . LDSSize , <NUM_LIT> ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . GDSSize , <NUM_LIT> ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . DynLDSAlign , Align ( ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . IsEntryFunction , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . NoSignedZerosFPMath , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . MemoryBound , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . WaveLimiter , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . HasSpilledSGPRs , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . HasSpilledVGPRs , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchRSrcReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . StackPtrOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . BytesInStackArgArea , <NUM_LIT> ) ; YamlIO . mapOptional ( <STR_LIT> , MFI .


<s> bool mayNeedAGPRs ( ) const { return MayNeedAGPRs


<s> void printCustom ( raw_ostream & OS )


<s> void printCustom ( raw_ostream & OS ) const override { OS <<


<s> void reserveWWMRegister ( Register Reg ) { WWMReservedRegs . insert (


<s> void setBytesInStackArgArea ( unsigned


<s> ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor ::


<s> assert ( Reg != <NUM_LIT> && <STR_LIT> ) ; ScratchRSrcReg =


<s> void setScratchRSrcReg ( Register Reg ) { assert ( Reg !=


<s> auto I = VGPRToAGPRSpills . find ( FrameIndex ) ; if ( I != VGPRToAGPRSpills . end ( ) )


<s> void setWorkItemIDX ( ArgDescriptor Arg )


<s> void setWorkItemIDZ ( ArgDescriptor Arg


<s> assert ( WWMReservedRegs . size ( ) == WWMReservedFrameIndexes


<s> auto wwmAllocation ( ) const { assert ( WWMReservedRegs . size ( )

== WWMReservedFrameIndexes . size ( ) ) ;


<s> ArgInfo . DispatchID = ArgDescriptor :: createRegister ( TRI . getMatchingSuperReg (

getNextUserSGPR ( ) , <STR_LIT> , & <STR_LIT> ) ) ;


<s> unsigned <ISA_LIT>MachineFunctionInfo :: addDispatchID ( const <ISA_LIT>RegisterInfo & TRI ) { ArgInfo . DispatchID = ArgDescriptor :: createRegister ( TRI . getMatchingSuperReg ( getNextUserSGPR ( ) , <STR_LIT> , & <STR_LIT> ) ) ; NumUserSGPRs +=


<s> ArgInfo . FlatScratchInit = ArgDescriptor :: createRegister ( TRI . getMatchingSuperReg ( getNextUserSGPR ( ) , <STR_LIT> , & <STR_LIT> ) ) ; NumUserSGPRs += <NUM_LIT> ; return ArgInfo . FlatScratchInit . getRegister


<s> unsigned <ISA_LIT>MachineFunctionInfo :: addFlatScratchInit ( const <ISA_LIT>RegisterInfo & TRI ) { ArgInfo . FlatScratchInit = ArgDescriptor :: createRegister ( TRI . getMatchingSuperReg ( getNextUserSGPR ( ) , <STR_LIT> , & <STR_LIT> ) ) ; NumUserSGPRs += <NUM_LIT>


<s> ArgInfo . ImplicitBufferPtr = ArgDescriptor :: createRegister ( TRI . getMatchingSuperReg ( getNextUserSGPR ( ) , <STR_LIT> , & <STR_LIT> ) ) ; NumUserSGPRs += <NUM_LIT>


<s> ArgInfo . KernargSegmentPtr = ArgDescriptor :: createRegister ( TRI . getMatchingSuperReg (


<s> ArgInfo . KernargSegmentPtr = ArgDescriptor :: createRegister ( TRI . getMatchingSuperReg ( getNextUserSGPR ( ) ,


<s> PrivateSegmentWaveByteOffsetSystemSGPR = getNextSystemSGPR


<s> unsigned <ISA_LIT>MachineFunctionInfo :: addQueuePtr ( const <ISA_LIT>RegisterInfo & TRI ) { ArgInfo . QueuePtr = ArgDescriptor :: createRegister ( TRI . getMatchingSuperReg ( getNextUserSGPR ( ) , <STR_LIT> , & <STR_LIT> ) ) ; NumUserSGPRs +=


<s> unsigned addWorkGroupIDY ( ) { WorkGroupIDYSystemSGPR = getNextSystemSGPR


<s> unsigned addWorkGroupIDY ( ) { WorkGroupIDYSystemSGPR = getNextSystemSGPR ( ) ; NumSystemSGPRs +=


<s> unsigned addWorkGroupIDZ ( ) { WorkGroupIDZSystemSGPR = getNextSystemSGPR ( ) ; NumSystemSGPRs +=


<s> assert ( TRI -> spillSGPRToVGPR ( ) && <STR_LIT> ) ; int NumLanes = Size / <NUM_LIT> ; const MCPhysReg * CSRegs = TRI -> getCalleeSavedRegs ( & MF ) ; for ( int I = <NUM_LIT> ; I < NumLanes ; ++ I , ++ NumVGPRSpillLanes ) { unsigned LaneVGPR ; unsigned VGPRIndex = ( NumVGPRSpillLanes % WaveSize ) ; if ( VGPRIndex == <NUM_LIT> ) { LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> , MF ) ; if ( LaneVGPR == <STR_LIT> ) { SGPRToVGPRSpills . erase ( FI ) ; NumVGPRSpillLanes -= I ; return false ; } Optional < int > CSRSpillFI ; if ( FrameInfo . hasCalls ( ) && CSRegs && isCalleeSavedReg ( CSRegs , LaneVGPR ) ) { CSRSpillFI = FrameInfo . CreateStackObject ( <NUM_LIT> , <NUM_LIT> , false ) ; } SpillVGPRs . push_back ( SGPRSpillVGPRCSR ( LaneVGPR , CSRSpillFI ) ) ; for ( MachineBasicBlock & BB : MF ) BB . addLiveIn ( LaneVGPR ) ; } else { LaneVGPR = SpillVGPRs . back ( ) . VGPR ; } SpillLanes . push_back ( SpilledReg ( LaneVGPR , VGPRIndex ) ) ; } return


<s> assert ( NumSystemSGPRs == <NUM_LIT> &&


<s> unsigned getNumUserSGPRs ( ) const { return NumUserSGPRs


<s> unsigned getScratchRSrcReg ( ) const


<s> unsigned getScratchWaveOffsetReg ( )


<s> unsigned getTIDReg ( ) const { return


<s> bool hasCalculatedTID ( ) const { return TIDReg !=


<s> bool hasCalculatedTID ( ) const { return TIDReg


<s> bool hasDispatchPtr ( ) const { return DispatchPtr


<s> bool hasDispatchPtr ( ) const { return


<s> bool hasGridWorkgroupCountX ( ) const { return GridWorkgroupCountX


<s> bool hasGridWorkgroupCountZ ( ) const { return GridWorkgroupCountZ


<s> bool hasKernargSegmentPtr ( ) const { return KernargSegmentPtr


<s> bool hasPrivateSegmentBuffer ( ) const { return


<s> bool hasPrivateSegmentWaveByteOffset ( )


<s> bool hasQueuePtr ( ) const { return QueuePtr


<s> bool hasWorkGroupIDY ( ) const { return WorkGroupIDY


<s> bool hasWorkItemIDX ( ) const { return


<s> bool hasWorkItemIDY ( ) const { return WorkItemIDY


<s> static bool isCalleeSavedReg ( const MCPhysReg * CSRegs , MCPhysReg Reg ) { for ( unsigned I = <NUM_LIT> ; CSRegs [ I ] ; ++ I ) { if ( CSRegs [ I ] == Reg ) return true ; } return


<s> for ( auto & R : SGPRToVGPRSpills ) MFI . RemoveStackObject ( R .


<s> void <ISA_LIT>MachineFunctionInfo :: removeSGPRToVGPRFrameIndices ( MachineFrameInfo & MFI ) { for ( auto & R : SGPRToVGPRSpills ) MFI . RemoveStackObject ( R


<s> void setHasSpilledSGPRs ( bool Spill = true ) { HasSpilledSGPRs = Spill


<s> void setHasSpilledSGPRs ( bool Spill


<s> void setScratchRSrcReg ( unsigned Reg ) { assert ( Reg != <STR_LIT> && <STR_LIT> ) ; ScratchRSrcReg =


<s> const MachineFrameInfo * FrameInfo = MF -> getFrameInfo ( ) ; const <ISA_LIT>RegisterInfo * TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( MF -> getSubtarget < <ISA_LIT>Subtarget > ( ) . getRegisterInfo ( ) ) ; MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; int64_t Offset = FrameInfo -> getObjectOffset ( FrameIndex ) ; Offset += SubIdx * <NUM_LIT> ; unsigned LaneVGPRIdx = Offset / ( <NUM_LIT> * <NUM_LIT> ) ; unsigned Lane = ( Offset / <NUM_LIT> ) % <NUM_LIT> ; struct SpilledReg Spill ; if ( ! LaneVGPRs . count ( LaneVGPRIdx ) ) { unsigned LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> ) ; LaneVGPRs [ LaneVGPRIdx ] = LaneVGPR ; for ( MachineFunction :: iterator BI = MF -> begin ( ) , BE = MF -> end ( ) ; BI != BE ; ++ BI ) { BI -> addLiveIn ( LaneVGPR ) ; } } Spill . VGPR = LaneVGPRs [ LaneVGPRIdx ] ; Spill . Lane = Lane ; return


<s> FlatScratchInitUserSGPR = TRI . getMatchingSuperReg ( getNextUserSGPR ( ) , <STR_LIT> , & <STR_LIT> ) ; NumUserSGPRs += <NUM_LIT> ; return FlatScratchInitUserSGPR


<s> unsigned <ISA_LIT>MachineFunctionInfo :: addPrivateMemoryPtr ( const <ISA_LIT>RegisterInfo & TRI ) { PrivateMemoryPtrUserSGPR = TRI . getMatchingSuperReg ( getNextUserSGPR ( )


<s> LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> , MF ) ; if ( LaneVGPR == <STR_LIT> ) { SGPRToVGPRSpills . erase ( FI ) ; NumVGPRSpillLanes -= I ; return false ; } SpillVGPRs . push_back ( LaneVGPR ) ; for ( MachineBasicBlock & BB : MF ) BB . addLiveIn ( LaneVGPR ) ; } else { LaneVGPR = SpillVGPRs . back ( ) ; } SpillLanes . push_back ( SpilledReg ( LaneVGPR , VGPRIndex ) ) ; } return true


<s> } CallingConv :: ID CC = F -> getCallingConv ( ) ; if ( CC == CallingConv :: <ISA_LIT>_KERNEL || CC == CallingConv :: SPIR_KERNEL ) { KernargSegmentPtr = true ; WorkGroupIDX = true ; WorkItemIDX = true ; } else if ( CC == CallingConv :: <ISA_LIT>_PS ) { P<ISA_LIT>nputAddr = <STR_LIT> ( * F ) ; } if ( ST . debuggerEmitPrologue ( ) ) { WorkGroupIDY = true ; WorkGroupIDZ = true ; WorkItemIDY = true ; WorkItemIDZ = true ; } else { if ( F -> hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; } if ( WorkItemIDZ ) WorkItemIDY = true ; const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; bool MaySpill = ST . isVGPRSpillingEnabled ( * F ) ; bool HasStackObjects = FrameInfo . hasStackObjects ( ) || FrameInfo . hasCalls ( ) ; if ( HasStackObjects || MaySpill ) { PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) PrivateSegmentWaveByteOffsetSystemSGPR = <STR_LIT> ; } if ( ST . isAmdCodeObjectV2 ( MF ) ) { if ( HasStackObjects || MaySpill ) PrivateSegmentBuffer = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) )


<s> LDSSize = YamlMFI . LDSSize ; IsEntryFunction = YamlMFI . IsEntryFunction ; NoSignedZerosFPMath = YamlMFI . NoSignedZerosFPMath ; MemoryBound = YamlMFI . MemoryBound ; WaveLimiter = YamlMFI


<s> MaxKernArgAlign = YamlMFI . MaxKernArgAlign ; LDSSize = YamlMFI . LDSSize ; IsEntryFunction = YamlMFI . IsEntryFunction ; NoSignedZerosFPMath = YamlMFI . NoSignedZerosFPMath ; MemoryBound = YamlMFI . MemoryBound ; WaveLimiter = YamlMFI


<s> static yaml :: StringValue regToString ( unsigned Reg , const TargetRegisterInfo & TRI ) { yaml :: StringValue Dest ; { raw_string_ostream OS ( Dest . Value ) ; OS << printReg ( Reg


<s> } for ( int i = MFI . getObjectIndexBegin ( ) , e = MFI . getObjectIndexEnd ( ) ; i != e ; ++ i ) if ( i != FramePointerSaveIndex ) MFI .

setStackID ( i , TargetStackID :: Default ) ;


<s> ArgInfo . PrivateSegmentBuffer = ArgDescriptor :: createRegister ( TRI . getMatchingSuperReg ( getNextUserSGPR ( ) , <STR_LIT> ,


<s> LDSSize = YamlMFI . LDSSize ; HighBitsOf32BitAddress = YamlMFI . HighBitsOf32BitAddress ; IsEntryFunction = YamlMFI . IsEntryFunction ; NoSignedZerosFPMath = YamlMFI . NoSignedZerosFPMath ; MemoryBound = YamlMFI . MemoryBound ; WaveLimiter = YamlMFI


<s> for ( auto & R : SGPRToVGPRSpills ) { if ( R . first != FramePointerSaveIndex ) MFI . RemoveStackObject ( R . first ) ; } for ( int i = MFI . getObjectIndexBegin ( ) , e = MFI . getObjectIndexEnd ( ) ;


<s> } for ( int i = MFI . getObjectIndexBegin ( ) , e = MFI . getObjectIndexEnd ( ) ; i != e ; ++ i ) if ( i != FramePointerSaveIndex ) MFI . setStackID ( i , TargetStackID :: Default ) ; for ( auto & R : VGPRToAGPRSpills ) { if ( R . second . FullyAllocated ) MFI . RemoveStackObject


<s> unsigned <ISA_LIT>MachineFunctionInfo :: getMaximumWorkGroupSize ( const MachineFunction & MF ) const { return MaximumWorkGroupSize


<s> void setStackPtrOffsetReg ( unsigned Reg ) { assert ( Reg != <STR_LIT> &&


<s> <ISA_LIT>MachineFunctionInfo :: <ISA_LIT>MachineFunctionInfo ( const MachineFunction & MF ) : <ISA_LIT>MachineFunction ( MF ) , TIDReg ( <STR_LIT> ) , ScratchRSrcReg ( <STR_LIT> ) , ScratchWaveOffsetReg ( <STR_LIT> ) , PrivateSegmentBufferUserSGPR ( <STR_LIT> ) , DispatchPtrUserSGPR ( <STR_LIT> ) , QueuePtrUserSGPR ( <STR_LIT> ) , KernargSegmentPtrUserSGPR ( <STR_LIT> ) , DispatchIDUserSGPR ( <STR_LIT> ) , FlatScratchInitUserSGPR ( <STR_LIT> ) , PrivateSegmentSizeUserSGPR ( <STR_LIT> ) , GridWorkGroupCountXUserSGPR ( <STR_LIT> ) , GridWorkGroupCountYUserSGPR ( <STR_LIT> ) , GridWorkGroupCountZUserSGPR ( <STR_LIT> ) , WorkGroupIDXSystemSGPR ( <STR_LIT> ) , WorkGroupIDYSystemSGPR ( <STR_LIT> ) , WorkGroupIDZSystemSGPR ( <STR_LIT> ) , WorkGroupInfoSystemSGPR ( <STR_LIT> ) , PrivateSegmentWaveByteOffsetSystemSGPR ( <STR_LIT> ) , P<ISA_LIT>nputAddr ( <NUM_LIT> ) , ReturnsVoid ( true ) , MaximumWorkGroupSize ( <NUM_LIT> ) , DebuggerReservedVGPRCount ( <NUM_LIT> ) , DebuggerWorkGroupIDStackObjectIndices ( { { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } } ) , DebuggerWorkItemIDStackObjectIndices ( { { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } } ) , LDSWaveSpillSize ( <NUM_LIT> ) , P<ISA_LIT>nputEna ( <NUM_LIT> ) , NumUserSGPRs ( <NUM_LIT> ) , NumSystemSGPRs ( <NUM_LIT> ) , HasSpilledSGPRs ( false ) , HasSpilledVGPRs ( false ) , HasNonSpillStackObjects ( false ) , HasFlatInstructions ( false ) , NumSpilledSGPRs ( <NUM_LIT> ) , NumSpilledVGPRs ( <NUM_LIT> ) , PrivateSegmentBuffer ( false ) , DispatchPtr ( false ) , QueuePtr ( false ) , DispatchID ( false ) , KernargSegmentPtr ( false ) , FlatScratchInit ( false ) , GridWorkgroupCountX ( false ) , GridWorkgroupCountY ( false ) , GridWorkgroupCountZ ( false ) , WorkGroupIDX ( false ) , WorkGroupIDY ( false ) , WorkGroupIDZ ( false ) , WorkGroupInfo ( false ) , PrivateSegmentWaveByteOffset ( false ) , WorkItemIDX ( false ) , WorkItemIDY ( false ) , WorkItemIDZ ( false ) { const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const Function * F = MF


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; unsigned WaveSize = ST . getWavefrontSize ( ) ; unsigned Size = FrameInfo . getObjectSize ( FI ) ; unsigned NumLanes = Size / <NUM_LIT> ; if ( NumLanes > WaveSize ) return false ; assert ( Size >= <NUM_LIT> && <STR_LIT> ) ; assert ( TRI -> spillSGPRToVGPR ( ) && <STR_LIT> ) ; for ( unsigned I = <NUM_LIT> ; I < NumLanes ; ++ I , ++ NumVGPRSpillLanes ) { Register LaneVGPR ; unsigned VGPRIndex = ( NumVGPRSpillLanes % WaveSize ) ; if ( VGPRIndex == <NUM_LIT> ) { LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> , MF ) ; if ( LaneVGPR ==


<s> assert ( TRI -> spillSGPRToVGPR ( ) && <STR_LIT> ) ; for ( unsigned I = <NUM_LIT> ; I < NumLanes ; ++ I , ++ NumVGPRSpillLanes ) { Register LaneVGPR ; unsigned VGPRIndex = ( NumVGPRSpillLanes % WaveSize ) ; if ( VGPRIndex == <NUM_LIT> ) { LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> , MF ) ; if ( LaneVGPR == <STR_LIT> ) { SGPRToVGPRSpills . erase ( FI ) ; NumVGPRSpillLanes -= I ; DiagnosticInfoResourceLimit DiagOutOfRegs ( MF . getFunction ( ) , <STR_LIT> , <NUM_LIT> , DS_Error ) ; MF . getFunction ( ) . getContext ( ) . diagnose ( DiagOutOfRegs ) ; return false ; } Optional < int > SpillFI ; if ( ! isEntryFunction ( ) ) { SpillFI = FrameInfo . CreateSpillStackObject ( <NUM_LIT> , Align (


<s> unsigned getPrivateMemoryPtrUserSGPR ( )


<s> bool hasPrivateMemoryInputPtr ( ) const { return PrivateMemoryInputPtr


<s> bool hasPrivateMemoryInputPtr ( ) const { return


<s> if ( ! F . hasFnAttribute ( <STR_LIT> ) ) ImplicitArgPtr = true ; } else { ImplicitArgPtr = false ; MaxKernArgAlign = std :: max ( ST . getAlignmentForImplicitArgPtr ( ) , MaxKernArgAlign ) ; } bool isAmdHsaOrMesa = ST . isAmdHsaOrMesa ( F ) ; if ( isAmdHsaOrMesa && ! ST . enableFlatScratch ( ) ) PrivateSegmentBuffer = true ; else if ( ST . isMesaGfxShader ( F ) ) ImplicitBufferPtr = true ; if ( ! <STR_LIT> ( CC ) ) { if ( IsKernel || ! F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( IsKernel || ! F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) && ST . getMaxWorkitemID ( F , <NUM_LIT> ) != <NUM_LIT> ) WorkItemIDY = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) && ST . getMaxWorkitemID ( F , <NUM_LIT> ) != <NUM_LIT> ) WorkItemIDZ = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } bool HasStackObjects = F . hasFnAttribute ( <STR_LIT> ) ; if ( ST . hasFlatAddressSpace ( ) && isEntryFunction ( ) && ( isAmdHsaOrMesa || ST . enableFlatScratch ( ) ) && ( HasCalls || HasStackObjects || ST . enableFlatScratch ( ) ) && ! ST . flatScratchIsArchitected ( ) ) { FlatScratchInit = true ; } if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; if ( ! ST . flatScratchIsArchitected ( ) ) { PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS )

) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ;


<s> const Register Reg = Register :: index2VirtReg ( I ) ; const TargetRegisterClass * RC = MRI . getRegClassOrNull ( Reg ) ; if ( RC && <ISA_LIT>RegisterInfo :: isAGPRClass ( RC ) ) { UsesAGPRs = true ; return true ; } else if ( ! RC && ! MRI . use_empty ( Reg ) && MRI . getType ( Reg ) . isValid ( ) ) { return true ; } } for ( MCRegister Reg : <STR_LIT> ) { if ( MRI . isPhysRegUsed (


<s> unsigned addWorkGroupIDZ ( ) { ArgInfo . WorkGroupIDZ = ArgDescriptor :: createRegister ( getNextSystemSGPR ( ) ) ; NumSystemSGPRs += <NUM_LIT> ; return ArgInfo . WorkGroupIDZ . getRegister (


<s> explicit <ISA_LIT>BufferPseudoSourceValue ( const TargetInstrInfo &


<s> PseudoSourceValue ( PseudoSourceValue :: TargetCustom


<s> return ArgInfo . ImplicitBufferPtr . getRegister


<s> return ArgInfo . getPreloadedValue ( Value ) . first -> getRegister


<s> std :: pair < const ArgDescriptor * , const TargetRegisterClass * >

getPreloadedValue ( <STR_LIT> Value ) const {


<s> return ArgInfo . QueuePtr . getRegister


<s> ArrayRef < SGPRSpillVGPRCSR > getSGPRSpillVGPRs ( ) const { return SpillVGPRs


<s> switch ( Dim ) { case <NUM_LIT> : assert ( hasWorkGroupIDX ( ) ) ; return ArgInfo . WorkGroupIDX . getRegister ( ) ; case <NUM_LIT> : assert ( hasWorkGroupIDY ( ) ) ; return ArgInfo . WorkGroupIDY . getRegister ( ) ; case <NUM_LIT> : assert ( hasWorkGroupIDZ ( ) ) ; return ArgInfo . WorkGroupIDZ .


<s> bool hasCalculatedTID ( ) const { return TIDReg != <STR_LIT>


<s> } } for ( int i = MFI . getObjectIndexBegin ( ) , e = MFI . getObjectIndexEnd ( ) ; i != e ; ++ i ) if ( i != FramePointerSaveIndex && i != BasePointerSaveIndex )

MFI . setStackID ( i , TargetStackID :: Default ) ;


<s> void setPrivateSegmentWaveByteOffset ( unsigned Reg )


<s> void setScratchWaveOffsetReg ( unsigned Reg ) { assert ( Reg != <STR_LIT> && <STR_LIT>


<s> MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; unsigned WaveSize = ST . getWavefrontSize ( ) ; unsigned Size = FrameInfo . getObjectSize ( FI ) ; assert ( Size >= <NUM_LIT> && Size <= <NUM_LIT> && <STR_LIT> ) ; assert ( TRI -> spillSGPRToVGPR ( ) && <STR_LIT> ) ; int NumLanes = Size / <NUM_LIT> ; const MCPhysReg * CSRegs = TRI -> getCalleeSavedRegs ( & MF ) ; for ( int I = <NUM_LIT> ; I < NumLanes ; ++ I , ++ NumVGPRSpillLanes ) { unsigned LaneVGPR


<s> unsigned get32BitAddressHighBits ( )


<s> const <ISA_LIT>BufferPseudoSourceValue * getBufferPSV ( const <ISA_LIT>InstrInfo &


<s> assert ( BufferRsrc ) ; auto PSV = BufferPSVs . try_emplace ( BufferRsrc , llvm ::

make_unique < <ISA_LIT>BufferPseudoSourceValue > ( TII ) ) ;


<s> const <ISA_LIT>ImagePseudoSourceValue * getImagePSV ( const <ISA_LIT>InstrInfo & TII ,


<s> void setScratchRSrcReg ( unsigned Reg ) { assert ( Reg != <NUM_LIT>


<s> assert ( Reg != <NUM_LIT> && <STR_LIT> ) ; StackPtrOffsetReg =


<s> void setStackPtrOffsetReg ( unsigned Reg ) { assert ( Reg != <NUM_LIT> && <STR_LIT> ) ; StackPtrOffsetReg =


<s> void setHasFlatInstructions ( bool UseFlat


<s> if ( CC == CallingConv :: <ISA_LIT>_KERNEL || CC == CallingConv :: SPIR_KERNEL ) { if ( ! F . arg_empty ( ) ) KernargSegmentPtr = true ; WorkGroupIDX = true ; WorkItemIDX = true ; } else if ( CC == CallingConv :: <ISA_LIT>_PS ) { P<ISA_LIT>nputAddr = <STR_LIT> ( F ) ; } if ( ! isEntryFunction ( ) ) { ScratchRSrcReg = <STR_LIT> ; FrameOffsetReg = <STR_LIT> ; StackPtrOffsetReg = <STR_LIT> ; ArgInfo . PrivateSegmentBuffer = ArgDescriptor :: createRegister ( ScratchRSrcReg ) ; if ( F . hasFnAttribute ( <STR_LIT> ) ) ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) { KernargSegmentPtr = true ; MaxKernArgAlign = std :: max ( ST . getAlignmentForImplicitArgPtr ( ) , MaxKernArgAlign ) ; } } if ( UseFixedABI ) { WorkGroupIDX = true ; WorkGroupIDY = true ; WorkGroupIDZ = true ; WorkItemIDX = true ; WorkItemIDY = true ; WorkItemIDZ = true ; ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; } bool HasStackObjects = F . hasFnAttribute ( <STR_LIT> ) ; if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ; } bool isAmdHsaOrMesa = ST . isAmdHsaOrMesa ( F ) ; if ( isAmdHsaOrMesa ) { PrivateSegmentBuffer = true ; if ( UseFixedABI ) { DispatchPtr = true ; QueuePtr = true ; DispatchID = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } } else if ( ST . isMesaGfxShader ( F ) ) { ImplicitBufferPtr = true ; } if ( UseFixedABI || F . hasFnAttribute ( <STR_LIT> ) ) KernargSegmentPtr = true ; if ( ST . hasFlatAddressSpace ( ) && isEntryFunction ( ) && isAmdHsaOrMesa ) { if ( HasCalls || HasStackObjects ) FlatScratchInit = true ; } Attribute A = F . getFnAttribute ( <STR_LIT> ) ; StringRef S = A . getValueAsString ( ) ; if ( ! S . empty ( ) ) S . consumeInteger ( <NUM_LIT> , GITPtrHigh ) ; A = F . getFnAttribute ( <STR_LIT> ) ; S = A . getValueAsString (


<s> assert ( BufferRsrc ) ; auto PSV = BufferPSVs . try_emplace ( BufferRsrc , std :: make_unique < <ISA_LIT>BufferPseudoSourceValue > ( TII ) ) ; return PSV . first -> second .


<s> auto Arg = ArgInfo . getPreloadedValue ( Value ) . first ; return Arg ? Arg -> getRegister ( ) :


<s> YamlIO . mapOptional ( <STR_LIT> , MFI . NoSignedZerosFPMath , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . MemoryBound , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . WaveLimiter , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchRSrcReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg ,


<s> YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchRSrcReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . StackPtrOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ArgInfo ) ; YamlIO . mapOptional ( <STR_LIT> , MFI .


<s> if ( Code . startswith ( <STR_LIT> ) ) return true ; } } continue ; } const Function * Callee = dyn_cast < Function > ( CB -> getCalledOperand ( ) -> stripPointerCasts ( ) ) ; if ( ! Callee ) return true ; if ( Callee -> getIntrinsicID ( ) == <STR_LIT> ) return true ; } } return false


<s> for ( StringRef Code : CI . Codes ) { Code . consume_front ( <STR_LIT> ) ; if ( Code . startswith ( <STR_LIT> ) ) return true ; } } continue ; } const Function * Callee = dyn_cast < Function > ( CB -> getCalledOperand ( ) -> stripPointerCasts ( ) ) ; if ( ! Callee ) return true ; if ( Callee -> getIntrinsicID ( ) == <STR_LIT>


<s> void ReserveWWMRegister ( Register Reg ) { WWMReservedRegs . set ( Reg


<s> if ( CC != CallingConv :: <ISA_LIT>_Gfx ) ArgInfo = <ISA_LIT>ArgumentUsageInfo :: FixedABIFunctionInfo ; FrameOffsetReg = <STR_LIT> ; StackPtrOffsetReg = <STR_LIT> ; if ( ! ST . enableFlatScratch ( ) ) { ScratchRSrcReg = <STR_LIT> ; ArgInfo . PrivateSegmentBuffer = ArgDescriptor :: createRegister ( ScratchRSrcReg ) ; } if ( ! F . hasFnAttribute ( <STR_LIT> ) ) ImplicitArgPtr = true ; } else { ImplicitArgPtr = false ; MaxKernArgAlign = std :: max ( ST . getAlignmentForImplicitArgPtr ( ) , MaxKernArgAlign ) ; if ( ST . hasGFX90AInsts ( ) && ST . getMaxNumVGPRs ( F ) <= <STR_LIT> . getNumRegs ( ) && ! mayUseAGPRs ( MF ) ) MayNeedAGPRs = false ; } bool isAmdHsaOrMesa = ST . isAmdHsaOrMesa ( F ) ; if ( isAmdHsaOrMesa && ! ST . enableFlatScratch ( ) ) PrivateSegmentBuffer = true ; else if ( ST . isMesaGfxShader ( F ) ) ImplicitBufferPtr = true ; if ( ! <STR_LIT> ( CC ) ) { if ( IsKernel || ! F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( IsKernel || ! F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) && ST . getMaxWorkitemID ( F , <NUM_LIT> ) != <NUM_LIT> ) WorkItemIDY = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) && ST . getMaxWorkitemID ( F , <NUM_LIT> ) != <NUM_LIT> ) WorkItemIDZ = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } bool HasStackObjects = F . hasFnAttribute ( <STR_LIT> ) ; if ( ST . hasFlatAddressSpace ( ) && isEntryFunction ( ) && ( isAmdHsaOrMesa || ST . enableFlatScratch ( ) ) && ( HasCalls || HasStackObjects || ST . enableFlatScratch ( ) ) && ! ST . flatScratchIsArchitected ( ) ) { FlatScratchInit = true ; } if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; if ( ! ST .


<s> MaxKernArgAlign = assumeAligned ( YamlMFI . MaxKernArgAlign ) ; LDSSize = YamlMFI . LDSSize ; DynLDSAlign = YamlMFI . DynLDSAlign ; HighBitsOf32BitAddress = YamlMFI . HighBitsOf32BitAddress ; Occupancy = YamlMFI . Occupancy ; IsEntryFunction = YamlMFI . IsEntryFunction ; NoSignedZerosFPMath = YamlMFI . NoSignedZerosFPMath ; MemoryBound = YamlMFI .


<s> MaxKernArgAlign = assumeAligned ( YamlMFI . MaxKernArgAlign ) ; LDSSize = YamlMFI . LDSSize ; DynLDSAlign = YamlMFI . DynLDSAlign ; HighBitsOf32BitAddress = YamlMFI . HighBitsOf32BitAddress ; Occupancy = YamlMFI . Occupancy ; IsEntryFunction = YamlMFI . IsEntryFunction ; NoSignedZerosFPMath = YamlMFI . NoSignedZerosFPMath ; MemoryBound = YamlMFI . MemoryBound ; WaveLimiter = YamlMFI


<s> YamlIO . mapOptional ( <STR_LIT> , MFI . MemoryBound , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . WaveLimiter , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchRSrcReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchWaveOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . StackPtrOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI .


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register LaneVGPR = TRI -> findUnusedRegister ( MF . getRegInfo ( ) , & <STR_LIT> , MF , true ) ; if ( LaneVGPR == Register ( ) ) return false ; SpillVGPRs . push_back ( SGPRSpillVGPRCSR ( LaneVGPR , None ) ) ; FuncInfo -> VGPRReservedForSGPRSpill = LaneVGPR


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register LaneVGPR = TRI -> findUnusedRegister ( MF . getRegInfo ( ) , & <STR_LIT> , MF , true ) ; if ( LaneVGPR == Register ( ) ) return false ; SpillVGPRs . push_back ( SGPRSpillVGPRCSR ( LaneVGPR , None


<s> assert ( Reg != <NUM_LIT> && <STR_LIT>


<s> if ( F . hasFnAttribute ( <STR_LIT> ) ) ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) { KernargSegmentPtr = true ; MaxKernArgAlign = std :: max ( ST . getAlignmentForImplicitArgPtr ( ) , MaxKernArgAlign ) ; } } if ( UseFixedABI ) { WorkGroupIDX = true ; WorkGroupIDY = true ; WorkGroupIDZ = true ; WorkItemIDX = true ; WorkItemIDY = true ; WorkItemIDZ = true ; ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; } bool HasStackObjects = F . hasFnAttribute ( <STR_LIT> ) ; if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ; } bool isAmdHsaOrMesa = ST . isAmdHsaOrMesa ( F ) ; if ( isAmdHsaOrMesa ) { if ( ! ST . enableFlatScratch ( ) ) PrivateSegmentBuffer = true ; if ( UseFixedABI ) { DispatchPtr = true ; QueuePtr = true ; DispatchID = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } } else if ( ST . isMesaGfxShader ( F ) ) { ImplicitBufferPtr = true ; } if ( UseFixedABI || F . hasFnAttribute ( <STR_LIT> ) ) KernargSegmentPtr = true ; if ( ST . hasFlatAddressSpace ( ) && isEntryFunction ( ) && ( isAmdHsaOrMesa || ST . enableFlatScratch ( ) ) ) { if ( HasCalls || HasStackObjects || ST . enableFlatScratch ( )


<s> if ( FuncInfo -> VGPRReservedForSGPRSpill && NumVGPRSpillLanes < WaveSize ) { assert ( FuncInfo -> VGPRReservedForSGPRSpill == SpillVGPRs . back ( ) . VGPR ) ; LaneVGPR = FuncInfo -> VGPRReservedForSGPRSpill ; } else if ( VGPRIndex == <NUM_LIT> ) { LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> , MF ) ; if ( LaneVGPR == <STR_LIT> ) { SGPRToVGPRSpills . erase ( FI ) ; NumVGPRSpillLanes -= I ; return false ; } Optional < int > SpillFI ; if ( ! isEntryFunction ( ) ) { SpillFI = FrameInfo . CreateSpillStackObject ( <NUM_LIT> , Align ( <NUM_LIT> ) ) ; } SpillVGPRs . push_back ( SGPRSpillVGPR ( LaneVGPR , SpillFI ) ) ; for ( MachineBasicBlock & BB : MF ) BB . addLiveIn ( LaneVGPR ) ; } else { LaneVGPR = SpillVGPRs . back ( ) .


<s> ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; } bool HasStackObjects = F . hasFnAttribute ( <STR_LIT> ) ; if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; if ( ! ST . flatScratchIsArchitected ( ) ) { PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ; } } bool isAmdHsaOrMesa = ST . isAmdHsaOrMesa ( F ) ; if ( isAmdHsaOrMesa ) { if ( ! ST . enableFlatScratch ( ) ) PrivateSegmentBuffer = true ; if ( UseFixedABI ) { DispatchPtr = true ; QueuePtr = true ; DispatchID = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } } else if ( ST . isMesaGfxShader ( F ) ) { ImplicitBufferPtr = true ; } if ( UseFixedABI || F . hasFnAttribute ( <STR_LIT> ) ) KernargSegmentPtr = true ; if ( ST . hasFlatAddressSpace ( ) && isEntryFunction ( ) && ( isAmdHsaOrMesa || ST . enableFlatScratch ( ) ) && ! ST . flatScratchIsArchitected ( ) ) { if ( HasCalls || HasStackObjects || ST . enableFlatScratch ( ) ) FlatScratchInit = true ; } Attribute A = F . getFnAttribute ( <STR_LIT> ) ; StringRef S = A . getValueAsString ( ) ; if ( ! S . empty ( ) ) S .

consumeInteger ( <NUM_LIT> , GITPtrHigh ) ;


<s> const bool HasCalls = F . hasFnAttribute ( <STR_LIT> ) ; const bool UseFixedABI = <ISA_LIT>TargetMachine :: EnableFixedFunctionABI && CC != CallingConv :: <ISA_LIT>_Gfx && ( ! isEntryFunction ( ) || HasCalls ) ; if ( CC == CallingConv :: <ISA_LIT>_KERNEL || CC == CallingConv :: SPIR_KERNEL ) { if ( ! F . arg_empty ( ) ) KernargSegmentPtr = true ; WorkGroupIDX = true ; WorkItemIDX = true ; } else if ( CC == CallingConv :: <ISA_LIT>_PS ) { P<ISA_LIT>nputAddr = <STR_LIT> ( F ) ; } if ( ! isEntryFunction ( ) ) { if ( UseFixedABI ) ArgInfo = <ISA_LIT>ArgumentUsageInfo :: FixedABIFunctionInfo ; FrameOffsetReg = <STR_LIT> ; StackPtrOffsetReg = <STR_LIT> ; if ( ! ST . enableFlatScratch ( ) ) { ScratchRSrcReg = <STR_LIT> ; ArgInfo . PrivateSegmentBuffer = ArgDescriptor :: createRegister ( ScratchRSrcReg ) ; } if ( F . hasFnAttribute ( <STR_LIT> ) ) ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) { KernargSegmentPtr = true ; MaxKernArgAlign = std :: max ( ST . getAlignmentForImplicitArgPtr ( ) , MaxKernArgAlign ) ; } } if ( UseFixedABI ) { WorkGroupIDX = true ; WorkGroupIDY = true ; WorkGroupIDZ = true ; WorkItemIDX = true ; WorkItemIDY = true ; WorkItemIDZ = true ; ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ =


<s> for ( auto & R : SGPRToVGPRSpills ) MFI . RemoveStackObject ( R . first ) ; for ( unsigned i = MFI . getObjectIndexBegin ( )


<s> for ( unsigned i = MFI . getObjectIndexBegin ( ) , e = MFI . getObjectIndexEnd ( ) ; i != e ; ++ i ) MFI . setStackID ( i , <NUM_LIT>


<s> const MCPhysReg * CSRegs = MRI . getCalleeSavedRegs ( ) ; for ( unsigned I = <NUM_LIT> ; I < NumLanes ; ++ I , ++ NumVGPRSpillLanes ) { Register LaneVGPR ; unsigned VGPRIndex = ( NumVGPRSpillLanes % WaveSize ) ; if ( VGPRIndex == <NUM_LIT> && ! FuncInfo -> VGPRReservedForSGPRSpill ) { LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> , MF ) ; if ( LaneVGPR == <STR_LIT> ) { SGPRToVGPRSpills . erase ( FI ) ; NumVGPRSpillLanes -= I ; return false ; } Optional < int > CSRSpillFI ; if ( ( FrameInfo . hasCalls ( ) || ! isEntryFunction ( ) ) && CSRegs && isCalleeSavedReg ( CSRegs , LaneVGPR ) ) { CSRSpillFI = FrameInfo . CreateSpillStackObject ( <NUM_LIT> , Align ( <NUM_LIT> ) ) ; } SpillVGPRs . push_back ( SGPRSpillVGPRCSR ( LaneVGPR


<s> Register LaneVGPR = TRI -> findUnusedRegister ( MF . getRegInfo ( ) , & <STR_LIT> , MF , true ) ; SpillVGPRs . push_back ( SGPRSpillVGPRCSR ( LaneVGPR , None ) ) ; FuncInfo -> VGPRReservedForSGPRSpill = LaneVGPR ; return true


<s> FlatWorkGroupSizes = ST . getFlatWorkGroupSizes ( F ) ; WavesPerEU = ST . getWavesPerEU ( F ) ; Occupancy = ST . computeOccupancy ( F , getLDSSize ( ) ) ; CallingConv :: ID CC = F . getCallingConv ( ) ; const bool HasCalls = F . hasFnAttribute ( <STR_LIT> ) ; const bool UseFixedABI = <ISA_LIT>TargetMachine :: EnableFixedFunctionABI && ( ! isEntryFunction ( ) || HasCalls ) ; if ( CC == CallingConv :: <ISA_LIT>_KERNEL || CC == CallingConv :: SPIR_KERNEL ) { if ( ! F . arg_empty ( ) ) KernargSegmentPtr = true ; WorkGroupIDX = true ; WorkItemIDX = true ; } else if ( CC == CallingConv :: <ISA_LIT>_PS ) { P<ISA_LIT>nputAddr = <STR_LIT> ( F ) ; } if ( ! isEntryFunction ( ) ) { ScratchRSrcReg = <STR_LIT> ; FrameOffsetReg = <STR_LIT> ; StackPtrOffsetReg = <STR_LIT> ; ArgInfo . PrivateSegmentBuffer = ArgDescriptor :: createRegister ( ScratchRSrcReg ) ; if ( F . hasFnAttribute ( <STR_LIT> ) ) ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) { KernargSegmentPtr = true ; MaxKernArgAlign = std :: max ( ST . getAlignmentForImplicitArgPtr ( ) , MaxKernArgAlign ) ; } } if ( UseFixedABI ) { WorkGroupIDX = true ; WorkGroupIDY = true ; WorkGroupIDZ = true ; WorkItemIDX = true ; WorkItemIDY = true ; WorkItemIDZ = true ; ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; } bool HasStackObjects = F . hasFnAttribute ( <STR_LIT>


<s> const Function & F = MF . getFunction ( ) ; FlatWorkGroupSizes = ST . getFlatWorkGroupSizes ( F ) ; WavesPerEU = ST . getWavesPerEU ( F ) ; Occupancy = getMaxWavesPerEU ( ) ; limitOccupancy ( MF ) ; CallingConv :: ID CC = F . getCallingConv ( ) ; if ( CC == CallingConv :: <ISA_LIT>_KERNEL || CC == CallingConv :: SPIR_KERNEL ) { if ( ! F . arg_empty ( ) ) KernargSegmentPtr = true ; WorkGroupIDX = true ; WorkItemIDX = true ; } else if ( CC == CallingConv :: <ISA_LIT>_PS ) { P<ISA_LIT>nputAddr = <STR_LIT> ( F ) ; } if ( ! isEntryFunction ( ) ) { ScratchRSrcReg = <STR_LIT> ; ScratchWaveOffsetReg = <STR_LIT> ; FrameOffsetReg = <STR_LIT> ; StackPtrOffsetReg = <STR_LIT> ; ArgInfo . PrivateSegmentBuffer = ArgDescriptor :: createRegister ( ScratchRSrcReg ) ; ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( ScratchWaveOffsetReg ) ; if ( F . hasFnAttribute ( <STR_LIT> ) ) ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) { KernargSegmentPtr = true ; MaxKernArgAlign = std :: max ( ST . getAlignmentForImplicitArgPtr ( ) , MaxKernArgAlign ) ; } } if ( ST . debuggerEmitPrologue ( ) ) { WorkGroupIDX = true ; WorkGroupIDY = true ; WorkGroupIDZ = true ; WorkItemIDX = true ; WorkItemIDY = true ; WorkItemIDZ = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; } const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; bool MaySpill = ST . isVGPRSpillingEnabled ( F ) ; bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; if ( HasStackObjects || MaySpill ) { PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ; } } bool IsCOV2 = ST . isAmdCodeObjectV2 ( F ) ; if ( IsCOV2 ) { if ( HasStackObjects || MaySpill ) PrivateSegmentBuffer = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } else if ( ST . isMesaGfxShader ( F ) ) { if ( HasStackObjects || MaySpill ) ImplicitBufferPtr


<s> } } if ( ST . debuggerEmitPrologue ( ) ) { WorkGroupIDX = true ; WorkGroupIDY = true ; WorkGroupIDZ = true ; WorkItemIDX = true ; WorkItemIDY = true ; WorkItemIDZ = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; } const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; bool MaySpill = ST . isVGPRSpillingEnabled ( F ) ; bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; if ( HasStackObjects || MaySpill ) { PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ; } } bool IsCOV2 = ST . isAmdCodeObjectV2 ( F ) ; if ( IsCOV2 )


<s> auto Arg = ArgInfo . getPreloadedValue ( Value ) . first ; return Arg ? Arg -> getRegister ( )


<s> YamlIO . mapOptional ( <STR_LIT> , MFI . WaveLimiter , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchRSrcReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchWaveOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . StackPtrOffsetReg


<s> const Function * F = MF . getFunction ( ) ; P<ISA_LIT>nputAddr = <STR_LIT> ( * F ) ; const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; if ( ! <STR_LIT> ( F -> getCallingConv ( ) ) ) { KernargSegmentPtr = true ; WorkGroupIDX = true ; WorkItemIDX = true ; } if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkGroupIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkGroupIDZ = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkItemIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkItemIDZ = true ; if ( WorkItemIDZ ) WorkItemIDY = true ; bool MaySpill = ST . isVGPRSpillingEnabled ( * F ) ; bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( HasStackObjects || MaySpill ) PrivateSegmentWaveByteOffset = true ; if ( ST . isAmdCodeObjectV2 ( MF ) ) { if ( HasStackObjects || MaySpill ) PrivateSegmentBuffer = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } else if ( ST . isMesaGfxShader ( MF ) ) { if ( HasStackObjects || MaySpill


<s> HighBitsOf32BitAddress = YamlMFI . HighBitsOf32BitAddress ; IsEntryFunction = YamlMFI . IsEntryFunction ; NoSignedZerosFPMath = YamlMFI . NoSignedZerosFPMath ; MemoryBound = YamlMFI . MemoryBound ; WaveLimiter = YamlMFI . WaveLimiter ; HasSpilledSGPRs = YamlMFI . HasSpilledSGPRs ; HasSpilledVGPRs = YamlMFI . HasSpilledVGPRs


<s> void <ISA_LIT>MachineFunctionInfo :: allocateWWMReservedSpillSlots ( MachineFrameInfo & MFI , const <ISA_LIT>RegisterInfo & TRI ) { assert ( WWMReservedFrameIndexes . empty ( ) ) ; WWMReservedFrameIndexes . resize ( WWMReservedRegs . size ( ) ) ; int I = <NUM_LIT> ; for ( Register VGPR : WWMReservedRegs ) { const TargetRegisterClass * RC = TRI . getPhysRegClass ( VGPR ) ; WWMReservedFrameIndexes [ I ++ ] = MFI . CreateSpillStackObject ( TRI . getSpillSize ( * RC ) , TRI . getSpillAlign


<s> LDSSize = YamlMFI . LDSSize ; GDSSize = YamlMFI . GDSSize ; DynLDSAlign = YamlMFI . DynLDSAlign ; HighBitsOf32BitAddress = YamlMFI . HighBitsOf32BitAddress ; Occupancy = YamlMFI . Occupancy ; IsEntryFunction = YamlMFI . IsEntryFunction ; NoSignedZerosFPMath = YamlMFI . NoSignedZerosFPMath ; MemoryBound = YamlMFI . MemoryBound ; WaveLimiter = YamlMFI . WaveLimiter ; HasSpilledSGPRs = YamlMFI . HasSpilledSGPRs ; HasSpilledVGPRs = YamlMFI . HasSpilledVGPRs ; BytesInStackArgArea = YamlMFI . BytesInStackArgArea ; ReturnsVoid = YamlMFI . ReturnsVoid ; if ( YamlMFI . ScavengeFI ) { auto FIOrErr = YamlMFI . ScavengeFI -> getFI ( MF . getFrameInfo ( ) ) ; if ( ! FIOrErr ) { const MemoryBuffer & Buffer = * PFS . SM -> getMemoryBuffer ( PFS . SM -> getMainFileID ( ) ) ; Error = SMDiagnostic ( * PFS . SM , SMLoc ( ) , Buffer . getBufferIdentifier ( ) , <NUM_LIT> , <NUM_LIT> , SourceMgr :: DK_Error , toString ( FIOrErr . takeError ( ) ) , <STR_LIT> , None , None ) ; SourceRange = YamlMFI . ScavengeFI -> SourceRange ; return true ; } ScavengeFI = * FIOrErr ; } else


<s> auto FIOrErr = YamlMFI . ScavengeFI -> getFI ( MF . getFrameInfo ( ) ) ; if ( ! FIOrErr ) { const MemoryBuffer & Buffer = * PFS . SM -> getMemoryBuffer ( PFS . SM -> getMainFileID ( ) ) ; Error = SMDiagnostic ( * PFS . SM , SMLoc ( ) , Buffer . getBufferIdentifier ( ) , <NUM_LIT> , <NUM_LIT> , SourceMgr :: DK_Error , toString ( FIOrErr . takeError ( ) ) , <STR_LIT> , None , None ) ; SourceRange = YamlMFI . ScavengeFI -> SourceRange ; return true ; } ScavengeFI = *


<s> YamlIO . mapOptional ( <STR_LIT> , Mode . IEEE , true ) ; YamlIO . mapOptional ( <STR_LIT> , Mode . DX10Clamp , true ) ; YamlIO . mapOptional ( <STR_LIT> , Mode . FP32Denormals , true ) ; YamlIO . mapOptional ( <STR_LIT> , Mode . FP64FP16Denormals ,


<s> HighBitsOf32BitAddress = YamlMFI . HighBitsOf32BitAddress ; IsEntryFunction = YamlMFI . IsEntryFunction ; NoSignedZerosFPMath = YamlMFI . NoSignedZerosFPMath ; MemoryBound = YamlMFI . MemoryBound ; WaveLimiter = YamlMFI .


<s> std :: vector < SpilledReg > & SpillLanes = SGPRToVGPRSpills [ FI ] ; if ( ! SpillLanes . empty ( ) ) return true ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; unsigned WaveSize = ST . getWavefrontSize ( ) ; unsigned Size = FrameInfo . getObjectSize ( FI ) ; assert ( Size >= <NUM_LIT> && Size <= <NUM_LIT> && <STR_LIT> ) ; assert ( TRI -> spillSGPRToVGPR ( ) && <STR_LIT> ) ; int NumLanes = Size / <NUM_LIT> ; const MCPhysReg * CSRegs = MRI .