<s> unsigned Opc = MI . getOpcode ( ) ; if ( TII -> isSDWA ( Opc ) ) return true ; if ( <STR_LIT> ( Opc ) == - <NUM_LIT> ) Opc = <STR_LIT> ( Opc ) ; if ( <STR_LIT> ( Opc ) == - <NUM_LIT> ) return false ; if ( ! ST . hasSDWAOmod ( ) && TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return false ; if ( TII -> isVOPC ( Opc ) ) { if ( ! ST . hasSDWASdst ( ) ) { const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; if ( SDst && SDst -> getReg ( ) != <STR_LIT> ) return false ;


<s> if ( <STR_LIT> ( Opc ) != - <NUM_LIT> ) return true ; int Opc32 = <STR_LIT> ( Opc ) ; if ( Opc32 != - <NUM_LIT> && <STR_LIT> ( Opc32 ) != - <NUM_LIT> ) return ! TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) && ! TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; return false ;


<s> MachineOperand & Op = MI . getOperand ( I ) ; if ( ! Op . isImm ( ) && ! ( Op . isReg ( ) && ! TRI -> isVGPR ( * MRI , Op . getReg ( ) ) ) ) continue ; if ( Desc . OpInfo [ I ] . RegClass == - <NUM_LIT> || ! TRI -> hasVGPRs ( TRI -> getRegClass ( Desc . OpInfo [ I ] . RegClass ) ) ) continue ; unsigned VGPR = MRI -> createVirtualRegister ( & <STR_LIT> ) ; auto Copy = BuildMI ( * MI . getParent ( ) , MI . getIterator ( ) , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , VGPR ) ; if ( Op . isImm ( ) ) Copy . addImm ( Op . getImm ( ) ) ;

else if ( Op . isReg ( ) ) Copy . addReg ( Op . getReg ( ) , Op . isKill ( ) ? RegState :: Kill : <NUM_LIT> , Op . getSubReg ( ) ) ;


<s> bool <ISA_LIT>PeepholeSDWA :: runOnMachineFunction ( MachineFunction & MF ) { const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( ! ST . hasSDWA ( ) || ! <STR_LIT> ( ST ) ) { return false ; } MRI = & MF . getRegInfo ( ) ; TRI = ST . getRegisterInfo ( ) ; TII = ST . getInstrInfo ( ) ;


<s> if ( ! NextOp ) return ; MachineInstr & MISucc = * NextOp -> getParent ( ) ; if ( ! TII -> canShrink ( MISucc , * MRI ) ) return ; int SuccOpc = <STR_LIT> ( MISucc . getOpcode ( ) ) ; MachineOperand * CarryIn = TII -> getNamedOperand ( MISucc , <STR_LIT> :: src2 ) ; if ( ! CarryIn ) return ; MachineOperand * CarryOut = TII -> getNamedOperand ( MISucc , <STR_LIT> :: sdst ) ; if ( ! CarryOut ) return ; if ( ! MRI -> hasOneUse ( CarryIn -> getReg ( ) ) || ! MRI -> use_empty ( CarryOut -> getReg ( ) ) ) return ; MachineBasicBlock & MBB = * MI . getParent ( ) ; auto Liveness = MBB . computeRegisterLiveness ( TRI , <STR_LIT> , MI , <NUM_LIT> ) ; if ( Liveness != MachineBasicBlock :: LQR_Dead ) return ; for ( auto I = std :: next ( MI . getIterator ( ) ) , E = MISucc . getIterator ( ) ; I != E ; ++ I ) { if ( I -> modifiesRegister ( <STR_LIT> , TRI ) ) return ; } BuildMI ( MBB , MI , MI . getDebugLoc ( ) , TII -> get ( Opc ) ) . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ) . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ) . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ) . setMIFlags ( MI . getFlags ( ) ) ; MI . eraseFromParent ( ) ; BuildMI ( MBB , MISucc , MISucc . getDebugLoc ( ) , TII -> get ( SuccOpc ) ) . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: vdst ) ) . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: src0 ) ) . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: src1 ) ) . setMIFlags ( MISucc . getFlags ( ) ) ;


<s> bool <ISA_LIT>PeepholeSDWA :: isConvertibleToSDWA ( const MachineInstr & MI , const GCNSubtarget & ST ) const { unsigned Opc = MI . getOpcode ( ) ; if ( TII -> isSDWA ( Opc ) ) return true ; if ( <STR_LIT> ( Opc ) == - <NUM_LIT> ) Opc = <STR_LIT> ( Opc ) ; if ( <STR_LIT> ( Opc ) == - <NUM_LIT> ) return false ; if ( ! ST . hasSDWAOmod ( ) && TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return false ; if ( TII -> isVOPC ( Opc ) ) { if ( ! ST . hasSDWASdst ( ) ) { const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; if ( SDst && SDst -> getReg ( ) != <STR_LIT> ) return false ; } if ( ! ST . hasSDWAOutModsVOPC ( ) && ( TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) ) return false ; } else if ( TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) || ! TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ) { return false ; }

if ( ! ST . hasSDWAMac ( ) && ( Opc == <STR_LIT> || Opc == <STR_LIT> ) ) return false ;


<s> matchSDWAOperands ( MBB ) ; for ( const auto & OperandPair : SDWAOperands ) { const auto & Operand = OperandPair . second ; MachineInstr * PotentialMI = Operand -> potentialToConvert ( TII ) ; if ( PotentialMI && isConvertibleToSDWA ( * PotentialMI , ST ) ) { PotentialMatches [ PotentialMI ] . push_back ( Operand . get ( ) ) ; } } for ( auto & PotentialPair : PotentialMatches ) { MachineInstr & PotentialMI = * PotentialPair . first ; convertToSDWA ( PotentialMI , PotentialPair . second ) ; } PotentialMatches . clear ( ) ; SDWAOperands . clear ( ) ;

Changed = ! ConvertedInstructions . empty ( ) ;


<s> for ( MachineOperand & Op : MI . explicit_uses ( ) ) { if ( ! Op . isImm ( ) && ! ( Op . isReg ( ) && ! TRI -> isVGPR ( * MRI , Op . getReg ( ) ) ) ) continue ; unsigned I = MI . getOperandNo ( & Op ) ; if ( Desc . OpInfo [ I ] . RegClass == - <NUM_LIT> || ! TRI -> isVGPRClass ( TRI -> getRegClass ( Desc . OpInfo [ I ] . RegClass ) ) ) continue ; if ( ST . hasSDWAScalar ( ) && ConstantBusCount == <NUM_LIT> && Op . isReg ( ) && TRI -> isSGPRReg ( * MRI , Op . getReg ( ) ) ) { ++ ConstantBusCount ; continue ; } Register VGPR = MRI -> createVirtualRegister ( & <STR_LIT> ) ; auto Copy = BuildMI ( * MI . getParent ( ) , MI . getIterator ( ) , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , VGPR ) ; if ( Op . isImm ( ) ) Copy . addImm ( Op . getImm ( ) ) ;


<s> if ( ! ST . hasSDWASdst ( ) ) { const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; if ( SDst && SDst -> getReg ( ) != <STR_LIT> ) return false ; } if ( ! ST . hasSDWAOutModsVOPC ( ) && ( TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) ) return false ; } else if ( TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) || ! TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ) { return false ; } if ( ! ST . hasSDWAMac ( ) && ( Opc == <STR_LIT> || Opc == <STR_LIT> ) ) return false ; if ( Opc == <STR_LIT> ) return false ; return true ;


<s> if ( ! ST . hasSDWAOmod ( ) && TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return false ; if ( TII -> isVOPC ( Opc ) ) { if ( ! ST . hasSDWASdst ( ) ) { const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; if ( SDst && SDst -> getReg ( ) != <STR_LIT> ) return false ; } if ( ! ST . hasSDWAOutModsVOPC ( ) && ( TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) ) return false ; } else if ( TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) || ! TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ) { return false ; } if ( ! ST . hasSDWAMac ( ) && ( Opc == <STR_LIT> || Opc == <STR_LIT> ) ) return false ; if ( Opc == <STR_LIT> ) return false ;


<s> if ( DstSel ) { SDWAInst . add ( * DstSel ) ; } else { SDWAInst . addImm ( <STR_LIT> :: SdwaSel :: DWORD ) ; } } if ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: dst_unused ) != - <NUM_LIT> ) { MachineOperand * DstUnused = TII -> getNamedOperand ( MI , <STR_LIT> :: dst_unused ) ; if ( DstUnused ) { SDWAInst . add ( * DstUnused ) ; } else { SDWAInst . addImm ( <STR_LIT> :: DstUnused :: UNUSED_PAD ) ; } } assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0_sel ) != - <NUM_LIT> ) ; MachineOperand * Src0Sel = TII -> getNamedOperand ( MI , <STR_LIT> :: src0_sel ) ; if ( Src0Sel ) { SDWAInst . add ( * Src0Sel ) ; } else { SDWAInst . addImm ( <STR_LIT> :: SdwaSel :: DWORD ) ; } if ( Src1 ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1_sel ) != - <NUM_LIT> ) ; MachineOperand * Src1Sel = TII -> getNamedOperand ( MI , <STR_LIT> :: src1_sel ) ; if ( Src1Sel ) { SDWAInst . add ( * Src1Sel ) ; } else { SDWAInst . addImm ( <STR_LIT> :: SdwaSel :: DWORD ) ; } } auto DstUnused = TII -> getNamedOperand ( MI , <STR_LIT> :: dst_unused ) ; if ( DstUnused && DstUnused -> getImm ( ) == <STR_LIT> :: DstUnused :: UNUSED_PRESERVE ) { assert ( Dst && Dst -> isTied ( ) ) ; assert ( Opcode == static_cast < unsigned int > ( SDWAOpcode ) ) ; auto PreserveDstIdx = <STR_LIT> ( SDWAOpcode , <STR_LIT> :: vdst ) ; assert ( PreserveDstIdx != - <NUM_LIT> ) ; auto TiedIdx = MI . findTiedOperandIdx ( PreserveDstIdx ) ; auto Tied = MI . getOperand ( TiedIdx ) ; SDWAInst . add ( Tied ) ; SDWAInst -> tieOperands ( PreserveDstIdx , SDWAInst -> getNumOperands ( ) - <NUM_LIT> ) ; } bool Converted = false ; for ( auto & Operand : SDWAOperands ) { LLVM_DEBUG ( dbgs ( ) << * SDWAInst << <STR_LIT> << * Operand ) ; if ( PotentialMatches . count ( Operand -> getParentInst ( ) ) == <NUM_LIT> ) Converted |= Operand -> convertToSDWA ( * SDWAInst , TII ) ; } if ( Converted ) { ConvertedInstructions . push_back ( SDWAInst ) ;


<s> To . setSubReg ( From . getSubReg ( ) ) ; To . setIsUndef ( From . isUndef ( ) ) ; if ( To . isUse ( ) ) { To . setIsKill ( From . isKill ( ) ) ; } else { To . setIsDead ( From . isDead ( ) ) ;


<s> To . setSubReg ( From . getSubReg ( ) ) ; To . setIsUndef ( From . isUndef ( ) ) ; if ( To . isUse ( ) ) { To . setIsKill ( From . isKill ( ) ) ; } else {


<s> FunctionPass * llvm :: create<ISA_LIT>PeepholeSDWAPass ( ) { return new <ISA_LIT>PeepholeSDWA ( ) ;


<s> if ( ! DefInstr ) return nullptr ; for ( auto & DefMO : DefInstr -> defs ( ) ) { if ( DefMO . isReg ( ) && DefMO . getReg ( ) == Reg -> getReg ( ) ) return & DefMO ; }


<s> if ( ! Reg -> isReg ( ) || ! Reg -> isDef ( ) ) return nullptr ; MachineOperand * ResMO = nullptr ; for ( MachineOperand & UseMO : MRI -> use_nodbg_operands ( Reg -> getReg ( ) ) ) { if ( ! isSameReg ( UseMO , * Reg ) ) return nullptr ; if ( ! ResMO ) {


<s> } if ( Op . isReg ( ) ) { for ( const MachineOperand & Def : MRI -> def_operands ( Op . getReg ( ) ) ) { if ( ! isSameReg ( Op , Def ) ) continue ; const MachineInstr * DefInst = Def . getParent ( ) ; if ( ! TII -> isFoldableCopy ( * DefInst ) ) return None ; const MachineOperand & Copied = DefInst -> getOperand ( <NUM_LIT> ) ; if ( ! Copied . isImm ( ) ) return None ; return Copied . getImm ( ) ; } } return None ;


<s> SdwaSel getDstSel ( ) const { return DstSel ;


<s> MachineRegisterInfo * getMRI ( ) const { return & getParentInst ( ) -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ;


<s> return & getParentInst ( ) -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ;


<s> bool getNeg ( ) const { return Neg ;


<s> MachineOperand * getPreservedOperand ( ) const {


<s> MachineOperand * getReplacedOperand ( ) const { return Replaced ;


<s> MachineOperand * getReplacedOperand ( ) const {


<s> uint64_t Mods = <NUM_LIT> ; const auto * MI = SrcOp -> getParent ( ) ; if ( TII -> getNamedOperand ( * MI , <STR_LIT> :: src0 ) == SrcOp ) { if ( auto * Mod = TII -> getNamedOperand ( * MI , <STR_LIT> :: src0_modifiers ) ) { Mods = Mod -> getImm ( ) ; } } else if ( TII -> getNamedOperand ( * MI , <STR_LIT> :: src1 ) == SrcOp ) { if ( auto * Mod = TII -> getNamedOperand ( * MI , <STR_LIT> :: src1_modifiers ) ) { Mods = Mod -> getImm ( ) ; } } if ( Abs || Neg ) { assert ( ! Sext && <STR_LIT> ) ; Mods |= Abs ? <STR_LIT> : <NUM_LIT> ;


<s> MachineOperand * getTargetOperand ( ) const {


<s> bool <ISA_LIT>PeepholeSDWA :: isConvertibleToSDWA ( MachineInstr & MI , const GCNSubtarget & ST ) const { unsigned Opc = MI . getOpcode ( ) ; if ( TII -> isSDWA ( Opc ) ) return true ; if ( <STR_LIT> ( Opc ) == - <NUM_LIT> ) Opc = <STR_LIT> ( Opc ) ; if ( <STR_LIT> ( Opc ) == - <NUM_LIT> ) return false ; if ( ! ST . hasSDWAOmod ( ) && TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return false ; if ( TII -> isVOPC ( Opc ) ) { if ( ! ST . hasSDWASdst ( ) ) { const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; if ( SDst && ( SDst -> getReg ( ) != <STR_LIT> && SDst -> getReg ( ) != <STR_LIT> ) ) return false ; } if ( ! ST . hasSDWAOutModsVOPC ( ) && ( TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) ) return false ; } else if ( TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) || ! TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ) { return false ; } if ( ! ST . hasSDWAMac ( ) && ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) ) return false ; if ( TII -> pseudoToMCOpcode ( Opc ) == - <NUM_LIT> ) return false ;


<s> if ( ! Op . isImm ( ) && ! ( Op . isReg ( ) && ! TRI -> isVGPR ( * MRI , Op . getReg ( ) ) ) ) continue ; unsigned I = MI . getOperandNo ( & Op ) ; if ( Desc . OpInfo [ I ] . RegClass == - <NUM_LIT> || ! TRI -> isVSSuperClass ( TRI -> getRegClass ( Desc . OpInfo [ I ] . RegClass ) ) ) continue ; if ( ST . hasSDWAScalar ( ) && ConstantBusCount == <NUM_LIT> && Op . isReg ( ) && TRI -> isSGPRReg ( * MRI , Op . getReg ( ) ) ) { ++ ConstantBusCount ; continue ; } Register VGPR = MRI -> createVirtualRegister ( & <STR_LIT> ) ;

auto Copy = BuildMI ( * MI . getParent ( ) , MI . getIterator ( ) , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , VGPR ) ;


<s> if ( ! Op . isImm ( ) && ! ( Op . isReg ( ) && ! TRI -> isVGPR ( * MRI , Op . getReg ( ) ) ) ) continue ; unsigned I = MI . getOperandNo ( & Op ) ; if ( Desc . OpInfo [ I ] . RegClass == - <NUM_LIT> || ! TRI -> isVSSuperClass ( TRI -> getRegClass ( Desc . OpInfo [ I ] . RegClass ) ) ) continue ;

if ( ST . hasSDWAScalar ( ) && ConstantBusCount == <NUM_LIT> && Op . isReg ( ) && TRI -> isSGPRReg ( * MRI , Op . getReg ( ) ) ) {


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * Operand << '\n' ) ;

SDWAOperands [ & MI ] = std :: move ( Operand ) ;


<s> MachineOperand * PotentialMO = findSingleRegDef ( getReplacedOperand ( ) , MRI ) ; if ( ! PotentialMO ) return nullptr ; for ( MachineInstr & UseInst : MRI -> use_nodbg_instructions ( PotentialMO -> getReg ( ) ) ) { if ( & UseInst != ParentMI ) return nullptr ; } return PotentialMO -> getParent ( ) ;


<s> OS << <STR_LIT> << * getTargetOperand ( ) << <STR_LIT> << getDstSel ( ) << <STR_LIT> << * getPreservedOperand ( ) << '\n' ;


<s> if ( Liveness != MachineBasicBlock :: LQR_Dead ) return ; for ( auto I = std :: next ( MI . getIterator ( ) ) , E = MISucc . getIterator ( ) ; I != E ; ++ I ) { if ( I -> modifiesRegister ( <STR_LIT> , TRI ) ) return ; } BuildMI ( MBB , MI , MI . getDebugLoc ( ) , TII -> get ( Opc ) ) . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ) . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ) . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ) . setMIFlags ( MI . getFlags ( ) ) ; MI . eraseFromParent ( ) ;

BuildMI ( MBB , MISucc , MISucc . getDebugLoc ( ) , TII -> get ( SuccOpc ) ) . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: vdst ) ) . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: src0 ) ) . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: src1 ) ) . setMIFlags ( MISucc . getFlags ( ) ) ;


<s> if ( Liveness != MachineBasicBlock :: LQR_Dead ) return ; for ( auto I = std :: next ( MI . getIterator ( ) ) , E = MISucc . getIterator ( ) ; I != E ; ++ I ) { if ( I -> modifiesRegister ( <STR_LIT> , TRI ) ) return ; } BuildMI ( MBB , MI , MI . getDebugLoc ( ) , TII -> get ( Opc ) ) . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ) . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ) . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ) . setMIFlags ( MI . getFlags ( ) ) ; MI . eraseFromParent ( ) ; BuildMI ( MBB , MISucc , MISucc . getDebugLoc ( ) , TII -> get ( SuccOpc ) ) . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: vdst ) ) . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: src0 ) ) . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: src1 ) ) . setMIFlags ( MISucc . getFlags ( ) ) ;


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; if ( ! ST . hasSDWA ( ) || skipFunction ( MF . getFunction ( ) ) ) return false ; MRI = & MF . getRegInfo ( ) ; TRI = ST . getRegisterInfo ( ) ; TII = ST . getInstrInfo ( ) ; bool Ret = false ; for ( MachineBasicBlock & MBB : MF ) { bool Changed = false ; do { matchSDWAOperands ( MBB ) ; for ( const auto & OperandPair : SDWAOperands ) { const auto & Operand = OperandPair . second ; MachineInstr * PotentialMI = Operand -> potentialToConvert ( TII ) ; if ( PotentialMI && ( PotentialMI -> getOpcode ( ) == <STR_LIT> || PotentialMI -> getOpcode ( ) == <STR_LIT> ) ) pseudoOpConvertToVOP2 ( * PotentialMI , ST ) ; } SDWAOperands . clear ( ) ; matchSDWAOperands ( MBB ) ; for ( const auto & OperandPair : SDWAOperands ) { const auto & Operand = OperandPair . second ; MachineInstr * PotentialMI = Operand -> potentialToConvert ( TII ) ; if ( PotentialMI && isConvertibleToSDWA ( * PotentialMI , ST ) ) { PotentialMatches [ PotentialMI ] . push_back ( Operand . get ( ) ) ; } } for ( auto & PotentialPair : PotentialMatches ) { MachineInstr & PotentialMI = * PotentialPair . first ; convertToSDWA ( PotentialMI , PotentialPair . second ) ; } PotentialMatches . clear ( ) ;


<s> SDWADstOperand ( MachineOperand * TargetOp , MachineOperand * ReplacedOp , SdwaSel DstSel_ = DWORD , DstUnused DstUn_ = UNUSED_PAD ) : SDWAOperand ( TargetOp , ReplacedOp ) , DstSel ( DstSel_ ) , DstUn ( DstUn_ ) {


<s> SDWADstPreserveOperand ( MachineOperand * TargetOp , MachineOperand * ReplacedOp , MachineOperand * PreserveOp , SdwaSel DstSel_ = DWORD ) : SDWADstOperand ( TargetOp , ReplacedOp , DstSel_ , UNUSED_PRESERVE ) , Preserve ( PreserveOp ) {


<s> SDWAOperand ( MachineOperand * TargetOp , MachineOperand * ReplacedOp ) : Target ( TargetOp ) , Replaced ( ReplacedOp ) {


<s> SDWAOperand ( MachineOperand * TargetOp , MachineOperand * ReplacedOp ) : Target ( TargetOp ) , Replaced ( ReplacedOp ) { assert ( Target -> isReg ( ) ) ; assert ( Replaced -> isReg ( ) ) ;


<s> SDWASrcOperand ( MachineOperand * TargetOp , MachineOperand * ReplacedOp , SdwaSel SrcSel_ = DWORD , bool Abs_ = false , bool Neg_ = false , bool Sext_ = false ) : SDWAOperand ( TargetOp , ReplacedOp ) , SrcSel ( SrcSel_ ) , Abs ( Abs_ ) , Neg ( Neg_ ) , Sext ( Sext_ ) {


<s> initialize<ISA_LIT>PeepholeSDWAPass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> return ( IMemFlags != <NUM_LIT> && MI . mayLoadOrStore ( ) && ! NextMI . isBundled ( ) && NextMI . mayLoad ( ) == MI . mayLoad ( ) && NextMI . mayStore ( ) == MI . mayStore ( ) && ( ( NextMI . getDesc ( ) . TSFlags & MemFlags ) == IMemFlags ) && ! isDependentLoad ( NextMI ) ) ;


<s> const uint64_t IMemFlags = MI . getDesc ( ) . TSFlags & MemFlags ; return ( IMemFlags != <NUM_LIT> && MI . mayLoadOrStore ( ) && ! NextMI . isBundled ( ) && NextMI . mayLoad ( ) == MI . mayLoad ( ) && NextMI . mayStore ( ) == MI . mayStore ( ) && ( ( NextMI . getDesc ( ) . TSFlags & MemFlags ) == IMemFlags ) && ! isDependentLoad ( NextMI ) ) ;


<s> assert ( ! Op . getSubReg ( ) && <STR_LIT> ) ; for ( MCRegUnitIterator Units ( Reg , TRI ) ; Units . isValid ( ) ; ++ Units ) UsedRegUnits . set ( * Units ) ;


<s> Register Reg = Op . getReg ( ) ; assert ( ! Op . getSubReg ( ) && <STR_LIT> ) ;

for ( MCRegUnitIterator Units ( Reg , TRI ) ; Units . isValid ( ) ; ++ Units ) UsedRegUnits . set ( * Units ) ;


<s> bool <ISA_LIT>PostRABundler :: isBundleCandidate ( const MachineInstr & MI ) const { const uint64_t IMemFlags = MI . getDesc ( ) . TSFlags & MemFlags ; return IMemFlags != <NUM_LIT> && MI . mayLoadOrStore ( ) && ! MI . isBundled ( ) ;


<s> if ( ClauseLength > <NUM_LIT> ) { Changed = true ; if ( Next != E && Next -> isKill ( ) ) { for ( const MachineInstr & BundleMI : make_range ( BundleStart , Next ) ) collectUsedRegUnits ( BundleMI , BundleUsedRegUnits ) ; BundleUsedRegUnits . flip ( ) ; while ( Next != E && Next -> isKill ( ) ) { MachineInstr & Kill = * Next ; collectUsedRegUnits ( Kill , KillUsedRegUnits ) ; KillUsedRegUnits &= BundleUsedRegUnits ; if ( KillUsedRegUnits . none ( ) ) { ++ Next ; Kill . eraseFromParent ( ) ; } else break ; KillUsedRegUnits . reset ( ) ; } BundleUsedRegUnits . reset ( ) ; } finalizeBundle ( MBB , BundleStart , Next ) ; } Defs . clear ( ) ; } } return Changed ;


<s> if ( ! MI . mayLoad ( ) ) return false ; for ( const MachineOperand & Op : MI . explicit_operands ( ) ) { if ( ! Op . isReg ( ) ) continue ;


<s> for ( const Register Def : Defs ) if ( TRI -> regsOverlap ( Reg , Def ) ) return true ; } return false ;


<s> if ( ! Op . isReg ( ) || ! Op . readsReg ( ) ) continue ; Register Reg = Op . getReg ( ) ; assert ( ! Op . getSubReg ( ) && <STR_LIT> ) ; for ( MCRegUnitIterator Units ( Reg , TRI ) ; Units . isValid ( ) ; ++ Units ) UsedRegUnits . set ( * Units ) ; }


<s> const uint64_t IMemFlags = I -> getDesc ( ) . TSFlags & MemFlags ; if ( IMemFlags == <NUM_LIT> || I -> isBundled ( ) || ! I -> mayLoadOrStore ( ) || B -> mayLoad ( ) != I -> mayLoad ( ) || B -> mayStore ( ) != I -> mayStore ( ) || ( ( B -> getDesc ( ) . TSFlags & MemFlags ) != IMemFlags ) || isDependentLoad ( * I ) ) { if ( B != I ) { if ( std :: next ( B ) != I ) { finalizeBundle ( MBB , B , I ) ; Changed = true ; } Next = I ;


<s> return new <ISA_LIT>PostRABundler ( ) ;


<s> StringRef getPassName ( ) const override {


<s> for ( const MachineOperand & Op : MI . explicit_operands ( ) ) { if ( ! Op . isReg ( ) ) continue ; Register Reg = Op . getReg ( ) ; for ( Register Def : Defs ) if ( TRI -> regsOverlap ( Reg , Def ) ) return true ; }


<s> MachineBasicBlock :: instr_iterator B = MBB . instr_begin ( ) ; MachineBasicBlock :: instr_iterator E = MBB . instr_end ( ) ; for ( auto I = B ; I != E ; I = Next ) { Next = std :: next ( I ) ; if ( I -> isBundled ( ) || ! I -> mayLoadOrStore ( ) || B -> mayLoad ( ) != I -> mayLoad ( ) || B -> mayStore ( ) != I -> mayStore ( ) || ( B -> getDesc ( ) . TSFlags & MemFlags ) != ( I -> getDesc ( ) . TSFlags & MemFlags ) || isDependentLoad ( * I ) ) { if ( B != I ) { if ( std :: next ( B ) != I ) { finalizeBundle ( MBB , B , I ) ;


<s> <ISA_LIT>PostRABundler ( ) : MachineFunctionPass ( ID ) { initialize<ISA_LIT>PostRABundlerPass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> if ( ! MO . isReg ( ) ) return false ; Register Reg = MO . getReg ( ) ; if ( Reg . isPhysical ( ) ) return false ; if ( ! TRI -> isVGPR ( * MRI , Reg ) ) return false ; if ( VRM -> hasPhys ( Reg ) ) return false ; LiveInterval & LI = LIS -> getInterval ( Reg ) ; for ( MCRegister PhysReg : RegClassInfo . getOrder ( MRI -> getRegClass ( Reg ) ) ) { if ( ! MRI -> isPhysRegUsed ( PhysReg ) && Matrix -> checkInterference ( LI , PhysReg ) == LiveRegMatrix :: IK_Free ) {


<s> LiveInterval & LI = LIS -> getInterval ( Reg ) ; for ( MCRegister PhysReg : RegClassInfo . getOrder ( MRI -> getRegClass ( Reg ) ) ) { if ( ! MRI -> isPhysRegUsed ( PhysReg ) && Matrix -> checkInterference ( LI , PhysReg ) == LiveRegMatrix :: IK_Free ) { Matrix -> assign ( LI , PhysReg ) ; assert ( PhysReg != <NUM_LIT> ) ; RegsToRewrite . push_back ( Reg ) ; return true ; } }


<s> unsigned Opc = MI . getOpcode ( ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) { dbgs ( ) << <STR_LIT> ; } else { assert ( Opc == <STR_LIT> || Opc == <STR_LIT> ) ; dbgs ( ) << <STR_LIT> ; } if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) {


<s> LLVM_DUMP_METHOD void <ISA_LIT>PreAllocateWWMRegs :: printWWMInfo ( const MachineInstr & MI ) { unsigned Opc = MI . getOpcode ( ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) { dbgs ( ) << <STR_LIT> ; } else { assert ( Opc == <STR_LIT> || Opc == <STR_LIT> ) ; dbgs ( ) << <STR_LIT> ; } if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) { dbgs ( ) << <STR_LIT> ; } else { assert ( Opc == <STR_LIT> || Opc == <STR_LIT> ) ; dbgs ( ) << <STR_LIT> ;


<s> if ( ! VRM -> hasPhys ( VirtReg ) ) continue ; Register PhysReg = VRM -> getPhys ( VirtReg ) ; const unsigned SubReg = MO . getSubReg ( ) ; if ( SubReg != <NUM_LIT> ) { PhysReg = TRI -> getSubReg ( PhysReg , SubReg ) ; MO . setSubReg ( <NUM_LIT> ) ; } MO . setReg ( PhysReg ) ; MO . setIsRenamable ( false ) ; } } } <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; for ( unsigned Reg : RegsToRewrite ) { LIS -> removeInterval ( Reg ) ; const Register PhysReg = VRM -> getPhys ( Reg ) ; assert ( PhysReg != <NUM_LIT> ) ; if ( ! MFI -> WWMReservedRegs . count ( PhysReg ) ) { Optional < int > FI ; if ( ! MFI -> isEntryFunction ( ) ) { const TargetRegisterClass * RC = TRI -> getPhysRegClass ( PhysReg ) ; FI = FrameInfo . CreateSpillStackObject ( TRI -> getSpillSize ( * RC ) , TRI -> getSpillAlign ( * RC ) ) ; } MFI -> reserveWWMRegister ( PhysReg , FI ) ; } }


<s> RegClassInfo . runOnMachineFunction ( MF ) ; bool RegsAssigned = false ; ReversePostOrderTraversal < MachineFunction * > RPOT ( & MF ) ; for ( MachineBasicBlock * MBB : RPOT ) { bool InWWM = false ; for ( MachineInstr & MI : * MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ) RegsAssigned |= processDef ( MI . getOperand ( <NUM_LIT> ) ) ;

if ( MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ) {


<s> if ( ! TRI -> isVGPR ( * MRI , Reg ) ) return false ; if ( VRM -> hasPhys ( Reg ) ) return false ; LiveInterval & LI = LIS -> getInterval ( Reg ) ; for ( MCRegister PhysReg : RegClassInfo . getOrder ( MRI -> getRegClass ( Reg ) ) ) { if ( ! MRI -> isPhysRegUsed ( PhysReg ) && Matrix -> checkInterference ( LI , PhysReg ) == LiveRegMatrix :: IK_Free ) { Matrix -> assign ( LI , PhysReg ) ; assert ( PhysReg != <NUM_LIT> ) ;


<s> const Register VirtReg = MO . getReg ( ) ; if ( VirtReg . isPhysical ( ) ) continue ; if ( ! VRM -> hasPhys ( VirtReg ) ) continue ; Register PhysReg = VRM -> getPhys ( VirtReg ) ; const unsigned SubReg = MO . getSubReg ( ) ; if ( SubReg != <NUM_LIT> ) { PhysReg = TRI -> getSubReg ( PhysReg , SubReg ) ; MO . setSubReg ( <NUM_LIT> ) ; } MO . setReg ( PhysReg ) ; MO . setIsRenamable ( false ) ; } } } <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; for ( unsigned Reg : RegsToRewrite ) { LIS -> removeInterval ( Reg ) ; const Register PhysReg = VRM -> getPhys ( Reg ) ; assert ( PhysReg != <NUM_LIT> ) ; MFI -> reserveWWMRegister ( PhysReg ) ;


<s> const unsigned VirtReg = MO . getReg ( ) ; if ( TRI -> isPhysicalRegister ( VirtReg ) ) continue ; if ( ! VRM -> hasPhys ( VirtReg ) ) continue ; unsigned PhysReg = VRM -> getPhys ( VirtReg ) ; const unsigned SubReg = MO . getSubReg ( ) ; if ( SubReg != <NUM_LIT> ) { PhysReg = TRI -> getSubReg ( PhysReg , SubReg ) ; MO . setSubReg ( <NUM_LIT> ) ; } MO . setReg ( PhysReg ) ; MO . setIsRenamable ( false ) ; } } } <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; for ( unsigned Reg : RegsToRewrite ) { LIS -> removeInterval ( Reg ) ; const unsigned PhysReg = VRM -> getPhys ( Reg ) ; assert ( PhysReg != <NUM_LIT> ) ; MFI -> ReserveWWMRegister ( PhysReg ) ; } RegsToRewrite . clear ( ) ; MRI -> freezeReservedRegs ( MF ) ;


<s> Register Reg = MO . getReg ( ) ; if ( ! TRI -> isVGPR ( * MRI , Reg ) ) return false ; if ( Reg . isPhysical ( ) ) return false ; if ( VRM -> hasPhys ( Reg ) ) return false ; LiveInterval & LI = LIS -> getInterval ( Reg ) ; for ( unsigned PhysReg : RegClassInfo . getOrder ( MRI -> getRegClass ( Reg ) ) ) { if ( ! MRI -> isPhysRegUsed ( PhysReg ) && Matrix -> checkInterference ( LI , PhysReg ) == LiveRegMatrix :: IK_Free ) { Matrix -> assign ( LI , PhysReg ) ; assert ( PhysReg != <NUM_LIT> ) ; RegsToRewrite . push_back ( Reg ) ; return true ; }


<s> if ( Reg . isPhysical ( ) ) return false ; if ( VRM -> hasPhys ( Reg ) ) return false ; LiveInterval & LI = LIS -> getInterval ( Reg ) ; for ( unsigned PhysReg : RegClassInfo . getOrder ( MRI -> getRegClass ( Reg ) ) ) { if ( ! MRI -> isPhysRegUsed ( PhysReg ) && Matrix -> checkInterference ( LI , PhysReg ) == LiveRegMatrix :: IK_Free ) { Matrix -> assign ( LI , PhysReg ) ; assert ( PhysReg != <NUM_LIT> ) ; RegsToRewrite . push_back ( Reg ) ; return true ; } } llvm_unreachable ( <STR_LIT> ) ;


<s> FunctionPass * llvm :: create<ISA_LIT>PreAllocateWWMRegsPass ( ) {

return new <ISA_LIT>PreAllocateWWMRegs ( ) ;


<s> AU . addRequired < VirtRegMap > ( ) ; AU . addRequired < LiveRegMatrix > ( ) ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < LiveIntervals > ( ) ; AU . addPreserved < LiveIntervals > ( ) ; AU . addRequired < VirtRegMap > ( ) ; AU . addRequired < LiveRegMatrix > ( ) ; AU . addPreserved < SlotIndexes > ( ) ; AU . setPreservesCFG ( ) ;

MachineFunctionPass :: getAnalysisUsage ( AU ) ;


<s> if ( ! MO . isReg ( ) ) return false ; Register Reg = MO . getReg ( ) ; if ( ! TRI -> isVGPR ( * MRI , Reg ) ) return false ;

if ( Register :: isPhysicalRegister ( Reg ) ) return false ;


<s> MO . setReg ( PhysReg ) ; MO . setIsRenamable ( false ) ; } } } <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; for ( unsigned Reg : RegsToRewrite ) { LIS -> removeInterval ( Reg ) ; const Register PhysReg = VRM -> getPhys ( Reg ) ; assert ( PhysReg != <NUM_LIT> ) ; MFI -> ReserveWWMRegister ( PhysReg ) ; }


<s> void <ISA_LIT>PreAllocateWWMRegs :: rewriteRegs ( MachineFunction & MF ) { for ( MachineBasicBlock & MBB : MF ) { for ( MachineInstr & MI : MBB ) { for ( MachineOperand & MO : MI . operands ( ) ) { if ( ! MO . isReg ( ) ) continue ; const Register VirtReg = MO . getReg ( ) ; if ( Register :: isPhysicalRegister ( VirtReg ) ) continue ; if ( ! VRM -> hasPhys ( VirtReg ) ) continue ; Register PhysReg = VRM -> getPhys ( VirtReg ) ; const unsigned SubReg = MO . getSubReg ( ) ; if ( SubReg != <NUM_LIT> ) { PhysReg = TRI -> getSubReg ( PhysReg , SubReg ) ; MO . setSubReg ( <NUM_LIT> ) ; }


<s> ReversePostOrderTraversal < MachineFunction * > RPOT ( & MF ) ; for ( MachineBasicBlock * MBB : RPOT ) { bool InWWM = false ; for ( MachineInstr & MI : * MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ) RegsAssigned |= processDef ( MI . getOperand ( <NUM_LIT> ) ) ; if ( MI . getOpcode ( ) == <STR_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> ) ; InWWM = true ; continue ; } if ( MI . getOpcode ( ) == <STR_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> ) ; InWWM = false ; } if ( ! InWWM ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> ) ; for ( MachineOperand & DefOpnd : MI . defs ( ) ) { RegsAssigned |= processDef ( DefOpnd ) ; } } } if ( ! RegsAssigned ) return false ; rewriteRegs ( MF ) ; return true ;


<s> <ISA_LIT>PreAllocateWWMRegs ( ) : MachineFunctionPass ( ID ) {


<s> if ( TII -> hasUnwantedEffectsWhenEXECEmpty ( MI ) ) return true ; if ( TII -> isSMRD ( MI ) || TII -> isVMEM ( MI ) || TII -> isFLAT ( MI ) || TII -> isDS ( MI ) || MI . getOpcode ( ) == <STR_LIT> ) return true ; ++ NumInstr ; if ( NumInstr >= SkipThreshold ) return true ; } } return false ;


<s> case <STR_LIT> : return false ; case <STR_LIT> : IdxOn = false ; ToRemove . push_back ( & * I ) ; break ; default : if ( I -> modifiesRegister ( <STR_LIT> , TRI ) ) return false ; if ( IdxReg && I -> modifiesRegister ( IdxReg , TRI ) ) return false ; if ( llvm :: any_of ( I -> operands ( ) , [ & MRI , this ] ( const MachineOperand & MO ) { return MO . isReg ( ) && TRI -> isVectorRegister ( MRI , MO . getReg ( ) ) ; } ) ) { if ( ! IdxOn || ! ( ( I -> getOpcode ( ) == <STR_LIT> && I -> hasRegisterImplicitUseOperand ( <STR_LIT> ) ) || I -> getOpcode ( ) == <STR_LIT> ) ) return false ; } } } MI . eraseFromParent ( ) ; for ( MachineInstr * RI : ToRemove ) RI -> eraseFromParent ( ) ; return true ;


<s> Register IdxReg = Idx -> isReg ( ) ? Idx -> getReg ( ) : Register ( ) ; SmallVector < MachineInstr * , <NUM_LIT> > ToRemove ; bool IdxOn = true ; if ( ! MI . isIdenticalTo ( First ) ) return false ; for ( MachineBasicBlock :: iterator I = std :: next ( First . getIterator ( ) ) , E = MI . getIterator ( ) ; I != E ; ++ I ) { switch ( I -> getOpcode ( ) ) { case <STR_LIT> : return false ; case <STR_LIT> : IdxOn = false ; ToRemove . push_back ( & * I ) ; break ; default : if ( I -> modifiesRegister ( <STR_LIT> , TRI ) ) return false ; if ( IdxReg && I -> modifiesRegister ( IdxReg , TRI ) ) return false ; if ( llvm :: any_of ( I -> operands ( ) , [ & MRI , this ] ( const MachineOperand & MO ) { return MO . isReg ( ) && TRI -> isVectorRegister ( MRI , MO . getReg ( ) ) ; } ) ) { if ( ! IdxOn || ! ( ( I -> getOpcode ( ) == <STR_LIT> && I -> hasRegisterImplicitUseOperand ( <STR_LIT> ) ) || I -> getOpcode ( ) == <STR_LIT> ) ) return false ; } } }


<s> MI . eraseFromParent ( ) ; MBBE = MBB . getFirstTerminator ( ) ; } break ; default : break ; } } if ( ! ST . hasVGPRIndexMode ( ) ) continue ; MachineInstr * SetGPRMI = nullptr ; const unsigned Threshold = <NUM_LIT> ; unsigned Count = <NUM_LIT> ; for ( MachineBasicBlock :: iterator MBBI = MBB . begin ( ) ; MBBI != MBBE ; ) { MachineInstr & MI = * MBBI ; ++ MBBI ; if ( Count == Threshold ) SetGPRMI = nullptr ; else ++ Count ; if ( MI . getOpcode ( ) != <STR_LIT> ) continue ; Count = <NUM_LIT> ;


<s> bool Changed = false ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator MBBE = MBB . getFirstTerminator ( ) ; if ( MBBE != MBB . end ( ) ) { MachineInstr & MI = * MBBE ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : Changed |= optimizeVccBranch ( MI ) ; continue ; case <STR_LIT> : assert ( ! MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ) ; if ( & MBB != & MF . back ( ) || & MI != & MBB . back ( ) ) { if ( ! EmptyMBBAtEnd ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } MBB . addSuccessor ( EmptyMBBAtEnd ) ; BuildMI ( MBB , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( EmptyMBBAtEnd ) ; MI . eraseFromParent ( ) ; MBBE = MBB . getFirstTerminator ( ) ; } break ; default : break ;


<s> if ( M -> definesRegister ( SReg , TRI ) ) break ; if ( M -> modifiesRegister ( SReg , TRI ) ) return Changed ; ReadsSreg |= M -> readsRegister ( SReg , TRI ) ; } if ( M == E || ! M -> isMoveImmediate ( ) || ! M -> getOperand ( <NUM_LIT> ) . isImm ( ) || M -> getOperand ( <NUM_LIT> ) . getImm ( ) != - <NUM_LIT> ) return Changed ; if ( ! ReadsSreg && Op2 . isKill ( ) ) { A -> getOperand ( <NUM_LIT> ) . ChangeToImmediate ( - <NUM_LIT> ) ; M -> eraseFromParent ( ) ; } } if ( ! ReadsCond && A -> registerDefIsDead ( <STR_LIT> ) && MI . killsRegister ( CondReg , TRI ) ) A -> eraseFromParent ( ) ; bool IsVCCZ = MI . getOpcode ( ) == <STR_LIT> ; if ( SReg == ExecReg ) { if ( IsVCCZ ) { MI . eraseFromParent ( ) ; return true ; }

MI . setDesc ( TII -> get ( <STR_LIT> ) ) ;


<s> if ( A -> modifiesRegister ( ExecReg , TRI ) ) return false ; if ( A -> modifiesRegister ( CondReg , TRI ) ) { if ( ! A -> definesRegister ( CondReg , TRI ) || A -> getOpcode ( ) != And ) return false ; break ; } ReadsCond |= A -> readsRegister ( CondReg , TRI ) ; } if ( A == E ) return false ; MachineOperand & Op1 = A -> getOperand ( <NUM_LIT> ) ; MachineOperand & Op2 = A -> getOperand ( <NUM_LIT> ) ; if ( Op1 . getReg ( ) != ExecReg && Op2 . isReg ( ) && Op2 . getReg ( ) == ExecReg ) { TII -> commuteInstruction ( * A ) ; Changed = true ; } if ( Op1 . getReg ( ) != ExecReg ) return Changed ; if ( Op2 . isImm ( ) && Op2 . getImm ( ) != - <NUM_LIT> ) return Changed ; Register SReg ; if ( Op2 . isReg ( ) ) { SReg = Op2 . getReg ( ) ; auto M = std :: next ( A ) ; bool ReadsSreg = false ; for ( ; M != E ; ++ M ) { if ( M -> definesRegister ( SReg , TRI ) ) break ; if ( M -> modifiesRegister ( SReg , TRI ) ) return Changed ; ReadsSreg |= M -> readsRegister ( SReg , TRI ) ; } if ( M == E || ! M -> isMoveImmediate ( ) || ! M -> getOperand ( <NUM_LIT> ) . isImm ( ) || M -> getOperand ( <NUM_LIT> ) . getImm ( ) != - <NUM_LIT> ) return Changed ; if ( ! ReadsSreg && Op2 . isKill ( ) ) { A -> getOperand ( <NUM_LIT> ) . ChangeToImmediate ( - <NUM_LIT> ) ; M -> eraseFromParent ( ) ; } } if ( ! ReadsCond && A -> registerDefIsDead ( <STR_LIT> ) && MI . killsRegister ( CondReg , TRI ) ) A -> eraseFromParent ( ) ; bool IsVCCZ = MI . getOpcode ( ) == <STR_LIT> ; if ( SReg == ExecReg ) { if ( IsVCCZ ) { MI . eraseFromParent ( ) ; return true ; } MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; } else { MI . setDesc ( TII -> get ( IsVCCZ ? <STR_LIT> : <STR_LIT> ) ) ; }

MI . RemoveOperand ( MI . findRegisterUseOperandIdx ( CondReg , false , TRI ) ) ;


<s> bool <ISA_LIT>PreEmitPeephole :: runOnMachineFunction ( MachineFunction & MF ) { const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ;


<s> MachineInstr & MI = * TermI ; if ( MI . getOpcode ( ) == <STR_LIT> ) { assert ( ! MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ) ; if ( & MBB != & MF . back ( ) || & MI != & MBB . back ( ) ) { if ( ! EmptyMBBAtEnd ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } MBB . addSuccessor ( EmptyMBBAtEnd ) ; BuildMI ( MBB , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( EmptyMBBAtEnd ) ; MI . eraseFromParent ( ) ; MBBE = MBB . getFirstTerminator ( ) ; TermI = MBBE ; continue ;


<s> if ( M -> modifiesRegister ( SReg , TRI ) ) return Changed ; ReadsSreg |= M -> readsRegister ( SReg , TRI ) ; } if ( M == E || ! M -> isMoveImmediate ( ) || ! M -> getOperand ( <NUM_LIT> ) . isImm ( ) || ( M -> getOperand ( <NUM_LIT> ) . getImm ( ) != - <NUM_LIT> && M -> getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ) ) return Changed ; MaskValue = M -> getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( ! ReadsSreg && Op2 . isKill ( ) ) { A -> getOperand ( <NUM_LIT> ) . ChangeToImmediate ( MaskValue ) ; M -> eraseFromParent ( ) ; } } else if ( Op2 . isImm ( ) ) { MaskValue = Op2 . getImm ( ) ; } else { llvm_unreachable ( <STR_LIT> ) ; } assert ( MaskValue == <NUM_LIT> || MaskValue == - <NUM_LIT> ) ; if ( A -> getOpcode ( ) == AndN2 ) MaskValue = ~ MaskValue ; if ( ! ReadsCond && A -> registerDefIsDead ( <STR_LIT> ) && MI . killsRegister ( CondReg , TRI ) ) A -> eraseFromParent ( ) ; bool IsVCCZ = MI . getOpcode ( ) == <STR_LIT> ; if ( SReg == ExecReg ) { if ( IsVCCZ ) { MI . eraseFromParent ( ) ; return true ; } MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; } else if ( IsVCCZ && MaskValue == <NUM_LIT> ) { MachineBasicBlock * Parent = MI . getParent ( ) ; SmallVector < MachineInstr * , <NUM_LIT> > ToRemove ; bool Found = false ; for ( MachineInstr & Term : Parent -> terminators ( ) ) { if ( Found ) { if ( Term . isBranch ( ) ) ToRemove . push_back ( & Term ) ; } else { Found = Term . isIdenticalTo ( MI ) ; } } assert ( Found && <STR_LIT> ) ; for ( auto BranchMI : ToRemove ) { MachineOperand & Dst = BranchMI -> getOperand ( <NUM_LIT> ) ; assert ( Dst . isMBB ( ) && <STR_LIT> ) ; Parent -> removeSuccessor ( Dst . getMBB ( ) ) ; BranchMI -> eraseFromParent ( ) ; } if ( MachineBasicBlock * Succ = Parent -> getFallThrough ( ) ) { Parent -> removeSuccessor ( Succ ) ; } MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; } else if ( ! IsVCCZ && MaskValue == <NUM_LIT> ) { MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ;


<s> const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; MachineOperand * Idx = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; Register IdxReg = Idx -> isReg ( ) ? Idx -> getReg ( ) : Register ( ) ; SmallVector < MachineInstr * , <NUM_LIT> > ToRemove ; bool IdxOn = true ; if ( ! MI . isIdenticalTo ( First ) ) return false ; for ( MachineBasicBlock :: instr_iterator I = std :: next ( First . getIterator ( ) ) , E = MI . getIterator ( ) ; I != E ; ++ I ) { if ( I -> isBundle ( ) ) continue ; switch ( I -> getOpcode ( ) ) { case <STR_LIT> : return false ; case <STR_LIT> : IdxOn = false ; ToRemove . push_back ( & * I ) ; break ; default : if ( I -> modifiesRegister ( <STR_LIT> , TRI ) ) return false ; if ( IdxReg && I -> modifiesRegister ( IdxReg , TRI ) ) return false ; if ( llvm :: any_of ( I -> operands ( ) , [ & MRI , this ] ( const MachineOperand & MO ) { return MO . isReg ( ) && TRI -> isVectorRegister ( MRI , MO . getReg ( ) ) ; } ) ) {

if ( ! IdxOn || ! ( ( I -> getOpcode ( ) == <STR_LIT> && I -> hasRegisterImplicitUseOperand ( <STR_LIT> ) ) || I -> getOpcode ( ) == <STR_LIT> ) ) return false ;


<s> Register IdxReg = Idx -> isReg ( ) ? Idx -> getReg ( ) : Register ( ) ; SmallVector < MachineInstr * , <NUM_LIT> > ToRemove ; bool IdxOn = true ; if ( ! MI . isIdenticalTo ( First ) ) return false ; for ( MachineBasicBlock :: instr_iterator I = std :: next ( First . getIterator ( ) ) , E = MI . getIterator ( ) ; I != E ; ++ I ) { if ( I -> isBundle ( ) ) continue ; switch ( I -> getOpcode ( ) ) { case <STR_LIT> : return false ; case <STR_LIT> : IdxOn = false ; ToRemove . push_back ( & * I ) ; break ; default : if ( I -> modifiesRegister ( <STR_LIT> , TRI ) ) return false ; if ( IdxReg && I -> modifiesRegister ( IdxReg , TRI ) ) return false ; if ( llvm :: any_of ( I -> operands ( ) , [ & MRI , this ] ( const MachineOperand & MO ) { return MO . isReg ( ) && TRI -> isVectorRegister ( MRI , MO . getReg ( ) ) ;


<s> bool <ISA_LIT>PreEmitPeephole :: runOnMachineFunction ( MachineFunction & MF ) { const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; bool Changed = false ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator MBBE = MBB . getFirstTerminator ( ) ; if ( MBBE != MBB . end ( ) ) { MachineInstr & MI = * MBBE ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : Changed |= optimizeVccBranch ( MI ) ; continue ; default : break ; } } if ( ! ST . hasVGPRIndexMode ( ) ) continue ; MachineInstr * SetGPRMI = nullptr ; const unsigned Threshold = <NUM_LIT> ; unsigned Count = <NUM_LIT> ; for ( MachineBasicBlock :: iterator MBBI = MBB . begin ( ) ; MBBI != MBBE ; ) { MachineInstr & MI = * MBBI ; ++ MBBI ;

if ( Count == Threshold ) SetGPRMI = nullptr ;


<s> bool <ISA_LIT>PreEmitPeephole :: getBlockDestinations ( MachineBasicBlock & SrcMBB , MachineBasicBlock * & TrueMBB , MachineBasicBlock * & FalseMBB , SmallVectorImpl < MachineOperand > & Cond ) {

if ( TII -> analyzeBranch ( SrcMBB , TrueMBB , FalseMBB , Cond ) ) return false ;


<s> MBBI != End && MBBI != ToI ; ++ MBBI ) { const MachineBasicBlock & MBB = * MBBI ; for ( MachineBasicBlock :: const_iterator I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; ++ I ) { if ( I -> isConditionalBranch ( ) ) return true ; if ( TII -> hasUnwantedEffectsWhenEXECEmpty ( * I ) ) return true ; if ( TII -> isSMRD ( * I ) || TII -> isVMEM ( * I ) || TII -> isFLAT ( * I ) || TII -> isDS ( * I ) || I -> getOpcode ( ) == <STR_LIT> ) return true ; ++ NumInstr ; if ( NumInstr >= SkipThreshold ) return true ; }


<s> if ( I -> isConditionalBranch ( ) ) return true ; if ( TII -> hasUnwantedEffectsWhenEXECEmpty ( * I ) ) return true ; if ( TII -> isSMRD ( * I ) || TII -> isVMEM ( * I ) || TII -> isFLAT ( * I ) || TII -> isDS ( * I ) || I -> getOpcode ( ) == <STR_LIT> ) return true ; ++ NumInstr ;

if ( NumInstr >= SkipThreshold ) return true ;


<s> Register IdxReg = Idx -> isReg ( ) ? Idx -> getReg ( ) : Register ( ) ; SmallVector < MachineInstr * , <NUM_LIT> > ToRemove ; bool IdxOn = true ; if ( ! MI . isIdenticalTo ( First ) ) return false ; for ( MachineBasicBlock :: instr_iterator I = std :: next ( First . getIterator ( ) ) , E = MI . getIterator ( ) ; I != E ; ++ I ) { if ( I -> isBundle ( ) ) continue ; switch ( I -> getOpcode ( ) ) { case <STR_LIT> : return false ; case <STR_LIT> : IdxOn = false ; ToRemove . push_back ( & * I ) ; break ; default : if ( I -> modifiesRegister ( <STR_LIT> , TRI ) ) return false ;

if ( IdxReg && I -> modifiesRegister ( IdxReg , TRI ) ) return false ;


<s> if ( ! ReadsSreg && Op2 . isKill ( ) ) { A -> getOperand ( <NUM_LIT> ) . ChangeToImmediate ( MaskValue ) ; M -> eraseFromParent ( ) ; } } else if ( Op2 . isImm ( ) ) { MaskValue = Op2 . getImm ( ) ; } else { llvm_unreachable ( <STR_LIT> ) ; } assert ( MaskValue == <NUM_LIT> || MaskValue == - <NUM_LIT> ) ; if ( A -> getOpcode ( ) == AndN2 ) MaskValue = ~ MaskValue ; if ( ! ReadsCond && A -> registerDefIsDead ( <STR_LIT> ) ) { if ( ! MI . killsRegister ( CondReg , TRI ) ) { if ( MaskValue == <NUM_LIT> ) { BuildMI ( * A -> getParent ( ) , * A , A -> getDebugLoc ( ) , TII -> get ( Mov ) , CondReg ) . addImm ( <NUM_LIT> ) ; } else { BuildMI ( * A -> getParent ( ) , * A , A -> getDebugLoc ( ) , TII -> get ( Mov ) , CondReg ) . addReg ( ExecReg ) ; } } A -> eraseFromParent ( ) ; } bool IsVCCZ = MI . getOpcode ( ) == <STR_LIT> ; if ( SReg == ExecReg ) { if ( IsVCCZ ) { MI . eraseFromParent ( ) ; return true ; } MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; } else if ( IsVCCZ && MaskValue == <NUM_LIT> ) { MachineBasicBlock * Parent = MI . getParent ( ) ; SmallVector < MachineInstr * , <NUM_LIT> > ToRemove ; bool Found = false ; for ( MachineInstr & Term : Parent -> terminators ( ) ) { if ( Found ) { if ( Term . isBranch ( ) ) ToRemove . push_back ( & Term ) ; } else { Found = Term . isIdenticalTo ( MI ) ; } } assert ( Found && <STR_LIT> ) ; for ( auto BranchMI : ToRemove ) {

MachineOperand & Dst = BranchMI -> getOperand ( <NUM_LIT> ) ;


<s> if ( ( SrcMBB . getNumber ( ) >= TrueMBB -> getNumber ( ) ) || mustRetainExeczBranch ( * FalseMBB , * TrueMBB ) ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MI . eraseFromParent ( ) ; SrcMBB . removeSuccessor ( TrueMBB ) ;


<s> TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; bool Changed = false ; MF . RenumberBlocks ( ) ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator TermI = MBB . getFirstTerminator ( ) ; if ( TermI != MBB . end ( ) ) { MachineInstr & MI = * TermI ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : Changed |= optimizeVccBranch ( MI ) ;


<s> <ISA_LIT>PreEmitPeephole ( ) : MachineFunctionPass ( ID ) { initialize<ISA_LIT>PreEmitPeepholePass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> RS . enterBasicBlock ( & MBB ) ; for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; ++ I ) { MachineInstr & MI = * I ; RS . forward ( I ) ; DebugLoc DL = MI . getDebugLoc ( ) ; if ( ! TII -> isVGPRSpill ( MI . getOpcode ( ) ) ) continue ; unsigned ScratchRsrcReg = RS . scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; uint64_t Rsrc23 = TII -> getScratchRsrcWords23 ( ) ; unsigned Rsrc0 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc1 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc2 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc3 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Rsrc0 ) . addExternalSymbol ( <STR_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Rsrc1 ) . addExternalSymbol ( <STR_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Rsrc2 ) . addImm ( Rsrc23 & <NUM_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Rsrc3 ) . addImm ( Rsrc23 >> <NUM_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; if ( ScratchOffsetReg == <STR_LIT> ) { ScratchOffsetReg = RS . scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchOffsetReg ) . addFrameIndex ( ScratchOffsetFI ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) ; } else if ( ! MBB . isLiveIn ( ScratchOffsetReg ) ) { MBB . addLiveIn ( ScratchOffsetReg ) ; } if ( ScratchRsrcReg == <STR_LIT> || ScratchOffsetReg == <STR_LIT> ) { LLVMContext & Ctx = MF . getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> ) ; ScratchRsrcReg = <STR_LIT> ; ScratchOffsetReg = <STR_LIT> ; } MI . getOperand ( <NUM_LIT> ) . setReg ( ScratchRsrcReg ) ; MI . getOperand ( <NUM_LIT> ) . setIsKill ( true ) ;

MI . getOperand ( <NUM_LIT> ) . setIsUndef ( false ) ;


<s> BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Rsrc3 ) . addImm ( Rsrc23 >> <NUM_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; if ( ScratchOffsetReg == <STR_LIT> ) { ScratchOffsetReg = RS . scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchOffsetReg ) . addFrameIndex ( ScratchOffsetFI ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) ; } else if ( ! MBB . isLiveIn ( ScratchOffsetReg ) ) { MBB . addLiveIn ( ScratchOffsetReg ) ; } if ( ScratchRsrcReg == <STR_LIT> || ScratchOffsetReg == <STR_LIT> ) { LLVMContext & Ctx = MF . getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> ) ; ScratchRsrcReg = <STR_LIT> ; ScratchOffsetReg = <STR_LIT> ; } MI . getOperand ( <NUM_LIT> ) . setReg ( ScratchRsrcReg ) ; MI . getOperand ( <NUM_LIT> ) . setIsKill ( true ) ; MI . getOperand ( <NUM_LIT> ) . setIsUndef ( false ) ; MI . getOperand ( <NUM_LIT> ) . setReg ( ScratchOffsetReg ) ; MI . getOperand ( <NUM_LIT> ) . setIsUndef ( false ) ; MI . getOperand ( <NUM_LIT> ) . setIsKill ( false ) ; MI . addOperand ( MachineOperand :: CreateReg ( Rsrc0 , false , true , true ) ) ; MI . addOperand ( MachineOperand :: CreateReg ( Rsrc1 , false , true , true ) ) ; MI . addOperand ( MachineOperand :: CreateReg ( Rsrc2 , false , true , true ) ) ; MI . addOperand ( MachineOperand :: CreateReg ( Rsrc3 , false , true , true ) ) ; } }


<s> unsigned ScratchPtrPreloadReg = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: SCRATCH_PTR ) ; unsigned ScratchOffsetPreloadReg = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: SCRATCH_WAVE_OFFSET ) ; if ( ! Entry -> isLiveIn ( ScratchPtrPreloadReg ) ) Entry -> addLiveIn ( ScratchPtrPreloadReg ) ; if ( ! Entry -> isLiveIn ( ScratchOffsetPreloadReg ) ) Entry -> addLiveIn ( ScratchOffsetPreloadReg ) ; unsigned ScratchOffsetReg = TRI -> findUnusedRegister ( MRI , & <STR_LIT> ) ; int ScratchOffsetFI = - <NUM_LIT> ; if ( ScratchOffsetReg != <STR_LIT> ) { BuildMI ( * Entry , I , DL , TII -> get ( <STR_LIT> ) , ScratchOffsetReg ) . addReg ( ScratchOffsetPreloadReg ) ; } else { ScratchOffsetFI = FrameInfo -> CreateSpillStackObject ( <NUM_LIT> , <NUM_LIT> ) ; BuildMI ( * Entry , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( ScratchOffsetPreloadReg ) . addFrameIndex ( ScratchOffsetFI ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) ; } RegScavenger RS ; unsigned ScratchRsrcFI = FrameInfo -> CreateSpillStackObject ( <NUM_LIT> , <NUM_LIT> ) ; RS . addScavengingFrameIndex ( ScratchRsrcFI ) ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ; if ( ! MBB . isLiveIn ( ScratchOffsetReg ) && ScratchOffsetReg != <STR_LIT> ) MBB . addLiveIn ( ScratchOffsetReg ) ; RS . enterBasicBlock ( & MBB ) ; for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; ++ I ) { MachineInstr & MI = * I ; RS . forward ( I ) ; DebugLoc DL = MI . getDebugLoc ( ) ; if ( ! TII -> isVGPRSpill ( MI . getOpcode ( ) ) ) continue ; unsigned ScratchRsrcReg = RS . scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ;


<s> FunctionPass * llvm :: create<ISA_LIT>PrepareScratchRegs ( ) {

return new <ISA_LIT>PrepareScratchRegs ( ) ;


<s> const char * getPassName ( ) const override { return <STR_LIT> ;


<s> <ISA_LIT>PrepareScratchRegs ( ) : MachineFunctionPass ( ID ) {


<s> return S_00B848_VGPRS ( VGPRBlocks ) | S_00B848_SGPRS ( SGPRBlocks ) | S_00B848_PRIORITY ( Priority ) | S_00B848_FLOAT_MODE ( FloatMode ) | S_00B848_PRIV ( Priv ) | S_00B848_DX10_CLAMP ( DX10Clamp ) | S_00B848_DEBUG_MODE ( DebugMode ) | S_00B848_IEEE_MODE ( IEEEMode ) | S_00B848_WGP_MODE ( WgpMode ) | S_00B848_MEM_ORDERED ( MemOrdered ) ;


<s> return getComputePGMRSrc1 ( ) ; } uint64_t Reg = S_00B848_VGPRS ( VGPRBlocks ) | S_00B848_SGPRS ( SGPRBlocks ) | S_00B848_PRIORITY ( Priority ) | S_00B848_FLOAT_MODE ( FloatMode ) | S_00B848_PRIV ( Priv ) | S_00B848_DX10_CLAMP ( DX10Clamp ) | S_00B848_DEBUG_MODE ( DebugMode ) | S_00B848_IEEE_MODE ( IEEEMode ) ; switch ( CC ) { case CallingConv :: <ISA_LIT>_PS : Reg |= S_00B028_MEM_ORDERED ( MemOrdered ) ; break ; case CallingConv :: <ISA_LIT>_VS : Reg |= S_00B128_MEM_ORDERED ( MemOrdered ) ; break ; case CallingConv :: <ISA_LIT>_GS : Reg |= S_00B228_WGP_MODE ( WgpMode ) | S_00B228_MEM_ORDERED ( MemOrdered ) ; break ; case CallingConv :: <ISA_LIT>_HS : Reg |= S_00B428_WGP_MODE ( WgpMode ) | S_00B428_MEM_ORDERED ( MemOrdered ) ; break ; default : break ; }


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; if ( ST . isXNACKEnabled ( ) ) reserveRegisterTuples ( Reserved , <STR_LIT> ) ; } if ( ST . hasSGPRInitBug ( ) ) { unsigned NumSGPRs = <STR_LIT> . getNumRegs ( ) ; unsigned Limit = <ISA_LIT>Subtarget :: FIXED_SGPR_COUNT_FOR_INIT_BUG - <NUM_LIT> ; if ( ST . isXNACKEnabled ( ) ) Limit -= <NUM_LIT> ; for ( unsigned i = Limit ; i < NumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } } const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned ScratchWaveOffsetReg = MFI -> getScratchWaveOffsetReg ( ) ; if ( ScratchWaveOffsetReg != <STR_LIT> ) {

reserveRegisterTuples ( Reserved , ScratchWaveOffsetReg ) ;


<s> if ( ST . hasSGPRInitBug ( ) ) { unsigned BaseIdx = <ISA_LIT>Subtarget :: FIXED_SGPR_COUNT_FOR_INIT_BUG - <NUM_LIT> - <NUM_LIT> ; if ( ST . isXNACKEnabled ( ) ) BaseIdx -= <NUM_LIT> ; unsigned BaseReg ( <STR_LIT> . getRegister ( BaseIdx ) ) ; return getMatchingSuperReg ( BaseReg , <STR_LIT> , & <STR_LIT> ) ; } if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { return <STR_LIT> ; }


<s> if ( ST . hasSGPRInitBug ( ) ) { unsigned BaseIdx = <ISA_LIT>Subtarget :: FIXED_SGPR_COUNT_FOR_INIT_BUG - <NUM_LIT> - <NUM_LIT> ; if ( ST . isXNACKEnabled ( ) ) BaseIdx -= <NUM_LIT> ; unsigned BaseReg ( <STR_LIT> . getRegister ( BaseIdx ) ) ; return getMatchingSuperReg ( BaseReg , <STR_LIT> , & <STR_LIT> ) ; } if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) {


<s> const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( ST . hasSGPRInitBug ( ) ) { unsigned Idx ; if ( ! ST . isXNACKEnabled ( ) ) Idx = <ISA_LIT>Subtarget :: FIXED_SGPR_COUNT_FOR_INIT_BUG - <NUM_LIT> - <NUM_LIT> ; else Idx = <ISA_LIT>Subtarget :: FIXED_SGPR_COUNT_FOR_INIT_BUG - <NUM_LIT> - <NUM_LIT> ; return <STR_LIT> . getRegister ( Idx ) ;


<s> bool RanOutOfSGPRs = false ; unsigned SOffset = ScratchOffset ; unsigned NumSubRegs = getNumSubRegsForSpillOp ( MI -> getOpcode ( ) ) ; unsigned Size = NumSubRegs * <NUM_LIT> ; if ( ! isUInt < <NUM_LIT> > ( Offset + Size ) ) { SOffset = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ; if ( SOffset == <STR_LIT> ) { RanOutOfSGPRs = true ; SOffset = <STR_LIT> ; } BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , SOffset ) . addReg ( ScratchOffset ) . addImm ( Offset ) ; Offset = <NUM_LIT> ; } if ( RanOutOfSGPRs ) Ctx . emitError ( <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i != e ; ++ i , Offset += <NUM_LIT> ) { unsigned SubReg = NumSubRegs > <NUM_LIT> ? getPhysRegSubReg ( Value , & <STR_LIT> , i ) : Value ; BuildMI ( * MBB , MI , DL , TII -> get ( LoadStoreOp ) ) . addReg ( SubReg , getDefRegState ( IsLoad ) ) . addReg ( ScratchRsrcReg ) . addReg ( SOffset ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addReg ( Value , RegState :: Implicit | getDefRegState ( IsLoad ) ) . setMemRefs ( MI -> memoperands_begin ( ) , MI -> memoperands_end ( ) ) ; }


<s> } if ( RanOutOfSGPRs ) Ctx . emitError ( <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i != e ; ++ i , Offset += <NUM_LIT> ) { unsigned SubReg = NumSubRegs > <NUM_LIT> ? getPhysRegSubReg ( Value , & <STR_LIT> , i ) : Value ; BuildMI ( * MBB , MI , DL , TII -> get ( LoadStoreOp ) ) . addReg ( SubReg , getDefRegState ( IsLoad ) ) . addReg ( ScratchRsrcReg ) . addReg ( SOffset ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addReg ( Value , RegState :: Implicit | getDefRegState ( IsLoad ) ) . setMemRefs ( MI -> memoperands_begin ( ) , MI -> memoperands_end ( ) ) ; }


<s> Ctx . emitError ( <STR_LIT> ) ; } BuildMI ( * MBB , MI , DL , TII -> getMCOpcodeFromPseudo ( <STR_LIT> ) , Spill . VGPR ) . addReg ( SubReg ) . addImm ( Spill . Lane ) ; } MI -> eraseFromParent ( ) ; break ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { unsigned NumSubRegs = getNumSubRegsForSpillOp ( MI -> getOpcode ( ) ) ; for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i < e ; ++ i ) { unsigned SubReg = getPhysRegSubReg ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) , & <STR_LIT> , i ) ; struct <ISA_LIT>MachineFunctionInfo :: SpilledReg Spill = MFI -> getSpilledReg ( MF , Index , i ) ; if ( Spill . VGPR == <STR_LIT> ) { LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> ) ; } BuildMI ( * MBB , MI , DL , TII -> getMCOpcodeFromPseudo ( <STR_LIT> ) , SubReg ) . addReg ( Spill . VGPR ) . addImm ( Spill . Lane ) . addReg ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) , RegState :: ImplicitDefine ) ; } switch ( MF -> getSubtarget < <ISA_LIT>Subtarget > ( ) . getGeneration ( ) ) { case <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS : TII -> insertWaitStates ( MI , <NUM_LIT> ) ; break ; case <ISA_LIT>Subtarget :: SEA_ISLANDS : break ; default :


<s> unsigned <ISA_LIT>RegisterInfo :: findUnusedRegister ( const MachineRegisterInfo & MRI , const TargetRegisterClass * RC ) const {

for ( unsigned Reg : * RC ) if ( ! MRI . isPhysRegUsed ( Reg ) ) return Reg ;


<s> return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ;


<s> const TargetRegisterClass * <ISA_LIT>RegisterInfo :: getEquivalentVGPRClass ( const TargetRegisterClass * SRC ) const { switch ( SRC -> getSize ( ) ) { case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; default :


<s> unsigned <ISA_LIT>RegisterInfo :: getHWRegIndex ( unsigned Reg ) const { return getEncodingValue ( Reg ) & <NUM_LIT> ;


<s> case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; default : return <NUM_LIT> ; } } else { switch ( WaveCount ) { case <NUM_LIT> : return <NUM_LIT> ;


<s> return <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> case <STR_LIT> : case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return <NUM_LIT> ;

default : llvm_unreachable ( <STR_LIT> ) ;


<s> case <ISA_LIT>RegisterInfo :: WORKGROUP_ID_Y : assert ( MFI -> hasWorkGroupIDY ( ) ) ; return MFI -> WorkGroupIDYSystemSGPR ; case <ISA_LIT>RegisterInfo :: WORKGROUP_ID_Z : assert ( MFI -> hasWorkGroupIDZ ( ) ) ; return MFI -> WorkGroupIDZSystemSGPR ; case <ISA_LIT>RegisterInfo :: PRIVATE_SEGMENT_WAVE_BYTE_OFFSET : return MFI -> PrivateSegmentWaveByteOffsetSystemSGPR ; case <ISA_LIT>RegisterInfo :: PRIVATE_SEGMENT_BUFFER : assert ( ST . isAmdHsaOS ( ) && <STR_LIT> ) ; assert ( MFI -> hasPrivateSegmentBuffer ( ) ) ; return MFI -> PrivateSegmentBufferUserSGPR ; case <ISA_LIT>RegisterInfo :: KERNARG_SEGMENT_PTR : assert ( MFI -> hasKernargSegmentPtr ( ) ) ; return MFI -> KernargSegmentPtrUserSGPR ; case <ISA_LIT>RegisterInfo :: DISPATCH_PTR : assert ( MFI -> hasDispatchPtr ( ) ) ; return MFI -> DispatchPtrUserSGPR ;


<s> unsigned VGPRLimit = getNumVGPRsAllowed ( STI . getMaxWavesPerCU ( ) ) ; unsigned VSLimit = SGPRLimit + VGPRLimit ; for ( regclass_iterator I = regclass_begin ( ) , E = regclass_end ( ) ; I != E ; ++ I ) { const TargetRegisterClass * RC = * I ;

unsigned NumSubRegs = std :: max ( ( int ) RC -> getSize ( ) / <NUM_LIT> , <NUM_LIT> ) ;


<s> unsigned VGPRLimit = getNumVGPRsAllowed ( STI . getMaxWavesPerCU ( ) ) ; unsigned VSLimit = SGPRLimit + VGPRLimit ; for ( regclass_iterator I = regclass_begin ( ) , E = regclass_end ( ) ; I != E ; ++ I ) { const TargetRegisterClass * RC = * I ; unsigned NumSubRegs = std :: max ( ( int ) RC -> getSize ( ) / <NUM_LIT> , <NUM_LIT> ) ; unsigned Limit ;


<s> BitVector Reserved ( getNumRegs ( ) ) ; Reserved . set ( <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; } if ( ST . hasSGPRInitBug ( ) ) { unsigned NumSGPRs = <STR_LIT> . getNumRegs ( ) ; unsigned Limit = <ISA_LIT>Subtarget :: FIXED_SGPR_COUNT_FOR_INIT_BUG - <NUM_LIT> ; for ( unsigned i = Limit ; i < NumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } } const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned ScratchWaveOffsetReg = MFI -> getScratchWaveOffsetReg ( ) ;

if ( ScratchWaveOffsetReg != <STR_LIT> ) {


<s> return getCommonSubClass ( & <STR_LIT> , RC ) != nullptr ; case <NUM_LIT> : return getCommonSubClass ( & <STR_LIT> , RC ) != nullptr ; case <NUM_LIT> : return getCommonSubClass ( & <STR_LIT> , RC ) != nullptr ; default : llvm_unreachable ( <STR_LIT> ) ; }


<s> bool <ISA_LIT>RegisterInfo :: opCanUseInlineConstant ( unsigned OpType ) const { if ( opCanUseLiteralConstant ( OpType ) ) return true ;


<s> if ( opCanUseLiteralConstant ( OpType ) ) return true ; return OpType == <STR_LIT> ;


<s> return Fn . getFrameInfo ( ) -> hasStackObjects ( ) ;


<s> unsigned BaseIdx = <ISA_LIT>Subtarget :: FIXED_SGPR_COUNT_FOR_INIT_BUG - <NUM_LIT> - <NUM_LIT> ; unsigned BaseReg ( <STR_LIT> . getRegister ( BaseIdx ) ) ; return getMatchingSuperReg ( BaseReg , <STR_LIT> , & <STR_LIT> ) ; } if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { return <STR_LIT> ; } return <STR_LIT> ;


<s> unsigned Idx = <ISA_LIT>Subtarget :: FIXED_SGPR_COUNT_FOR_INIT_BUG - <NUM_LIT> - <NUM_LIT> ; return <STR_LIT> . getRegister ( Idx ) ; }

if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) {


<s> if ( ST . hasSGPRInitBug ( ) ) { unsigned Idx = <ISA_LIT>Subtarget :: FIXED_SGPR_COUNT_FOR_INIT_BUG - <NUM_LIT> - <NUM_LIT> ; return <STR_LIT> . getRegister ( Idx ) ; }


<s> void <ISA_LIT>RegisterInfo :: reserveRegisterTuples ( BitVector & Reserved , unsigned Reg ) const { MCRegAliasIterator R ( Reg , this , true ) ; for ( ; R . isValid ( ) ; ++ R ) Reserved . set ( * R ) ;


<s> return getCommonSubClass ( DefRC , SrcRC ) != nullptr ;


<s> <ISA_LIT>RegisterInfo :: <ISA_LIT>RegisterInfo ( ) : <ISA_LIT>RegisterInfo ( ) {


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; MachineBasicBlock * MBB = MI -> getParent ( ) ; const DebugLoc & DL = MI -> getDebugLoc ( ) ; bool IsStore = MI -> mayStore ( ) ; unsigned Opc = MI -> getOpcode ( ) ; int LoadStoreOp = IsStore ? getOffsetMUBUFStore ( Opc ) : getOffsetMUBUFLoad ( Opc ) ; if ( LoadStoreOp == - <NUM_LIT> ) return false ; const MachineOperand * Reg = TII -> getNamedOperand ( * MI , <STR_LIT> :: vdata ) ; if ( spillVGPRtoAGPR ( ST , * MBB , MI , Index , <NUM_LIT> , Reg -> getReg ( ) , false ) . getInstr ( ) ) return true ; MachineInstrBuilder NewMI = BuildMI ( * MBB , MI , DL , TII -> get ( LoadStoreOp ) ) . add ( * Reg ) . add ( * TII -> getNamedOperand ( * MI , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * MI , <STR_LIT> :: soffset ) ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . cloneMemRefs ( * MI ) ; const MachineOperand * VDataIn = TII -> getNamedOperand ( * MI , <STR_LIT> :: vdata_in ) ;

if ( VDataIn ) NewMI . add ( * VDataIn ) ;


<s> void <ISA_LIT>RegisterInfo :: buildVGPRSpillLoadStore ( SGPRSpillBuilder & SB , int Index , int Offset , bool IsLoad , bool IsKill ) const { MachineFrameInfo & FrameInfo = SB . MF . getFrameInfo ( ) ; assert ( FrameInfo . getStackID ( Index ) != TargetStackID :: SGPRSpill ) ; Register FrameReg = FrameInfo . isFixedObjectIndex ( Index ) && hasBasePointer ( SB . MF ) ? getBaseRegister ( ) : getFrameRegister ( SB . MF ) ; Align Alignment = FrameInfo . getObjectAlign ( Index ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( SB . MF , Index ) ; MachineMemOperand * MMO = SB . MF . getMachineMemOperand ( PtrInfo , IsLoad ? MachineMemOperand :: MOLoad : MachineMemOperand :: MOStore , SB . EltSize , Alignment ) ; if ( IsLoad ) { unsigned Opc = ST . enableFlatScratch ( ) ? <STR_LIT> : <STR_LIT> ; buildSpillLoadStore ( * SB . MBB , SB . MI , SB . DL , Opc , Index , SB . TmpVGPR , false , FrameReg , Offset * SB . EltSize , MMO , SB . RS ) ; } else { unsigned Opc = ST . enableFlatScratch ( ) ? <STR_LIT> : <STR_LIT> ; buildSpillLoadStore ( * SB . MBB , SB . MI , SB . DL , Opc , Index , SB . TmpVGPR , IsKill , FrameReg , Offset * SB . EltSize , MMO , SB . RS ) ; SB . MFI . addToSpilledVGPRs ( <NUM_LIT> ) ; }


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return spillSGPR ( MI , FI , RS , LIS , true ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> SlotIndex UseIdx = LIS -> getInstructionIndex ( Use ) ; SlotIndex DefIdx ; if ( Reg . isVirtual ( ) ) { if ( ! LIS -> hasInterval ( Reg ) ) return nullptr ; LiveInterval & LI = LIS -> getInterval ( Reg ) ; LaneBitmask SubLanes = SubReg ? getSubRegIndexLaneMask ( SubReg ) : MRI . getMaxLaneMaskForVReg ( Reg ) ; VNInfo * V = nullptr ; if ( LI . hasSubRanges ( ) ) { for ( auto & S : LI . subranges ( ) ) { if ( ( S . LaneMask & SubLanes ) == SubLanes ) { V = S . getVNInfoAt ( UseIdx ) ; break ; } }


<s> if ( ! V ) return nullptr ; DefIdx = V -> def ; } else { for ( MCRegUnitIterator Units ( Reg . asMCReg ( ) , this ) ; Units . isValid ( ) ; ++ Units ) { LiveRange & LR = LIS -> getRegUnit ( * Units ) ; if ( VNInfo * V = LR . getVNInfoAt ( UseIdx ) ) { if ( ! DefIdx . isValid ( ) || MDT . dominates ( LIS -> getInstructionFromIndex ( DefIdx ) , LIS -> getInstructionFromIndex ( V -> def ) ) ) DefIdx = V -> def ; } else {


for ( MCRegister Reg : * RC ) if ( MRI . isAllocatable ( Reg ) && ! MRI . isPhysRegUsed ( Reg ) ) return Reg ;


<s> assert ( getRegSizeInBits ( * getPhysRegClass ( Reg ) ) <= <NUM_LIT> ) ; for ( const TargetRegisterClass & RC : { <STR_LIT> , <STR_LIT> , <STR_LIT> } ) { if ( MCPhysReg Super = getMatchingSuperReg ( Reg , <STR_LIT> , & RC ) ) return Super ; } if ( MCPhysReg Super = getMatchingSuperReg ( Reg , <STR_LIT> , & <STR_LIT> ) ) { return Super ; }


<s> if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ;


<s> const TargetRegisterClass * <ISA_LIT>RegisterInfo :: getAGPRClassForBitWidth ( unsigned BitWidth ) const { if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; return ST . needsAlignedVGPRs ( ) ? getAlignedAGPRClassForBitWidth ( BitWidth ) : getAnyAGPRClassForBitWidth ( BitWidth ) ;


<s> static const TargetRegisterClass * getAlignedAGPRClassForBitWidth ( unsigned BitWidth ) { if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ;


<s> if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ;


<s> const uint32_t * <ISA_LIT>RegisterInfo :: getAllAGPRRegMask ( ) const { return <ISA_LIT>_AllAGPRs_RegMask ;


<s> return makeArrayRef ( <STR_LIT> . begin ( ) , ST . getMaxNumSGPRs ( MF ) / <NUM_LIT> ) ;


<s> return makeArrayRef ( <STR_LIT> . begin ( ) , ST . getMaxNumSGPRs ( MF ) ) ;


<s> const uint32_t * <ISA_LIT>RegisterInfo :: getAllVectorRegMask ( ) const {


<s> const uint32_t * <ISA_LIT>RegisterInfo :: getAllVGPRRegMask ( ) const {


<s> static const TargetRegisterClass * getAnyAGPRClassForBitWidth ( unsigned BitWidth ) { if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ;


<s> if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ;


<s> Register <ISA_LIT>RegisterInfo :: getBaseRegister ( ) const { return <STR_LIT> ;


<s> Register <ISA_LIT>RegisterInfo :: getBaseRegister ( ) const {


<s> case CallingConv :: Fast : case CallingConv :: Cold : return ST . hasGFX90AInsts ( ) ? CSR_<ISA_LIT>_GFX90AInsts_RegMask : CSR_<ISA_LIT>_RegMask ;


<s> const uint32_t * <ISA_LIT>RegisterInfo :: getCallPreservedMask ( const MachineFunction & MF , CallingConv :: ID CC ) const { switch ( CC ) { case CallingConv :: C : case CallingConv :: Fast : case CallingConv :: Cold :

return ST . hasGFX90AInsts ( ) ? CSR_<ISA_LIT>_GFX90AInsts_RegMask : CSR_<ISA_LIT>_RegMask ;


<s> const TargetRegisterClass * MatchRC = getMatchingSuperRegClass ( SuperRC , SubRC , SubIdx ) ;

return MatchRC && MatchRC -> hasSubClassEq ( SuperRC ) ? MatchRC : nullptr ;


<s> if ( const RegisterBank * RB = RCOrRB . dyn_cast < const RegisterBank * > ( ) ) return getRegClassForTypeOnBank ( MRI . getType ( MO . getReg ( ) ) , * RB ) ; if ( const auto * RC = RCOrRB . dyn_cast < const TargetRegisterClass * > ( ) ) return getAllocatableClass ( RC ) ; return nullptr ;


<s> if ( isAGPRClass ( RC ) && ! ST . hasGFX90AInsts ( ) ) return getEquivalentVGPRClass ( RC ) ;


<s> const TargetRegisterClass * ARC = getAGPRClassForBitWidth ( Size ) ; assert ( ARC && <STR_LIT> ) ;


<s> const TargetRegisterClass * <ISA_LIT>RegisterInfo :: getEquivalentAGPRClass ( const TargetRegisterClass * SRC ) const { unsigned Size = getRegSizeInBits ( * SRC ) ; const TargetRegisterClass * ARC = getAGPRClassForBitWidth ( Size ) ; assert ( ARC && <STR_LIT> ) ; return ARC ;


<s> unsigned Size = getRegSizeInBits ( * VRC ) ; if ( Size == <NUM_LIT> ) return & <STR_LIT> ; const TargetRegisterClass * SRC = getSGPRClassForBitWidth ( Size ) ; assert ( SRC && <STR_LIT> ) ;


<s> const TargetRegisterClass * <ISA_LIT>RegisterInfo :: getEquivalentVGPRClass ( const TargetRegisterClass * SRC ) const { unsigned Size = getRegSizeInBits ( * SRC ) ;

const TargetRegisterClass * VRC = getVGPRClassForBitWidth ( Size ) ;


<s> int64_t <ISA_LIT>RegisterInfo :: getFrameIndexInstrOffset ( const MachineInstr * MI , int Idx ) const { if ( ! <ISA_LIT>InstrInfo :: isMUBUF ( * MI ) && ! <ISA_LIT>InstrInfo :: isFLATScratch ( * MI ) ) return <NUM_LIT> ; assert ( ( Idx == <STR_LIT> ( MI -> getOpcode ( ) , <STR_LIT> :: vaddr ) || ( Idx == <STR_LIT> ( MI -> getOpcode ( ) , <STR_LIT> :: saddr ) ) ) && <STR_LIT> ) ;


<s> const <ISA_LIT>FrameLowering * TFI = ST . getFrameLowering ( ) ; const <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( FuncInfo -> isEntryFunction ( ) ) { return TFI -> hasFP ( MF ) ? FuncInfo -> getFrameOffsetReg ( ) : Register ( ) ; } return TFI -> hasFP ( MF ) ? FuncInfo -> getFrameOffsetReg ( ) : FuncInfo -> getStackPtrOffsetReg ( ) ;


<s> const <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( FuncInfo -> isEntryFunction ( ) ) { return TFI -> hasFP ( MF ) ? FuncInfo -> getFrameOffsetReg ( ) : Register ( ) ; }

return TFI -> hasFP ( MF ) ? FuncInfo -> getFrameOffsetReg ( ) : FuncInfo -> getStackPtrOffsetReg ( ) ;


<s> const uint32_t * <ISA_LIT>RegisterInfo :: getNoPreservedMask ( ) const {


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> :


<s> return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; default : return - <NUM_LIT> ;


<s> case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; default :


<s> PerVGPRData getPerVGPRData ( ) { PerVGPRData Data ; Data . PerVGPR = IsWave32 ? <NUM_LIT> : <NUM_LIT> ;

Data . NumVGPRs = ( NumSubRegs + ( Data . PerVGPR - <NUM_LIT> ) ) / Data . PerVGPR ;


<s> if ( isVGPRClass ( RC ) ) return getAlignedVGPRClassForBitWidth ( Size ) ; if ( isAGPRClass ( RC ) ) return getAlignedAGPRClassForBitWidth ( Size ) ; if ( isVectorSuperClass ( RC ) ) return getAlignedVectorSuperClassForBitWidth ( Size ) ; return RC ;


<s> return <ISA_LIT>InstPrinter :: getRegisterName ( Reg ) ;


<s> return isWave32 ? & <STR_LIT> : & <STR_LIT> ; case - <NUM_LIT> : return nullptr ; default : return <ISA_LIT>GenRegisterInfo :: getRegClass ( RCID ) ;


<s> case <STR_LIT> : return getBoolRC ( ) ; case <STR_LIT> : return isWave32 ? & <STR_LIT> : & <STR_LIT> ;


<s> const TargetRegisterClass * <ISA_LIT>RegisterInfo :: getRegClassForReg ( const MachineRegisterInfo & MRI , Register Reg ) const {

return Reg . isVirtual ( ) ? MRI . getRegClass ( Reg ) : getPhysRegClass ( Reg ) ;


<s> return getVGPRClassForBitWidth ( std :: max ( <NUM_LIT> , Size ) ) ; case <STR_LIT> : assert ( Size == <NUM_LIT> ) ; return isWave32 ? & <STR_LIT> : & <STR_LIT> ;


<s> case <STR_LIT> : return getSGPRClassForBitWidth ( std :: max ( <NUM_LIT> , Size ) ) ; case <STR_LIT> : return getAGPRClassForBitWidth ( std :: max ( <NUM_LIT> , Size ) ) ; default : llvm_unreachable ( <STR_LIT> ) ; }


<s> const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned Occupancy = ST . getOccupancyWithLocalMemSize ( MFI -> getLDSSize ( ) , MF . getFunction ( ) ) ; switch ( RC -> getID ( ) ) { default : return <ISA_LIT>GenRegisterInfo :: getRegPressureLimit ( RC , MF ) ; case <STR_LIT> :


<s> if ( Idx == <STR_LIT> :: SReg_32 ) return getRegPressureLimit ( & <STR_LIT> , const_cast < MachineFunction & > ( MF ) ) ;


<s> ArrayRef < int16_t > <ISA_LIT>RegisterInfo :: getRegSplitParts ( const TargetRegisterClass * RC , unsigned EltSize ) const { const unsigned RegBitWidth = <STR_LIT> ( * RC -> MC ) ; assert ( RegBitWidth >= <NUM_LIT> && RegBitWidth <= <NUM_LIT> ) ; const unsigned RegDWORDs = RegBitWidth / <NUM_LIT> ; const unsigned EltDWORDs = EltSize / <NUM_LIT> ; assert ( RegSplitParts . size ( ) + <NUM_LIT> >= EltDWORDs ) ; const std :: vector < int16_t > & Parts = RegSplitParts [ EltDWORDs - <NUM_LIT> ] ;

const unsigned NumParts = RegDWORDs / EltDWORDs ;


<s> assert ( RegSplitParts . size ( ) + <NUM_LIT> >= EltDWORDs ) ; const std :: vector < int16_t > & Parts = RegSplitParts [ EltDWORDs - <NUM_LIT> ] ;


<s> static const int Empty [ ] = { - <NUM_LIT> } ;

if ( RegPressureIgnoredUnits [ RegUnit ] ) return Empty ;


<s> const int * <ISA_LIT>RegisterInfo :: getRegUnitPressureSets ( unsigned RegUnit ) const { static const int Empty [ ] = { - <NUM_LIT> } ; if ( RegPressureIgnoredUnits [ RegUnit ] ) return Empty ;

return <ISA_LIT>GenRegisterInfo :: getRegUnitPressureSets ( RegUnit ) ;


<s> Reserved . set ( getSubReg ( Reg , <STR_LIT> ) ) ; Register Low = getSubReg ( Reg , <STR_LIT> ) ; if ( ! <STR_LIT> . contains ( Low ) ) Reserved . set ( Low ) ; } Register ScratchRSrcReg = MFI -> getScratchRSrcReg ( ) ; if ( ScratchRSrcReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , ScratchRSrcReg ) ; } MCRegister StackPtrReg = MFI -> getStackPtrOffsetReg ( ) ; if ( StackPtrReg ) { reserveRegisterTuples ( Reserved , StackPtrReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg , StackPtrReg ) ) ; } MCRegister FrameReg = MFI -> getFrameOffsetReg ( ) ; if ( FrameReg ) { reserveRegisterTuples ( Reserved , FrameReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg , FrameReg ) ) ; } if ( hasBasePointer ( MF ) ) { MCRegister BasePtrReg = getBaseRegister ( ) ; reserveRegisterTuples ( Reserved , BasePtrReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg , BasePtrReg ) ) ; } unsigned MaxNumVGPRs = ST . getMaxNumVGPRs ( MF ) ; unsigned MaxNumAGPRs = MaxNumVGPRs ; unsigned TotalNumVGPRs = <STR_LIT> . getNumRegs ( ) ; if ( ! ST . hasMAIInsts ( ) ) { for ( unsigned i = <NUM_LIT> ; i < MaxNumAGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } } for ( auto Reg : <STR_LIT> ) { Reserved . set ( getSubReg ( Reg , <STR_LIT> ) ) ; } if ( ST . hasGFX90AInsts ( ) ) {


<s> MCRegister <ISA_LIT>RegisterInfo :: getReturnAddressReg ( const MachineFunction & MF ) const {


<s> int64_t <ISA_LIT>RegisterInfo :: getScratchInstrOffset ( const MachineInstr * MI ) const {

assert ( <ISA_LIT>InstrInfo :: isMUBUF ( * MI ) || <ISA_LIT>InstrInfo :: isFLATScratch ( * MI ) ) ;


<s> RC = getVGPRClassForBitWidth ( Size ) ; } else if ( isVectorSuperClass ( RC ) ) { RC = getVectorSuperClassForBitWidth ( Size ) ; } else { RC = getSGPRClassForBitWidth ( Size ) ; } assert ( RC && <STR_LIT> ) ;


<s> unsigned <ISA_LIT>RegisterInfo :: getSubRegFromChannel ( unsigned Channel , unsigned NumRegs ) { assert ( NumRegs < SubRegFromChannelTableWidthMap . size ( ) ) ; unsigned NumRegIndex = SubRegFromChannelTableWidthMap [ NumRegs ] ; assert ( NumRegIndex && <STR_LIT> ) ;

assert ( Channel < SubRegFromChannelTable [ NumRegIndex - <NUM_LIT> ] . size ( ) ) ;


<s> MCRegister <ISA_LIT>RegisterInfo :: getVCC ( ) const { return isWave32 ? <STR_LIT> : <STR_LIT> ;


<s> const TargetRegisterClass * <ISA_LIT>RegisterInfo :: getVectorSuperClassForBitWidth ( unsigned BitWidth ) const {


<s> const TargetRegisterClass * <ISA_LIT>RegisterInfo :: getVectorSuperClassForBitWidth ( unsigned BitWidth ) const { if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ;

return ST . needsAlignedVGPRs ( ) ? getAlignedVectorSuperClassForBitWidth ( BitWidth ) : getAnyVectorSuperClassForBitWidth ( BitWidth ) ;


<s> return ST . needsAlignedVGPRs ( ) ? & <STR_LIT> : & <STR_LIT> ;


<s> const TargetRegisterClass * <ISA_LIT>RegisterInfo :: getVGPR64Class ( ) const { return ST . needsAlignedVGPRs ( ) ? & <STR_LIT> : & <STR_LIT> ;


<s> const TargetRegisterClass * <ISA_LIT>RegisterInfo :: getVGPRClassForBitWidth ( unsigned BitWidth ) const {

if ( BitWidth == <NUM_LIT> ) return & <STR_LIT> ;


<s> const TargetRegisterClass * <ISA_LIT>RegisterInfo :: getVGPRClassForBitWidth ( unsigned BitWidth ) const { if ( BitWidth == <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ;


<s> return MFI . getNumFixedObjects ( ) && shouldRealignStack ( MF ) ;


<s> bool <ISA_LIT>RegisterInfo :: isAGPR ( const MachineRegisterInfo & MRI , Register Reg ) const { const TargetRegisterClass * RC = getRegClassForReg ( MRI , Reg ) ; return RC && isAGPRClass ( RC ) ;


<s> bool <ISA_LIT>RegisterInfo :: isAsmClobberable ( const MachineFunction & MF , MCRegister PhysReg ) const {

return ! MF . getRegInfo ( ) . isReserved ( PhysReg ) ;


<s> switch ( PhysReg ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; default : return false ;


<s> bool <ISA_LIT>RegisterInfo :: isFrameOffsetLegal ( const MachineInstr * MI , Register BaseReg , int64_t Offset ) const { if ( ! <ISA_LIT>InstrInfo :: isMUBUF ( * MI ) && ! <ISA_LIT>InstrInfo :: isFLATScratch ( * MI ) ) return false ; int64_t NewOffset = Offset + getScratchInstrOffset ( MI ) ;

if ( <ISA_LIT>InstrInfo :: isMUBUF ( * MI ) ) return <ISA_LIT>InstrInfo :: isLegalMUBUFImmOffset ( NewOffset ) ;


<s> if ( ! ST . needsAlignedVGPRs ( ) ) return true ;

if ( isVGPRClass ( & RC ) ) return RC . hasSuperClassEq ( getVGPRClassForBitWidth ( getRegSizeInBits ( RC ) ) ) ;


<s> bool <ISA_LIT>RegisterInfo :: isSGPRReg ( const MachineRegisterInfo & MRI , Register Reg ) const {


<s> bool <ISA_LIT>RegisterInfo :: isSGPRReg ( const MachineRegisterInfo & MRI , Register Reg ) const { const TargetRegisterClass * RC ; if ( Reg . isVirtual ( ) ) RC = MRI . getRegClass ( Reg ) ;


<s> bool <ISA_LIT>RegisterInfo :: isVGPR ( const MachineRegisterInfo & MRI , Register Reg ) const {

const TargetRegisterClass * RC = getRegClassForReg ( MRI , Reg ) ;


<s> const TargetRegisterClass * RC = getRegClassForReg ( MRI , Reg ) ; return RC && isVGPRClass ( RC ) ;


<s> MachineBasicBlock :: iterator Ins = MBB -> begin ( ) ; DebugLoc DL ; if ( Ins != MBB -> end ( ) ) DL = Ins -> getDebugLoc ( ) ; MachineFunction * MF = MBB -> getParent ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; unsigned MovOpc = ST . enableFlatScratch ( ) ? <STR_LIT> : <STR_LIT> ; Register BaseReg = MRI . createVirtualRegister ( ST . enableFlatScratch ( ) ? & <STR_LIT> : & <STR_LIT> ) ; if ( Offset == <NUM_LIT> ) { BuildMI ( * MBB , Ins , DL , TII -> get ( MovOpc ) , BaseReg ) . addFrameIndex ( FrameIdx ) ; return BaseReg ;


<s> bool <ISA_LIT>RegisterInfo :: needsFrameBaseReg ( MachineInstr * MI , int64_t Offset ) const {

if ( ! <ISA_LIT>InstrInfo :: isMUBUF ( * MI ) && ! <ISA_LIT>InstrInfo :: isFLATScratch ( * MI ) ) return false ;


<s> if ( ! <ISA_LIT>InstrInfo :: isMUBUF ( * MI ) && ! <ISA_LIT>InstrInfo :: isFLATScratch ( * MI ) ) return false ; int64_t FullOffset = Offset + getScratchInstrOffset ( MI ) ;

if ( <ISA_LIT>InstrInfo :: isMUBUF ( * MI ) ) return ! <ISA_LIT>InstrInfo :: isLegalMUBUFImmOffset ( FullOffset ) ;


<s> bool <ISA_LIT>RegisterInfo :: opCanUseInlineConstant ( unsigned OpType ) const {

if ( OpType >= <STR_LIT> && OpType <= <STR_LIT> ) return ! ST . hasMFMAInlineLiteralBug ( ) ;


<s> bool <ISA_LIT>RegisterInfo :: opCanUseLiteralConstant ( unsigned OpType ) const { return OpType >= <STR_LIT> && OpType <= <STR_LIT> ;


<s> if ( RS -> isRegUsed ( <STR_LIT> ) ) MI -> emitError ( <STR_LIT> ) ; if ( TmpVGPRLive ) TRI . buildVGPRSpillLoadStore ( * this , TmpVGPRIndex , <NUM_LIT> , false , false ) ; auto I = BuildMI ( * MBB , MI , DL , TII . get ( NotOpc ) , ExecReg ) . addReg ( ExecReg ) ; if ( ! TmpVGPRLive ) I . addReg ( TmpVGPR , RegState :: ImplicitDefine ) ; I -> getOperand ( <NUM_LIT> ) . setIsDead ( true ) ; TRI . buildVGPRSpillLoadStore ( * this , TmpVGPRIndex , <NUM_LIT> , false ) ;


<s> TRI . buildVGPRSpillLoadStore ( * this , Index , Offset , IsLoad ) ; auto Not1 = BuildMI ( * MBB , MI , DL , TII . get ( NotOpc ) , ExecReg ) . addReg ( ExecReg ) ; Not1 -> getOperand ( <NUM_LIT> ) . setIsDead ( ) ;


<s> bool <ISA_LIT>RegisterInfo :: requiresFrameIndexScavenging ( const MachineFunction & MF ) const {


<s> const MachineFrameInfo & MFI = Fn . getFrameInfo ( ) ; return MFI . hasStackObjects ( ) || MFI . hasCalls ( ) ;


<s> bool <ISA_LIT>RegisterInfo :: requiresVirtualBaseRegisters ( const MachineFunction & ) const { return true ;


<s> unsigned BaseIdx = alignDown ( ST . getMaxNumSGPRs ( MF ) , <NUM_LIT> ) - <NUM_LIT> ;

MCRegister BaseReg ( <STR_LIT> . getRegister ( BaseIdx ) ) ;


<s> MCRegister <ISA_LIT>RegisterInfo :: reservedPrivateSegmentBufferReg ( const MachineFunction & MF ) const { unsigned BaseIdx = alignDown ( ST . getMaxNumSGPRs ( MF ) , <NUM_LIT> ) - <NUM_LIT> ; MCRegister BaseReg ( <STR_LIT> . getRegister ( BaseIdx ) ) ; return getMatchingSuperReg ( BaseReg , <STR_LIT> , & <STR_LIT> ) ;


<s> MCRegAliasIterator R ( Reg , this , true ) ; for ( ; R . isValid ( ) ; ++ R ) Reserved . set ( * R ) ;


<s> for ( const MachineOperand & MO : MI . operands ( ) ) { if ( MO . isFI ( ) ) { if ( SeenFI ) llvm_unreachable ( <STR_LIT> ) ; SeenFI = true ; } } MachineOperand * FIOp = TII -> getNamedOperand ( MI , IsFlat ? <STR_LIT> :: saddr : <STR_LIT> :: vaddr ) ; MachineOperand * OffsetOp = TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) ;

int64_t NewOffset = OffsetOp -> getImm ( ) + Offset ;


<s> TRI . buildVGPRSpillLoadStore ( * this , TmpVGPRIndex , <NUM_LIT> , true , false ) ; auto I = BuildMI ( * MBB , MI , DL , TII . get ( MovOpc ) , ExecReg ) . addReg ( SavedExecReg , RegState :: Kill ) ; if ( ! TmpVGPRLive ) { I . addReg ( TmpVGPR , RegState :: ImplicitKill ) ; } } else { TRI . buildVGPRSpillLoadStore ( * this , TmpVGPRIndex , <NUM_LIT> , true , false ) ; auto I = BuildMI ( * MBB , MI , DL , TII . get ( NotOpc ) , ExecReg ) . addReg ( ExecReg ) ; if ( ! TmpVGPRLive ) I . addReg ( TmpVGPR , RegState :: ImplicitKill ) ; I -> getOperand ( <NUM_LIT> ) . setIsDead ( true ) ;

if ( TmpVGPRLive ) TRI . buildVGPRSpillLoadStore ( * this , TmpVGPRIndex , <NUM_LIT> , true ) ;


<s> for ( unsigned i = <NUM_LIT> , e = SB . NumSubRegs ; i < e ; ++ i ) { Register SubReg = SB . NumSubRegs == <NUM_LIT> ? SB . SuperReg : Register ( getSubReg ( SB . SuperReg , SB . SplitParts [ i ] ) ) ; <ISA_LIT>MachineFunctionInfo :: SpilledReg Spill = VGPRSpills [ i ] ; auto MIB = BuildMI ( * SB . MBB , MI , SB . DL , SB . TII . get ( <STR_LIT> ) , SubReg ) . addReg ( Spill . VGPR ) . addImm ( Spill . Lane ) ; if ( SB . NumSubRegs > <NUM_LIT> && i == <NUM_LIT> ) MIB . addReg ( SB . SuperReg , RegState :: ImplicitDefine ) ; if ( LIS ) { if ( i == e - <NUM_LIT> ) LIS -> ReplaceMachineInstrInMaps ( * MI , * MIB ) ; else LIS -> InsertMachineInstrInMaps ( * MIB ) ; } } } else { SB . prepare ( ) ; auto PVD = SB . getPerVGPRData ( ) ; for ( unsigned Offset = <NUM_LIT> ; Offset < PVD . NumVGPRs ; ++ Offset ) { SB . readWriteTmpVGPR ( Offset , true ) ;

for ( unsigned i = Offset * PVD . PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PVD . PerVGPR , SB . NumSubRegs ) ;


<s> SB . prepare ( ) ; auto PVD = SB . getPerVGPRData ( ) ; for ( unsigned Offset = <NUM_LIT> ; Offset < PVD . NumVGPRs ; ++ Offset ) { SB . readWriteTmpVGPR ( Offset , true ) ; for ( unsigned i = Offset * PVD . PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PVD . PerVGPR , SB . NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = SB . NumSubRegs == <NUM_LIT> ? SB . SuperReg : Register ( getSubReg ( SB . SuperReg , SB . SplitParts [ i ] ) ) ; bool LastSubReg = ( i + <NUM_LIT> == e ) ; auto MIB = BuildMI ( * SB . MBB , MI , SB . DL , SB . TII . get ( <STR_LIT> ) , SubReg ) . addReg ( SB . TmpVGPR , getKillRegState ( LastSubReg ) ) . addImm ( i ) ; if ( SB . NumSubRegs > <NUM_LIT> && i == <NUM_LIT> ) MIB . addReg ( SB . SuperReg , RegState :: ImplicitDefine ) ; if ( LIS ) {

if ( i == e - <NUM_LIT> ) LIS -> ReplaceMachineInstrInMaps ( * MI , * MIB ) ;


<s> void setMI ( MachineBasicBlock * NewMBB , MachineBasicBlock :: iterator NewMI ) { assert ( MBB -> getParent ( ) == & MF ) ; MI = NewMI ; MBB = NewMBB ;


<s> SplitParts = TRI . getRegSplitParts ( RC , EltSize ) ; NumSubRegs = SplitParts . empty ( ) ? <NUM_LIT> : SplitParts . size ( ) ; if ( IsWave32 ) { ExecReg = <STR_LIT> ; MovOpc = <STR_LIT> ; NotOpc = <STR_LIT> ; } else { ExecReg = <STR_LIT> ; MovOpc = <STR_LIT> ; NotOpc = <STR_LIT> ; } assert ( SuperReg != <STR_LIT> && <STR_LIT> ) ;

assert ( SuperReg != <STR_LIT> && SuperReg != <STR_LIT> && SuperReg != <STR_LIT> && <STR_LIT> ) ;


<s> unsigned SrcSize = getRegSizeInBits ( * SrcRC ) ; unsigned DstSize = getRegSizeInBits ( * DstRC ) ;

unsigned NewSize = getRegSizeInBits ( * NewRC ) ;


<s> unsigned DstSize = getRegSizeInBits ( * DstRC ) ; unsigned NewSize = getRegSizeInBits ( * NewRC ) ;

if ( SrcSize <= <NUM_LIT> || DstSize <= <NUM_LIT> ) return true ;


<s> bool <ISA_LIT>RegisterInfo :: shouldRealignStack ( const MachineFunction & MF ) const { const <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( Info -> isEntryFunction ( ) ) return false ;

return TargetRegisterInfo :: shouldRealignStack ( MF ) ;


<s> bool <ISA_LIT>RegisterInfo :: shouldRealignStack ( const MachineFunction & MF ) const { const <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;

if ( Info -> isEntryFunction ( ) ) return false ;


<s> RegPressureIgnoredUnits . set ( * MCRegUnitIterator ( MCRegister :: from ( <STR_LIT> ) , this ) ) ; for ( auto Reg : <STR_LIT> ) RegPressureIgnoredUnits . set ( * MCRegUnitIterator ( Reg , this ) ) ; static llvm :: once_flag InitializeRegSplitPartsFlag ; static auto InitializeRegSplitPartsOnce = [ this ] ( ) { for ( unsigned Idx = <NUM_LIT> , E = getNumSubRegIndices ( ) - <NUM_LIT> ; Idx < E ; ++ Idx ) { unsigned Size = getSubRegIdxSize ( Idx ) ; if ( Size & <NUM_LIT> ) continue ; std :: vector < int16_t > & Vec = RegSplitParts [ Size / <NUM_LIT> - <NUM_LIT> ] ; unsigned Pos = getSubRegIdxOffset ( Idx ) ; if ( Pos % Size ) continue ; Pos /= Size ;


<s> } MI = RestoreMBB . end ( ) ; SB . setMI ( & RestoreMBB , MI ) ; for ( unsigned Offset = <NUM_LIT> ; Offset < PVD . NumVGPRs ; ++ Offset ) { for ( unsigned i = Offset * PVD . PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PVD . PerVGPR , SB . NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = SB . NumSubRegs == <NUM_LIT> ? SB . SuperReg : Register ( getSubReg ( SB . SuperReg , SB . SplitParts [ i ] ) ) ; bool LastSubReg = ( i + <NUM_LIT> == e ) ; auto MIB = BuildMI ( * SB . MBB , MI , SB . DL , SB . TII . get ( <STR_LIT> ) , SubReg ) . addReg ( SB . TmpVGPR , getKillRegState ( LastSubReg ) ) . addImm ( i ) ;

if ( SB . NumSubRegs > <NUM_LIT> && i == <NUM_LIT> ) MIB . addReg ( SB . SuperReg , RegState :: ImplicitDefine ) ;


<s> if ( i + <NUM_LIT> == SB . NumSubRegs ) SuperKillState |= getKillRegState ( SB . IsKill ) ; WriteLane . addReg ( SB . SuperReg , RegState :: Implicit | SuperKillState ) ; } } } MI = RestoreMBB . end ( ) ; SB . setMI ( & RestoreMBB , MI ) ; for ( unsigned Offset = <NUM_LIT> ; Offset < PVD . NumVGPRs ; ++ Offset ) { for ( unsigned i = Offset * PVD . PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PVD . PerVGPR , SB . NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = SB . NumSubRegs == <NUM_LIT> ? SB . SuperReg : Register ( getSubReg ( SB . SuperReg , SB . SplitParts [ i ] ) ) ; bool LastSubReg = ( i + <NUM_LIT> == e ) ; auto MIB = BuildMI ( * SB . MBB , MI , SB . DL , SB . TII . get ( <STR_LIT> ) , SubReg ) . addReg ( SB . TmpVGPR , getKillRegState ( LastSubReg ) ) . addImm ( i ) ; if ( SB . NumSubRegs > <NUM_LIT> && i == <NUM_LIT> ) MIB . addReg ( SB . SuperReg , RegState :: ImplicitDefine ) ; } }


<s> } else { SB . prepare ( ) ; unsigned SubKillState = getKillRegState ( ( SB . NumSubRegs == <NUM_LIT> ) && SB . IsKill ) ; auto PVD = SB . getPerVGPRData ( ) ; for ( unsigned Offset = <NUM_LIT> ; Offset < PVD . NumVGPRs ; ++ Offset ) { unsigned TmpVGPRFlags = RegState :: Undef ; for ( unsigned i = Offset * PVD . PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PVD . PerVGPR , SB . NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = SB . NumSubRegs == <NUM_LIT> ? SB . SuperReg : Register ( getSubReg ( SB . SuperReg , SB . SplitParts [ i ] ) ) ; MachineInstrBuilder WriteLane = BuildMI ( * SB . MBB , MI , SB . DL , SB . TII . get ( <STR_LIT> ) , SB . TmpVGPR ) . addReg ( SubReg , SubKillState ) . addImm ( i % PVD . PerVGPR ) . addReg ( SB . TmpVGPR , TmpVGPRFlags ) ; TmpVGPRFlags = <NUM_LIT> ; if ( LIS ) { if ( i == <NUM_LIT> ) LIS -> ReplaceMachineInstrInMaps ( * MI , * WriteLane ) ; else LIS -> InsertMachineInstrInMaps ( * WriteLane ) ; } if ( SB . NumSubRegs > <NUM_LIT> ) { unsigned SuperKillState = <NUM_LIT> ; if ( i + <NUM_LIT> == SB . NumSubRegs ) SuperKillState |= getKillRegState ( SB . IsKill ) ; WriteLane . addReg ( SB . SuperReg , RegState :: Implicit | SuperKillState ) ; } } SB . readWriteTmpVGPR ( Offset , false ) ; } SB . restore ( ) ; } MI -> eraseFromParent ( ) ; SB . MFI . addToSpilledSGPRs ( SB . NumSubRegs ) ; if ( LIS ) LIS -> removeAllRegUnitsForPhysReg ( SB . SuperReg ) ;


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; MCPhysReg Reg = MFI -> getVGPRToAGPRSpill ( Index , Lane ) ; if ( Reg == <STR_LIT> ) return MachineInstrBuilder ( ) ; bool IsStore = MI -> mayStore ( ) ; MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; auto * TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( MRI . getTargetRegisterInfo ( ) ) ; unsigned Dst = IsStore ? Reg : ValueReg ; unsigned Src = IsStore ? ValueReg : Reg ; bool IsVGPR = TRI -> isVGPR ( MRI , Reg ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; if ( IsVGPR == TRI -> isVGPR ( MRI , ValueReg ) ) { auto CopyMIB = BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Src , getKillRegState ( IsKill ) ) ; CopyMIB -> setAsmPrinterFlag ( MachineInstr :: ReloadReuse ) ; return CopyMIB ; }

unsigned Opc = ( IsStore ^ IsVGPR ) ? <STR_LIT> : <STR_LIT> ;


<s> if ( LoadStoreOp == - <NUM_LIT> ) return false ; const MachineOperand * Reg = TII -> getNamedOperand ( * MI , <STR_LIT> :: vdata ) ;

MachineInstrBuilder NewMI = BuildMI ( * MBB , MI , DL , TII -> get ( LoadStoreOp ) ) . add ( * Reg ) . add ( * TII -> getNamedOperand ( * MI , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * MI , <STR_LIT> :: soffset ) ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . setMemRefs ( MI -> memoperands_begin ( ) , MI -> memoperands_end ( ) ) ;


<s> unsigned Opc = MI -> getOpcode ( ) ; int LoadStoreOp = IsStore ? getOffsetMUBUFStore ( Opc ) : getOffsetMUBUFLoad ( Opc ) ; if ( LoadStoreOp == - <NUM_LIT> ) return false ; const MachineOperand * Reg = TII -> getNamedOperand ( * MI , <STR_LIT> :: vdata ) ; MachineInstrBuilder NewMI = BuildMI ( * MBB , MI , DL , TII -> get ( LoadStoreOp ) ) . add ( * Reg ) . add ( * TII -> getNamedOperand ( * MI , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * MI , <STR_LIT> :: soffset ) ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . setMemRefs ( MI -> memoperands_begin ( ) , MI -> memoperands_end ( ) ) ; const MachineOperand * VDataIn = TII -> getNamedOperand ( * MI , <STR_LIT> :: vdata_in ) ; if ( VDataIn ) NewMI . add ( * VDataIn ) ;


<s> for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i != e ; ++ i , Offset += EltSize ) { unsigned SubReg = NumSubRegs == <NUM_LIT> ? ValueReg : getSubReg ( ValueReg , getSubRegFromChannel ( i ) ) ; unsigned SOffsetRegState = <NUM_LIT> ; unsigned SrcDstRegState = getDefRegState ( ! IsStore ) ; if ( i + <NUM_LIT> == e ) { SOffsetRegState |= getKillRegState ( Scavenged ) ; SrcDstRegState |= getKillRegState ( IsKill ) ; } MachinePointerInfo PInfo = BasePtrInfo . getWithOffset ( EltSize * i ) ; MachineMemOperand * NewMMO = MF -> getMachineMemOperand ( PInfo , MMO -> getFlags ( ) , EltSize , MinAlign ( Align , EltSize * i ) ) ; auto MIB = BuildMI ( * MBB , MI , DL , Desc ) . addReg ( SubReg , getDefRegState ( ! IsStore ) | getKillRegState ( IsKill ) ) . addReg ( ScratchRsrcReg ) . addReg ( SOffset , SOffsetRegState ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( NewMMO ) ; if ( NumSubRegs > <NUM_LIT> ) MIB . addReg ( ValueReg , RegState :: Implicit | SrcDstRegState ) ; } if ( ScratchOffsetRegDelta != <NUM_LIT> ) {

BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , ScratchOffsetReg ) . addReg ( ScratchOffsetReg ) . addImm ( ScratchOffsetRegDelta ) ;


<s> case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ;


<s> case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> :


<s> & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , } ;

for ( const TargetRegisterClass * BaseClass : BaseClasses ) {


<s> if ( Idx == getVGPRPressureSet ( ) ) return getRegPressureLimit ( & <STR_LIT> , const_cast < MachineFunction & > ( MF ) ) ; if ( Idx == getSGPRPressureSet ( ) ) return getRegPressureLimit ( & <STR_LIT> , const_cast < MachineFunction & > ( MF ) ) ;

return <ISA_LIT>RegisterInfo :: getRegPressureSetLimit ( MF , Idx ) ;