<s> unsigned DestReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; SmallVector < MachineInstr * , <NUM_LIT> > Users ; for ( auto & User : MRI . use_nodbg_instructions ( DestReg ) ) { if ( ( User . getOpcode ( ) == <STR_LIT> ) || ( User . getOpcode ( ) == <STR_LIT> ) ) { Users . push_back ( & User ) ; Worklist . insert ( & User ) ; } } for ( auto & U : Users ) U -> getOperand ( <NUM_LIT> ) . setReg ( RI . getVCC ( ) ) ; CopyToDelete . push_back ( & MI ) ; } else Worklist . insert ( & MI ) ; } if ( MI . findRegisterDefOperandIdx ( <STR_LIT> , false , false , & RI ) != - <NUM_LIT> ) break ; } for ( auto & Copy : CopyToDelete ) Copy -> eraseFromParent ( ) ;


<s> if ( ! memOpsHaveSameBaseOperands ( BaseOps0 , BaseOps1 ) ) return false ; if ( ! MIa . hasOneMemOperand ( ) || ! MIb . hasOneMemOperand ( ) ) { return false ; } unsigned Width0 = MIa . memoperands ( ) . front ( ) -> getSize ( ) ; unsigned Width1 = MIb . memoperands ( ) . front ( ) -> getSize ( ) ;

return offsetsDoNotOverlap ( Width0 , Offset0 , Width1 , Offset1 ) ;


<s> int64_t Offset0 , Offset1 ; bool Offset0IsScalable , Offset1IsScalable ; if ( ! getMemOperandsWithOffset ( MIa , BaseOps0 , Offset0 , Offset0IsScalable , & RI ) || ! getMemOperandsWithOffset ( MIb , BaseOps1 , Offset1 , Offset1IsScalable , & RI ) ) return false ;

if ( ! memOpsHaveSameBaseOperands ( BaseOps0 , BaseOps1 ) ) return false ;


<s> case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; default :


<s> case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; default :


<s> case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ;


<s> case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ;


<s> static unsigned getIndirectVGPRWritePseudoOpc ( unsigned VecSize ) { switch ( VecSize ) { case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ;


<s> static const int16_t Sub0_15 [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , } ; static const int16_t Sub0_15_64 [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , } ; unsigned SelOp = <STR_LIT> ; const TargetRegisterClass * EltRC = & <STR_LIT> ; const int16_t * SubIndices = Sub0_15 ; int NElts = DstSize / <NUM_LIT> ; if ( Pred == SCC_TRUE ) { if ( NElts % <NUM_LIT> ) { SelOp = <STR_LIT> ; EltRC = & <STR_LIT> ; } else { SelOp = <STR_LIT> ; EltRC = & <STR_LIT> ; SubIndices = Sub0_15_64 ; NElts /= <NUM_LIT> ; } } MachineInstrBuilder MIB = BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) ; I = MIB -> getIterator ( ) ; SmallVector < Register , <NUM_LIT> > Regs ; for ( int Idx = <NUM_LIT> ; Idx != NElts ; ++ Idx ) { Register DstElt = MRI . createVirtualRegister ( EltRC ) ; Regs . push_back ( DstElt ) ;


<s> bool <ISA_LIT>InstrInfo :: isSchedulingBoundary ( const MachineInstr & MI , const MachineBasicBlock * MBB , const MachineFunction & MF ) const { return TargetInstrInfo :: isSchedulingBoundary ( MI , MBB , MF ) || MI . modifiesRegister ( <STR_LIT> , & RI ) || MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || changesVGPRIndexingMode ( MI ) ;


<s> bool <ISA_LIT>InstrInfo :: isSchedulingBoundary ( const MachineInstr & MI , const MachineBasicBlock * MBB , const MachineFunction & MF ) const {

return TargetInstrInfo :: isSchedulingBoundary ( MI , MBB , MF ) || MI . modifiesRegister ( <STR_LIT> , & RI ) || MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || changesVGPRIndexingMode ( MI ) ;


<s> const MCInstrDesc & OpDesc = get ( getSGPRSpillRestoreOpcode ( SpillSize ) ) ; if ( DestReg . isVirtual ( ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( DestReg , & <STR_LIT> ) ; } if ( RI . spillSGPRToVGPR ( ) ) FrameInfo . setStackID ( FrameIndex , TargetStackID :: SGPRSpill ) ; BuildMI ( MBB , MI , DL , OpDesc , DestReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ; return ; }

unsigned Opcode = RI . hasAGPRs ( RC ) ? getAGPRSpillRestoreOpcode ( SpillSize ) : getVGPRSpillRestoreOpcode ( SpillSize ) ;


<s> for ( size_t I = <NUM_LIT> , E = BaseOps1 . size ( ) ; I < E ; ++ I ) if ( ! BaseOps1 [ I ] -> isIdenticalTo ( * BaseOps2 [ I ] ) ) return false ; return true ;


<s> auto Base1 = MO1 -> getValue ( ) ; auto Base2 = MO2 -> getValue ( ) ; if ( ! Base1 || ! Base2 ) return false ; const MachineFunction & MF = * MI1 . getParent ( ) -> getParent ( ) ; const DataLayout & DL = MF . getFunction ( ) . getParent ( ) -> getDataLayout ( ) ;

Base1 = GetUnderlyingObject ( Base1 , DL ) ;


<s> if ( memOpsHaveSameBaseOperands ( BaseOps1 , BaseOps2 ) ) return true ; if ( ! MI1 . hasOneMemOperand ( ) || ! MI2 . hasOneMemOperand ( ) ) return false ; auto MO1 = * MI1 . memoperands_begin ( ) ; auto MO2 = * MI2 . memoperands_begin ( ) ;

if ( MO1 -> getAddrSpace ( ) != MO2 -> getAddrSpace ( ) ) return false ;


<s> MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( SrcReg , & <STR_LIT> ) ; } BuildMI ( MBB , MI , DL , OpDesc ) . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ; if ( RI . spillSGPRToVGPR ( ) ) FrameInfo . setStackID ( FrameIndex , TargetStackID :: SGPRSpill ) ; return ; } unsigned Opcode = RI . hasAGPRs ( RC ) ? getAGPRSpillSaveOpcode ( SpillSize ) : getVGPRSpillSaveOpcode ( SpillSize ) ; MFI -> setHasSpilledVGPRs ( ) ; auto MIB = BuildMI ( MBB , MI , DL , get ( Opcode ) ) ;


<s> if ( NonRegOp . isImm ( ) ) RegOp . ChangeToImmediate ( NonRegOp . getImm ( ) ) ; else if ( NonRegOp . isFI ( ) ) RegOp . ChangeToFrameIndex ( NonRegOp . getIndex ( ) ) ; else return nullptr ;

NonRegOp . ChangeToRegister ( Reg , false , false , IsKill , IsDead , IsUndef , IsDebug ) ;


<s> TIDReg = RI . findUnusedRegister ( MF -> getRegInfo ( ) , & <STR_LIT> , * MF ) ; if ( TIDReg == <STR_LIT> ) return TIDReg ; if ( ! <STR_LIT> ( MF -> getFunction ( ) . getCallingConv ( ) ) && WorkGroupSize > WavefrontSize ) { Register TIDIGXReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; Register TIDIGYReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; Register TIDIGZReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; Register InputPtrReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; for ( unsigned Reg : { TIDIGXReg , TIDIGYReg , TIDIGZReg } ) { if ( ! Entry . isLiveIn ( Reg ) ) Entry . addLiveIn ( Reg ) ; } RS -> enterBasicBlock ( Entry ) ; unsigned STmp0 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; unsigned STmp1 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp0 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Z ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Y ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( STmp1 ) . addReg ( STmp0 ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp1 ) . addReg ( TIDIGXReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp0 ) . addReg ( TIDIGYReg ) . addReg ( TIDReg ) ; getAddNoCarry ( Entry , Insert , DL , TIDReg ) . addReg ( TIDReg ) . addReg ( TIDIGZReg ) . addImm ( <NUM_LIT> ) ; } else { BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( - <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( - <NUM_LIT> ) . addReg ( TIDReg ) ; } BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( <NUM_LIT> ) . addReg ( TIDReg ) ; MFI -> setTIDReg ( TIDReg ) ; } unsigned LDSOffset = MFI -> getLDSSize ( ) + ( FrameOffset * WorkGroupSize ) ; getAddNoCarry ( MBB , MI , DL , TmpReg ) . addImm ( LDSOffset ) . addReg ( TIDReg ) . addImm ( <NUM_LIT> ) ; return TmpReg ;


<s> unsigned SaveExecOpc = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; unsigned XorTermOpc = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; unsigned AndOpc = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; const auto * BoolXExecRC = TRI -> getRegClass ( <STR_LIT> ) ; MachineBasicBlock :: iterator I = LoopBB . begin ( ) ; Register VRsrc = Rsrc . getReg ( ) ; unsigned VRsrcUndef = getUndefRegState ( Rsrc . isUndef ( ) ) ; Register SaveExec = MRI . createVirtualRegister ( BoolXExecRC ) ; Register CondReg0 = MRI . createVirtualRegister ( BoolXExecRC ) ; Register CondReg1 = MRI . createVirtualRegister ( BoolXExecRC ) ; Register AndCond = MRI . createVirtualRegister ( BoolXExecRC ) ; Register SRsrcSub0 = MRI . createVirtualRegister ( & <STR_LIT> ) ; Register SRsrcSub1 = MRI . createVirtualRegister ( & <STR_LIT> ) ; Register SRsrcSub2 = MRI . createVirtualRegister ( & <STR_LIT> ) ; Register SRsrcSub3 = MRI . createVirtualRegister ( & <STR_LIT> ) ; Register SRsrc = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub0 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub1 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub2 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ;

BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub3 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ;


<s> BuildMI ( MBB , MI , MI . getDebugLoc ( ) , TII . get ( <STR_LIT> ) , Zero64 ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , MI , MI . getDebugLoc ( ) , TII . get ( <STR_LIT> ) , SRsrcFormatLo ) . addImm ( RsrcDataFormat & <NUM_LIT> ) ; BuildMI ( MBB , MI , MI . getDebugLoc ( ) , TII . get ( <STR_LIT> ) , SRsrcFormatHi ) . addImm ( RsrcDataFormat >> <NUM_LIT> ) ; BuildMI ( MBB , MI , MI . getDebugLoc ( ) , TII . get ( <STR_LIT> ) , NewSRsrc ) . addReg ( Zero64 ) . addImm ( <STR_LIT> ) . addReg ( SRsrcFormatLo ) . addImm ( <STR_LIT> ) . addReg ( SRsrcFormatHi ) . addImm ( <STR_LIT> ) ; return std :: make_tuple ( RsrcPtr , NewSRsrc ) ;


<s> const MCInstrDesc & Desc = MI . getDesc ( ) ; unsigned SGPRReg = findImplicitSGPRRead ( MI ) ; if ( SGPRReg != <STR_LIT> ) return SGPRReg ; unsigned UsedSGPRs [ <NUM_LIT> ] = { <STR_LIT> } ; const MachineRegisterInfo & MRI = MI . getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) { int Idx = OpIndices [ i ] ; if ( Idx == - <NUM_LIT> ) break ; const MachineOperand & MO = MI . getOperand ( Idx ) ; if ( ! MO . isReg ( ) ) continue ; const TargetRegisterClass * OpRC = RI . getRegClass ( Desc . OpInfo [ Idx ] . RegClass ) ; bool IsRequiredSGPR = RI . isSGPRClass ( OpRC ) ; if ( IsRequiredSGPR ) return MO . getReg ( ) ; Register Reg = MO . getReg ( ) ; const TargetRegisterClass * RegRC = MRI . getRegClass ( Reg ) ;

if ( RI . isSGPRClass ( RegRC ) ) UsedSGPRs [ i ] = Reg ;


<s> if ( ST . hasAddNoCarry ( ) ) return BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DestReg ) ;

Register UnusedCarry = RS . scavengeRegister ( RI . getBoolRC ( ) , I , <NUM_LIT> , false ) ;


<s> if ( ! UnusedCarry . isValid ( ) ) report_fatal_error ( <STR_LIT> ) ; return BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DestReg ) . addReg ( UnusedCarry , RegState :: Define | RegState :: Dead ) ;


<s> unsigned <ISA_LIT>InstrInfo :: insertEQ ( MachineBasicBlock * MBB , MachineBasicBlock :: iterator I , const DebugLoc & DL , unsigned SrcReg , int Value ) const { MachineRegisterInfo & MRI = MBB -> getParent ( ) -> getRegInfo ( ) ; Register Reg = MRI . createVirtualRegister ( RI . getBoolRC ( ) ) ; BuildMI ( * MBB , I , DL , get ( <STR_LIT> ) , Reg ) . addImm ( Value ) . addReg ( SrcReg ) ;


<s> unsigned <ISA_LIT>InstrInfo :: insertEQ ( MachineBasicBlock * MBB , MachineBasicBlock :: iterator I , const DebugLoc & DL , unsigned SrcReg , int Value ) const {

MachineRegisterInfo & MRI = MBB -> getParent ( ) -> getRegInfo ( ) ;


<s> MachineRegisterInfo & MRI = MBB -> getParent ( ) -> getRegInfo ( ) ; Register Reg = MRI . createVirtualRegister ( RI . getBoolRC ( ) ) ; BuildMI ( * MBB , I , DL , get ( <STR_LIT> ) , Reg ) . addImm ( Value ) . addReg ( SrcReg ) ; return Reg ;


<s> BranchPredicate Pred = static_cast < BranchPredicate > ( Cond [ <NUM_LIT> ] . getImm ( ) ) ; if ( Pred == VCCZ || Pred == SCC_FALSE ) { Pred = static_cast < BranchPredicate > ( - Pred ) ; std :: swap ( TrueReg , FalseReg ) ; } MachineRegisterInfo & MRI = MBB . getParent ( ) -> getRegInfo ( ) ; const TargetRegisterClass * DstRC = MRI . getRegClass ( DstReg ) ; unsigned DstSize = RI . getRegSizeInBits ( * DstRC ) ; if ( DstSize == <NUM_LIT> ) { unsigned SelOp = Pred == SCC_TRUE ? <STR_LIT> : <STR_LIT> ; MachineInstr * Select = BuildMI ( MBB , I , DL , get ( SelOp ) , DstReg ) . addReg ( FalseReg ) . addReg ( TrueReg ) ; preserveCondRegFlags ( Select -> getOperand ( <NUM_LIT> ) , Cond [ <NUM_LIT> ] ) ; return ; } if ( DstSize == <NUM_LIT> && Pred == SCC_TRUE ) { MachineInstr * Select = BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addReg ( FalseReg ) . addReg ( TrueReg ) ; preserveCondRegFlags ( Select -> getOperand ( <NUM_LIT> ) , Cond [ <NUM_LIT> ] ) ; return ; } static const int16_t Sub0_15 [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , } ; static const int16_t Sub0_15_64 [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , } ;


<s> } case <ISA_LIT>InstrInfo :: SCC_FALSE : { Register SReg = MRI . createVirtualRegister ( BoolXExecRC ) ; BuildMI ( MBB , I , DL , get ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , SReg ) . addImm ( <NUM_LIT> ) . addImm ( - <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addImm ( <NUM_LIT> ) . addReg ( FalseReg ) . addImm ( <NUM_LIT> ) . addReg ( TrueReg ) . addReg ( SReg ) ; break ; } case <ISA_LIT>InstrInfo :: VCCNZ : { MachineOperand RegOp = Cond [ <NUM_LIT> ] ; RegOp . setImplicit ( false ) ; Register SReg = MRI . createVirtualRegister ( BoolXExecRC ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , SReg ) . add ( RegOp ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addImm ( <NUM_LIT> ) . addReg ( FalseReg ) . addImm ( <NUM_LIT> ) . addReg ( TrueReg ) . addReg ( SReg ) ; break ; } case <ISA_LIT>InstrInfo :: VCCZ : { MachineOperand RegOp = Cond [ <NUM_LIT> ] ; RegOp . setImplicit ( false ) ; Register SReg = MRI . createVirtualRegister ( BoolXExecRC ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , SReg ) . add ( RegOp ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addImm ( <NUM_LIT> ) . addReg ( TrueReg ) . addImm ( <NUM_LIT> ) . addReg ( FalseReg ) . addReg ( SReg ) ; break ; } case <ISA_LIT>InstrInfo :: EXECNZ : { Register SReg = MRI . createVirtualRegister ( BoolXExecRC ) ;

Register SReg2 = MRI . createVirtualRegister ( RI . getBoolRC ( ) ) ;


<s> Register SReg = MRI . createVirtualRegister ( BoolXExecRC ) ; BuildMI ( MBB , I , DL , get ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , SReg ) . addImm ( <NUM_LIT> ) . addImm ( - <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addImm ( <NUM_LIT> ) . addReg ( FalseReg ) . addImm ( <NUM_LIT> ) . addReg ( TrueReg ) . addReg ( SReg ) ; break ; } case <ISA_LIT>InstrInfo :: VCCNZ : { MachineOperand RegOp = Cond [ <NUM_LIT> ] ; RegOp . setImplicit ( false ) ; Register SReg = MRI . createVirtualRegister ( BoolXExecRC ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , SReg ) . add ( RegOp ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addImm ( <NUM_LIT> ) . addReg ( FalseReg ) . addImm ( <NUM_LIT> ) . addReg ( TrueReg ) . addReg ( SReg ) ; break ; } case <ISA_LIT>InstrInfo :: VCCZ : { MachineOperand RegOp = Cond [ <NUM_LIT> ] ; RegOp . setImplicit ( false ) ; Register SReg = MRI . createVirtualRegister ( BoolXExecRC ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , SReg ) . add ( RegOp ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addImm ( <NUM_LIT> ) . addReg ( TrueReg ) . addImm ( <NUM_LIT> ) . addReg ( FalseReg ) . addReg ( SReg ) ; break ; } case <ISA_LIT>InstrInfo :: EXECNZ : { Register SReg = MRI . createVirtualRegister ( BoolXExecRC ) ; Register SReg2 = MRI . createVirtualRegister ( RI . getBoolRC ( ) ) ; BuildMI ( MBB , I , DL , get ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , SReg2 ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , SReg ) . addImm ( - <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addImm ( <NUM_LIT> ) . addReg ( FalseReg ) . addImm ( <NUM_LIT> ) . addReg ( TrueReg ) . addReg ( SReg ) ; break ; }


<s> Register Reg = MO . getReg ( ) ; const TargetRegisterClass * RC = Register :: isVirtualRegister ( Reg ) ? MRI . getRegClass ( Reg ) : RI . getPhysRegClass ( Reg ) ; const <ISA_LIT>RegisterInfo * TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( MRI . getTargetRegisterInfo ( ) ) ; RC = TRI -> getSubRegClass ( RC , MO . getSubReg ( ) ) ;

return RI . getCommonSubClass ( RC , RI . getRegClass ( OpInfo . RegClass ) ) == RC ;


<s> Register DstReg = MRI . createVirtualRegister ( DstRC ) ; MachineInstr * Copy = BuildMI ( InsertMBB , I , DL , get ( <STR_LIT> ) , DstReg ) . add ( Op ) ; Op . setReg ( DstReg ) ; Op . setSubReg ( <NUM_LIT> ) ; MachineInstr * Def = MRI . getVRegDef ( OpReg ) ; if ( ! Def ) return ; if ( Def -> isMoveImmediate ( ) ) FoldImmediate ( * Copy , * Def , OpReg , & MRI ) ; bool ImpDef = Def -> isImplicitDef ( ) ; while ( ! ImpDef && Def && Def -> isCopy ( ) ) { Def = MRI . getUniqueVRegDef ( Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ;

ImpDef = Def && Def -> isImplicitDef ( ) ;


<s> MachineFunction * MF = MBB . getParent ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineFrameInfo & FrameInfo = MF -> getFrameInfo ( ) ; const DebugLoc & DL = MBB . findDebugLoc ( MI ) ; unsigned Align = FrameInfo . getObjectAlignment ( FrameIndex ) ; unsigned Size = FrameInfo . getObjectSize ( FrameIndex ) ; unsigned SpillSize = TRI -> getSpillSize ( * RC ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , Size , Align ) ; if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ;

const MCInstrDesc & OpDesc = get ( getSGPRSpillRestoreOpcode ( SpillSize ) ) ;


<s> MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , Size , Align ) ; if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillRestoreOpcode ( SpillSize ) ) ; if ( Register :: isVirtualRegister ( DestReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( DestReg , & <STR_LIT> ) ; } if ( RI . spillSGPRToVGPR ( ) ) FrameInfo . setStackID ( FrameIndex , TargetStackID :: SGPRSpill ) ; MachineInstrBuilder Spill = BuildMI ( MBB , MI , DL , OpDesc , DestReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ; if ( ST . hasScalarStores ( ) ) { Spill . addReg ( <STR_LIT> , RegState :: ImplicitDefine | RegState :: Dead ) ; } return ;


<s> unsigned Exec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; unsigned MovExecOpc = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; const auto * BoolXExecRC = TRI -> getRegClass ( <STR_LIT> ) ; Register SaveExec = MRI . createVirtualRegister ( BoolXExecRC ) ; BuildMI ( MBB , I , DL , TII . get ( MovExecOpc ) , SaveExec ) . addReg ( Exec ) ; for ( auto & MO : MI . uses ( ) ) { if ( MO . isReg ( ) && MO . isUse ( ) ) { MRI . clearKillFlags ( MO . getReg ( ) ) ; } } MachineBasicBlock * LoopBB = MF . CreateMachineBasicBlock ( ) ; MachineBasicBlock * RemainderBB = MF . CreateMachineBasicBlock ( ) ; MachineFunction :: iterator MBBI ( MBB ) ; ++ MBBI ; MF . insert ( MBBI , LoopBB ) ; MF . insert ( MBBI , RemainderBB ) ; LoopBB -> addSuccessor ( LoopBB ) ; LoopBB -> addSuccessor ( RemainderBB ) ; MachineBasicBlock :: iterator J = I ++ ; RemainderBB -> transferSuccessorsAndUpdatePHIs ( & MBB ) ; RemainderBB -> splice ( RemainderBB -> begin ( ) , & MBB , I , MBB . end ( ) ) ; LoopBB -> splice ( LoopBB -> begin ( ) , & MBB , J ) ; MBB . addSuccessor ( LoopBB ) ; if ( MDT ) {


<s> void <ISA_LIT>InstrInfo :: lowerScalarXnor ( SetVectorType & Worklist , MachineInstr & Inst ) const { MachineBasicBlock & MBB = * Inst . getParent ( ) ; MachineRegisterInfo & MRI = MBB . getParent ( ) -> getRegInfo ( ) ; MachineBasicBlock :: iterator MII = Inst ; const DebugLoc & DL = Inst . getDebugLoc ( ) ; MachineOperand & Dest = Inst . getOperand ( <NUM_LIT> ) ; MachineOperand & Src0 = Inst . getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = Inst . getOperand ( <NUM_LIT> ) ; if ( ST . hasDLInsts ( ) ) { Register NewDest = MRI . createVirtualRegister ( & <STR_LIT> ) ; legalizeGenericOperand ( MBB , MII , & <STR_LIT> , Src0 , MRI , DL ) ; legalizeGenericOperand ( MBB , MII , & <STR_LIT> , Src1 , MRI , DL ) ; BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , NewDest ) . add ( Src0 ) . add ( Src1 ) ; MRI . replaceRegWith ( Dest . getReg ( ) , NewDest ) ; addUsersToMoveToVALUWorklist ( NewDest , MRI , Worklist ) ; } else {

bool Src0IsSGPR = Src0 . isReg ( ) && RI . isSGPRClass ( MRI . getRegClass ( Src0 . getReg ( ) ) ) ;


<s> if ( ST . hasDLInsts ( ) ) { Register NewDest = MRI . createVirtualRegister ( & <STR_LIT> ) ; legalizeGenericOperand ( MBB , MII , & <STR_LIT> , Src0 , MRI , DL ) ; legalizeGenericOperand ( MBB , MII , & <STR_LIT> , Src1 , MRI , DL ) ; BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , NewDest ) . add ( Src0 ) . add ( Src1 ) ; MRI . replaceRegWith ( Dest . getReg ( ) , NewDest ) ; addUsersToMoveToVALUWorklist ( NewDest , MRI , Worklist ) ; } else { bool Src0IsSGPR = Src0 . isReg ( ) && RI . isSGPRClass ( MRI . getRegClass ( Src0 . getReg ( ) ) ) ; bool Src1IsSGPR = Src1 . isReg ( ) && RI . isSGPRClass ( MRI . getRegClass ( Src1 . getReg ( ) ) ) ; MachineInstr * Xor ; Register Temp = MRI . createVirtualRegister ( & <STR_LIT> ) ; Register NewDest = MRI . createVirtualRegister ( & <STR_LIT> ) ; if ( Src0IsSGPR ) { BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , Temp ) . add ( Src0 ) ; Xor = BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , NewDest ) . addReg ( Temp ) . add ( Src1 ) ; } else if ( Src1IsSGPR ) { BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , Temp ) . add ( Src1 ) ; Xor = BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , NewDest ) . add ( Src0 ) . addReg ( Temp ) ; } else { Xor = BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , Temp ) . add ( Src0 ) . add ( Src1 ) ;

MachineInstr * Not = BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , NewDest ) . addReg ( Temp ) ;


<s> BuildMI ( * UseMI . getParent ( ) , UseMI , UseMI . getDebugLoc ( ) , get ( TargetOpcode :: COPY ) , NewSrcReg ) . addReg ( SrcReg ) ; SrcReg = NewSrcReg ; } if ( SubRegs == <NUM_LIT> ) { BuildMI ( * UseMI . getParent ( ) , UseMI , UseMI . getDebugLoc ( ) , get ( <STR_LIT> ) , DstReg ) . addReg ( SrcReg ) ; return DstReg ; } SmallVector < unsigned , <NUM_LIT> > SRegs ; for ( unsigned i = <NUM_LIT> ; i < SubRegs ; ++ i ) { Register SGPR = MRI . createVirtualRegister ( & <STR_LIT> ) ;

BuildMI ( * UseMI . getParent ( ) , UseMI , UseMI . getDebugLoc ( ) , get ( <STR_LIT> ) , SGPR ) . addReg ( SrcReg , <NUM_LIT> , RI . getSubRegFromChannel ( i ) ) ;


<s> if ( NumLoads > MaxGlobalLoadCluster ) return false ; FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> :: vdata ) ; if ( ! FirstDst ) FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> :: vdst ) ; SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: vdata ) ; if ( ! SecondDst ) SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: vdst ) ; } else if ( isSMRD ( FirstLdSt ) && isSMRD ( SecondLdSt ) ) { FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> :: sdst ) ; SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: sdst ) ; } else if ( isDS ( FirstLdSt ) && isDS ( SecondLdSt ) ) { FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> :: vdst ) ; SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: vdst ) ; } if ( ! FirstDst || ! SecondDst ) return false ; unsigned LoadClusterThreshold = <NUM_LIT> ; const MachineRegisterInfo & MRI = FirstLdSt . getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; const Register Reg = FirstDst -> getReg ( ) ;

const TargetRegisterClass * DstRC = Register :: isVirtualRegister ( Reg ) ? MRI . getRegClass ( Reg ) : RI . getPhysRegClass ( Reg ) ;


<s> MachineInstr & Op = * BuildMI ( MBB , MII , DL , get ( Opcode ) , Interm ) . add ( Src0 ) . add ( Src1 ) ; MachineInstr & Not = * BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , NewDest ) . addReg ( Interm ) ; Worklist . insert ( & Op ) ;


<s> const DebugLoc & DL = Inst . getDebugLoc ( ) ; MachineOperand & Dest = Inst . getOperand ( <NUM_LIT> ) ; MachineOperand & Src0 = Inst . getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = Inst . getOperand ( <NUM_LIT> ) ; Register NewDest = MRI . createVirtualRegister ( & <STR_LIT> ) ; Register Interm = MRI . createVirtualRegister ( & <STR_LIT> ) ; MachineInstr & Op = * BuildMI ( MBB , MII , DL , get ( Opcode ) , Interm ) . add ( Src0 ) . add ( Src1 ) ; MachineInstr & Not = * BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , NewDest ) . addReg ( Interm ) ; Worklist . insert ( & Op ) ; Worklist . insert ( & Not ) ; MRI . replaceRegWith ( Dest . getReg ( ) , NewDest ) ;

addUsersToMoveToVALUWorklist ( NewDest , MRI , Worklist ) ;


<s> const DebugLoc & DL = MBB . findDebugLoc ( MI ) ; unsigned Size = FrameInfo . getObjectSize ( FrameIndex ) ; unsigned Align = FrameInfo . getObjectAlignment ( FrameIndex ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOStore , Size , Align ) ; unsigned SpillSize = TRI -> getSpillSize ( * RC ) ; if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillSaveOpcode ( SpillSize ) ) ; if ( Register :: isVirtualRegister ( SrcReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( SrcReg , & <STR_LIT> ) ; } MachineInstrBuilder Spill = BuildMI ( MBB , MI , DL , OpDesc ) . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ; if ( RI . spillSGPRToVGPR ( ) ) FrameInfo . setStackID ( FrameIndex , TargetStackID :: SGPRSpill ) ; if ( ST . hasScalarStores ( ) ) { Spill . addReg ( <STR_LIT> , RegState :: ImplicitDefine | RegState :: Dead ) ; }


<s> case <STR_LIT> : MI . eraseFromParent ( ) ; break ; case <STR_LIT> : { unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned DstLo = RI . getSubReg ( Dst , <STR_LIT> ) ; unsigned DstHi = RI . getSubReg ( Dst , <STR_LIT> ) ; const MachineOperand & SrcOp = MI . getOperand ( <NUM_LIT> ) ; assert ( ! SrcOp . isFPImm ( ) ) ; if ( SrcOp . isImm ( ) ) { APInt Imm ( <NUM_LIT> , SrcOp . getImm ( ) ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DstLo ) . addImm ( Imm . getLoBits ( <NUM_LIT> ) . getZExtValue ( ) ) . addReg ( Dst , RegState :: Implicit | RegState :: Define ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DstHi ) . addImm ( Imm . getHiBits ( <NUM_LIT> ) . getZExtValue ( ) ) . addReg ( Dst , RegState :: Implicit | RegState :: Define ) ; } else { assert ( SrcOp . isReg ( ) ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DstLo ) . addReg ( RI . getSubReg ( SrcOp . getReg ( ) , <STR_LIT> ) ) . addReg ( Dst , RegState :: Implicit | RegState :: Define ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DstHi ) . addReg ( RI . getSubReg ( SrcOp . getReg ( ) , <STR_LIT> ) ) . addReg ( Dst , RegState :: Implicit | RegState :: Define ) ; } MI . eraseFromParent ( ) ; break ; } case <STR_LIT> : { unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned DstLo = RI . getSubReg ( Dst , <STR_LIT> ) ; unsigned DstHi = RI . getSubReg ( Dst , <STR_LIT> ) ; unsigned Src0 = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src1 = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;

const MachineOperand & SrcCond = MI . getOperand ( <NUM_LIT> ) ;


<s> bool <ISA_LIT>InstrInfo :: areMemAccessesTriviallyDisjoint ( const MachineInstr & MIa , const MachineInstr & MIb ) const { assert ( ( MIa . mayLoad ( ) || MIa . mayStore ( ) ) && <STR_LIT> ) ; assert ( ( MIb . mayLoad ( ) || MIb . mayStore ( ) ) && <STR_LIT> ) ; if ( MIa . hasUnmodeledSideEffects ( ) || MIb . hasUnmodeledSideEffects ( ) ) return false ; if ( MIa . hasOrderedMemoryRef ( ) || MIb . hasOrderedMemoryRef ( ) ) return false ; if ( isDS ( MIa ) ) { if ( isDS ( MIb ) ) return checkInstOffsetsDoNotOverlap ( MIa , MIb ) ; return ! isFLAT ( MIb ) || isSegmentSpecificFLAT ( MIb ) ; } if ( isMUBUF ( MIa ) || isMTBUF ( MIa ) ) {

if ( isMUBUF ( MIb ) || isMTBUF ( MIb ) ) return checkInstOffsetsDoNotOverlap ( MIa , MIb ) ;


<s> if ( MIa . hasOrderedMemoryRef ( ) || MIb . hasOrderedMemoryRef ( ) ) return false ; if ( isDS ( MIa ) ) { if ( isDS ( MIb ) ) return checkInstOffsetsDoNotOverlap ( MIa , MIb ) ; return ! isFLAT ( MIb ) || isSegmentSpecificFLAT ( MIb ) ; } if ( isMUBUF ( MIa ) || isMTBUF ( MIa ) ) { if ( isMUBUF ( MIb ) || isMTBUF ( MIb ) ) return checkInstOffsetsDoNotOverlap ( MIa , MIb ) ; return ! isFLAT ( MIb ) && ! isSMRD ( MIb ) ; } if ( isSMRD ( MIa ) ) { if ( isSMRD ( MIb ) ) return checkInstOffsetsDoNotOverlap ( MIa , MIb ) ; return ! isFLAT ( MIb ) && ! isMUBUF ( MIa ) && ! isMTBUF ( MIa ) ; } if ( isFLAT ( MIa ) ) {

if ( isFLAT ( MIb ) ) return checkInstOffsetsDoNotOverlap ( MIa , MIb ) ;


<s> if ( InsPt != MBB . end ( ) && ( InsPt -> getOpcode ( ) == <STR_LIT> || InsPt -> getOpcode ( ) == <STR_LIT> || InsPt -> getOpcode ( ) == <STR_LIT> ) && InsPt -> definesRegister ( Src ) ) { InsPt ++ ;

return BuildMI ( MBB , InsPt , InsPt -> getDebugLoc ( ) , get ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , Dst ) . addReg ( Src , <NUM_LIT> , SrcSubReg ) . addReg ( <STR_LIT> , RegState :: Implicit ) ;


<s> if ( ST . hasAddNoCarry ( ) ) return BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DestReg ) ; Register UnusedCarry = RS . scavengeRegister ( RI . getBoolRC ( ) , I , <NUM_LIT> , false ) ; if ( ! UnusedCarry . isValid ( ) ) return MachineInstrBuilder ( ) ;

return BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DestReg ) . addReg ( UnusedCarry , RegState :: Define | RegState :: Dead ) ;


<s> case <STR_LIT> : NewDstRC = RI . getEquivalentAGPRClass ( NewDstRC ) ; break ; default : NewDstRC = RI . getEquivalentVGPRClass ( NewDstRC ) ; } if ( ! NewDstRC ) return nullptr ; } else { if ( RI . hasVGPRs ( NewDstRC ) ) return nullptr ; NewDstRC = RI . getEquivalentVGPRClass ( NewDstRC ) ;


<s> if ( ! BaseOp ) return false ; Offset = OffsetImm -> getImm ( ) ; assert ( BaseOp -> isReg ( ) && <STR_LIT> <STR_LIT> ) ; return true ; } const MachineOperand * Offset0Imm = getNamedOperand ( LdSt , <STR_LIT> :: offset0 ) ; const MachineOperand * Offset1Imm = getNamedOperand ( LdSt , <STR_LIT> :: offset1 ) ; uint8_t Offset0 = Offset0Imm -> getImm ( ) ; uint8_t Offset1 = Offset1Imm -> getImm ( ) ; if ( Offset1 > Offset0 && Offset1 - Offset0 == <NUM_LIT> ) { unsigned EltSize ; if ( LdSt . mayLoad ( ) ) EltSize = TRI -> getRegSizeInBits ( * getOpRegClass ( LdSt , <NUM_LIT> ) ) / <NUM_LIT> ; else { assert ( LdSt . mayStore ( ) ) ; int Data0Idx = <STR_LIT> ( Opc , <STR_LIT> :: data0 ) ; EltSize = TRI -> getRegSizeInBits ( * getOpRegClass ( LdSt , Data0Idx ) ) / <NUM_LIT> ; } if ( isStride64 ( Opc ) ) EltSize *= <NUM_LIT> ; BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: addr ) ; Offset = EltSize * Offset0 ; assert ( BaseOp -> isReg ( ) && <STR_LIT> <STR_LIT> ) ; return true ; } return false ; } if ( isMUBUF ( LdSt ) || isMTBUF ( LdSt ) ) { const MachineOperand * SOffset = getNamedOperand ( LdSt , <STR_LIT> :: soffset ) ; if ( SOffset && SOffset -> isReg ( ) ) { const MachineOperand * AddrReg = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ; if ( AddrReg && ! AddrReg -> isFI ( ) ) return false ; const MachineOperand * RSrc = getNamedOperand ( LdSt , <STR_LIT> :: srsrc ) ; const <ISA_LIT>MachineFunctionInfo * MFI = LdSt . getParent ( ) -> getParent ( ) -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( RSrc -> getReg ( ) != MFI -> getScratchRSrcReg ( ) ) return false ; const MachineOperand * OffsetImm = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; BaseOp = SOffset ; Offset = OffsetImm -> getImm ( ) ; return true ;


<s> for ( unsigned Reg : { TIDIGXReg , TIDIGYReg , TIDIGZReg } ) { if ( ! Entry . isLiveIn ( Reg ) ) Entry . addLiveIn ( Reg ) ; } RS -> enterBasicBlock ( Entry ) ; unsigned STmp0 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; unsigned STmp1 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp0 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Z ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Y ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( STmp1 ) . addReg ( STmp0 ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp1 ) . addReg ( TIDIGXReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp0 ) . addReg ( TIDIGYReg ) . addReg ( TIDReg ) ; getAddNoCarry ( Entry , Insert , DL , TIDReg ) . addReg ( TIDReg ) . addReg ( TIDIGZReg ) ;


<s> case <STR_LIT> : break ; } } const MachineOperand * Src1 = getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 && ( ! Src1 -> isReg ( ) || ! RI . isVGPR ( MRI , Src1 -> getReg ( ) ) || hasModifiersSet ( MI , <STR_LIT> :: src1_modifiers ) ) ) return false ; if ( hasModifiersSet ( MI , <STR_LIT> :: src0_modifiers ) ) return false ; return ! hasModifiersSet ( MI , <STR_LIT> :: omod ) && ! hasModifiersSet ( MI , <STR_LIT> :: clamp ) ;


<s> const MachineOperand * Src1 = getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( ! Src1 -> isReg ( ) || ! RI . isVGPR ( MRI , Src1 -> getReg ( ) ) ) return false ; return true ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( ! Src2 -> isReg ( ) || ! RI . isVGPR ( MRI , Src2 -> getReg ( ) ) || hasModifiersSet ( MI , <STR_LIT> :: src2_modifiers ) ) return false ;


<s> MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , Size , Align ) ; if ( RI . isSGPRClass ( RC ) ) { const MCInstrDesc & OpDesc = get ( getSGPRSpillRestoreOpcode ( SpillSize ) ) ; if ( TargetRegisterInfo :: isVirtualRegister ( DestReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( DestReg , & <STR_LIT> ) ; } FrameInfo . setStackID ( FrameIndex , <STR_LIT> ) ; MachineInstrBuilder Spill = BuildMI ( MBB , MI , DL , OpDesc , DestReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getFrameOffsetReg ( ) , RegState :: Implicit ) ;


<s> } } if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { unsigned Size = ( Opcode == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; Inst . addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ; Inst . addOperand ( MachineOperand :: CreateImm ( Size ) ) ; } else if ( Opcode == <STR_LIT> ) { Inst . addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ; } Inst . addImplicitDefUseOperands ( * Inst . getParent ( ) -> getParent ( ) ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { const MachineOperand & OffsetWidthOp = Inst . getOperand ( <NUM_LIT> ) ; assert ( OffsetWidthOp . isImm ( ) && <STR_LIT> ) ; uint32_t Imm = OffsetWidthOp . getImm ( ) ; uint32_t Offset = Imm & <NUM_LIT> ; uint32_t BitWidth = ( Imm & <NUM_LIT> ) >> <NUM_LIT> ; Inst . RemoveOperand ( <NUM_LIT> ) ; Inst . addOperand ( MachineOperand :: CreateImm ( Offset ) ) ; Inst . addOperand ( MachineOperand :: CreateImm ( BitWidth ) ) ; } bool HasDst = Inst . getOperand ( <NUM_LIT> ) . isReg ( ) && Inst . getOperand ( <NUM_LIT> ) . isDef ( ) ; unsigned NewDstReg = <STR_LIT> ; if ( HasDst ) { unsigned DstReg = Inst . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( TargetRegisterInfo :: isPhysicalRegister ( DstReg ) ) continue ; const TargetRegisterClass * NewDstRC = getDestEquivalentVGPRClass ( Inst ) ; if ( ! NewDstRC ) continue ; if ( Inst . isCopy ( ) && TargetRegisterInfo :: isVirtualRegister ( Inst . getOperand ( <NUM_LIT> ) . getReg ( ) ) && NewDstRC == RI . getRegClassForReg ( MRI , Inst . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { addUsersToMoveToVALUWorklist ( DstReg , MRI , Worklist ) ; MRI . replaceRegWith ( DstReg , Inst . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MRI . clearKillFlags ( Inst . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Inst . getOperand ( <NUM_LIT> ) . setReg ( DstReg ) ; for ( unsigned I = Inst . getNumOperands ( ) - <NUM_LIT> ; I != <NUM_LIT> ; -- I ) Inst . RemoveOperand ( I ) ; Inst . setDesc ( get ( <STR_LIT> ) ) ; continue ; }

NewDstReg = MRI . createVirtualRegister ( NewDstRC ) ;


<s> MachineInstr & LoHalf = * BuildMI ( MBB , MII , DL , InstDesc , DestSub0 ) . add ( SrcReg0Sub0 ) . add ( SrcReg1Sub0 ) ; MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm ( MII , MRI , Src0 , Src0RC , <STR_LIT> , Src0SubRC ) ; MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm ( MII , MRI , Src1 , Src1RC , <STR_LIT> , Src1SubRC ) ; unsigned DestSub1 = MRI . createVirtualRegister ( NewDestSubRC ) ; MachineInstr & HiHalf = * BuildMI ( MBB , MII , DL , InstDesc , DestSub1 ) . add ( SrcReg0Sub1 ) . add ( SrcReg1Sub1 ) ; unsigned FullDestReg = MRI . createVirtualRegister ( NewDestRC ) ; BuildMI ( MBB , MII , DL , get ( TargetOpcode :: REG_SEQUENCE ) , FullDestReg ) . addReg ( DestSub0 ) . addImm ( <STR_LIT> ) . addReg ( DestSub1 ) . addImm ( <STR_LIT> ) ;

MRI . replaceRegWith ( Dest . getReg ( ) , FullDestReg ) ;


<s> const TargetRegisterClass * Src0RC = Src0 . isReg ( ) ? MRI . getRegClass ( Src0 . getReg ( ) ) : & <STR_LIT> ; const TargetRegisterClass * Src0SubRC = RI . getSubRegClass ( Src0RC , <STR_LIT> ) ; const TargetRegisterClass * Src1RC = Src1 . isReg ( ) ? MRI . getRegClass ( Src1 . getReg ( ) ) : & <STR_LIT> ; const TargetRegisterClass * Src1SubRC = RI . getSubRegClass ( Src1RC , <STR_LIT> ) ; MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm ( MII , MRI , Src0 , Src0RC , <STR_LIT> , Src0SubRC ) ; MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm ( MII , MRI , Src1 , Src1RC , <STR_LIT> , Src1SubRC ) ; const TargetRegisterClass * DestRC = MRI . getRegClass ( Dest . getReg ( ) ) ; const TargetRegisterClass * NewDestRC = RI . getEquivalentVGPRClass ( DestRC ) ; const TargetRegisterClass * NewDestSubRC = RI . getSubRegClass ( NewDestRC , <STR_LIT> ) ; unsigned DestSub0 = MRI . createVirtualRegister ( NewDestSubRC ) ; MachineInstr & LoHalf = * BuildMI ( MBB , MII , DL , InstDesc , DestSub0 ) . add ( SrcReg0Sub0 ) . add ( SrcReg1Sub0 ) ; MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm ( MII , MRI , Src0 , Src0RC , <STR_LIT> , Src0SubRC ) ; MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm ( MII , MRI , Src1 , Src1RC , <STR_LIT> , Src1SubRC ) ; unsigned DestSub1 = MRI . createVirtualRegister ( NewDestSubRC ) ; MachineInstr & HiHalf = * BuildMI ( MBB , MII , DL , InstDesc , DestSub1 ) . add ( SrcReg0Sub1 ) . add ( SrcReg1Sub1 ) ;

unsigned FullDestReg = MRI . createVirtualRegister ( NewDestRC ) ;


<s> MachineBasicBlock :: iterator Insert = Entry . front ( ) ; DebugLoc DL = Insert -> getDebugLoc ( ) ; TIDReg = RI . findUnusedRegister ( MF -> getRegInfo ( ) , & <STR_LIT> , * MF ) ; if ( TIDReg == <STR_LIT> ) return TIDReg ; if ( ! <STR_LIT> ( MF -> getFunction ( ) -> getCallingConv ( ) ) && WorkGroupSize > WavefrontSize ) { unsigned TIDIGXReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: WORKGROUP_ID_X ) ; unsigned TIDIGYReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: WORKGROUP_ID_Y ) ; unsigned TIDIGZReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: WORKGROUP_ID_Z ) ; unsigned InputPtrReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: KERNARG_SEGMENT_PTR ) ; for ( unsigned Reg : { TIDIGXReg , TIDIGYReg , TIDIGZReg } ) { if ( ! Entry . isLiveIn ( Reg ) ) Entry . addLiveIn ( Reg ) ; } RS -> enterBasicBlock ( Entry ) ; unsigned STmp0 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; unsigned STmp1 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp0 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Z ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Y ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( STmp1 ) . addReg ( STmp0 ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp1 ) . addReg ( TIDIGXReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp0 ) . addReg ( TIDIGYReg ) . addReg ( TIDReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( TIDReg ) . addReg ( TIDIGZReg ) ; } else { BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( - <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( - <NUM_LIT> ) . addReg ( TIDReg ) ; } BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( <NUM_LIT> ) . addReg ( TIDReg ) ; MFI -> setTIDReg ( TIDReg ) ; } unsigned LDSOffset = MFI -> LDSSize + ( FrameOffset * WorkGroupSize ) ;

BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , TmpReg ) . addImm ( LDSOffset ) . addReg ( TIDReg ) ;


<s> return ; } if ( RC == & <STR_LIT> ) { if ( DestReg == <STR_LIT> ) { if ( <STR_LIT> . contains ( SrcReg ) ) { BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) ; } else { assert ( <STR_LIT> . contains ( SrcReg ) ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) ; } return ; } if ( ! <STR_LIT> . contains ( SrcReg ) ) { reportIllegalCopy ( this , MBB , MI , DL , DestReg , SrcReg , KillSrc ) ; return ; } BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) ; return ; } if ( DestReg == <STR_LIT> ) { assert ( <STR_LIT> . contains ( SrcReg ) ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) . addImm ( <NUM_LIT> ) ; return ; } unsigned EltSize = <NUM_LIT> ; unsigned Opcode = <STR_LIT> ; if ( RI . isSGPRClass ( RC ) ) { if ( ! ( RI . getRegSizeInBits ( * RC ) % <NUM_LIT> ) ) { Opcode = <STR_LIT> ; EltSize = <NUM_LIT> ; } else { Opcode = <STR_LIT> ; EltSize = <NUM_LIT> ; } if ( ! RI . isSGPRClass ( RI . getPhysRegClass ( SrcReg ) ) ) { reportIllegalCopy ( this , MBB , MI , DL , DestReg , SrcReg , KillSrc ) ; return ;


<s> reportIllegalCopy ( this , MBB , MI , DL , DestReg , SrcReg , KillSrc ) ; return ; } BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) ; return ; } if ( DestReg == <STR_LIT> ) { assert ( <STR_LIT> . contains ( SrcReg ) ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) . addImm ( <NUM_LIT> ) ; return ; } unsigned EltSize = <NUM_LIT> ; unsigned Opcode = <STR_LIT> ; if ( RI . isSGPRClass ( RC ) ) { if ( ! ( RI . getRegSizeInBits ( * RC ) % <NUM_LIT> ) ) { Opcode = <STR_LIT> ; EltSize = <NUM_LIT> ; } else { Opcode = <STR_LIT> ; EltSize = <NUM_LIT> ; } if ( ! RI . isSGPRClass ( RI . getPhysRegClass ( SrcReg ) ) ) { reportIllegalCopy ( this , MBB , MI , DL , DestReg , SrcReg , KillSrc ) ; return ; } } ArrayRef < int16_t > SubIndices = RI . getRegSplitParts ( RC , EltSize ) ; bool Forward = RI . getHWRegIndex ( DestReg ) <= RI . getHWRegIndex ( SrcReg ) ; for ( unsigned Idx = <NUM_LIT> ; Idx < SubIndices . size ( ) ; ++ Idx ) { unsigned SubIdx ; if ( Forward ) SubIdx = SubIndices [ Idx ] ; else SubIdx = SubIndices [ SubIndices . size ( ) - Idx - <NUM_LIT> ] ; MachineInstrBuilder Builder = BuildMI ( MBB , MI , DL , get ( Opcode ) , RI . getSubReg ( DestReg , SubIdx ) ) ; Builder . addReg ( RI . getSubReg ( SrcReg , SubIdx ) ) ;

if ( Idx == <NUM_LIT> ) Builder . addReg ( DestReg , RegState :: Define | RegState :: Implicit ) ;


<s> if ( UseMI -> getParent ( ) != DefBB ) return true ; } else { int NumUse = <NUM_LIT> ; const int MaxUseScan = <NUM_LIT> ; for ( auto & UseInst : MRI . use_nodbg_instructions ( VReg ) ) { if ( UseInst . getParent ( ) != DefBB ) return true ; if ( NumUse ++ > MaxUseScan ) return true ; } } const int MaxInstScan = <NUM_LIT> ; int NumScan = <NUM_LIT> ; auto E = UseMI ? UseMI -> getIterator ( ) : DefBB -> end ( ) ; for ( auto I = std :: next ( DefMI . getIterator ( ) ) ; I != E ; ++ I ) { if ( I -> isDebugInstr ( ) ) continue ; if ( NumScan ++ > MaxInstScan ) return true ; if ( I -> modifiesRegister ( <STR_LIT> , TRI ) ) return true ; }


<s> } if ( isLegalRegOperand ( MRI , InstrDesc . OpInfo [ Src1Idx ] , Src1 ) ) return ; if ( Opc == <STR_LIT> && Src1 . isReg ( ) && RI . isVGPR ( MRI , Src1 . getReg ( ) ) ) { unsigned Reg = MRI . createVirtualRegister ( & <STR_LIT> ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; BuildMI ( * MI . getParent ( ) , MI , DL , get ( <STR_LIT> ) , Reg ) . add ( Src1 ) ; Src1 . ChangeToRegister ( Reg , false ) ; return ; } if ( HasImplicitSGPR || ! MI . isCommutable ( ) ) { legalizeOpWithMove ( MI , Src1Idx ) ; return ; } int Src0Idx = <STR_LIT> ( Opc , <STR_LIT> :: src0 ) ; MachineOperand & Src0 = MI . getOperand ( Src0Idx ) ; if ( ( ! Src1 . isImm ( ) && ! Src1 . isReg ( ) ) || ! isLegalRegOperand ( MRI , InstrDesc . OpInfo [ Src1Idx ] , Src0 ) ) { legalizeOpWithMove ( MI , Src1Idx ) ; return ; } int CommutedOpc = commuteOpcode ( MI ) ; if ( CommutedOpc == - <NUM_LIT> ) { legalizeOpWithMove ( MI , Src1Idx ) ; return ;


<s> MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , Size , Align ) ; if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillRestoreOpcode ( SpillSize ) ) ; if ( TargetRegisterInfo :: isVirtualRegister ( DestReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( DestReg , & <STR_LIT> ) ; } FrameInfo . setStackID ( FrameIndex , TargetStackID :: SGPRSpill ) ; MachineInstrBuilder Spill = BuildMI ( MBB , MI , DL , OpDesc , DestReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ; if ( ST . hasScalarStores ( ) ) { Spill . addReg ( <STR_LIT> , RegState :: ImplicitDefine | RegState :: Dead ) ; }


<s> if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillSaveOpcode ( SpillSize ) ) ; if ( TargetRegisterInfo :: isVirtualRegister ( SrcReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( SrcReg , & <STR_LIT> ) ; } MachineInstrBuilder Spill = BuildMI ( MBB , MI , DL , OpDesc ) . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ; FrameInfo . setStackID ( FrameIndex , TargetStackID :: SGPRSpill ) ; if ( ST . hasScalarStores ( ) ) { Spill . addReg ( <STR_LIT> , RegState :: ImplicitDefine | RegState :: Dead ) ; } return ; }

assert ( RI . hasVGPRs ( RC ) && <STR_LIT> ) ;


<s> switch ( Opc ) { default : return nullptr ; case <STR_LIT> : case <STR_LIT> : IsF16 = true ; LLVM_FALLTHROUGH ; case <STR_LIT> : case <STR_LIT> : break ; case <STR_LIT> : case <STR_LIT> : IsF16 = true ; LLVM_FALLTHROUGH ; case <STR_LIT> : case <STR_LIT> : { int Src0Idx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: src0 ) ; const MachineOperand * Src0 = & MI . getOperand ( Src0Idx ) ; if ( ! Src0 -> isReg ( ) && ! Src0 -> isImm ( ) ) return nullptr ; if ( Src0 -> isImm ( ) && ! isInlineConstant ( MI , Src0Idx , * Src0 ) ) return nullptr ; break ; } } const MachineOperand * Dst = getNamedOperand ( MI , <STR_LIT> :: vdst ) ; const MachineOperand * Src0 = getNamedOperand ( MI , <STR_LIT> :: src0 ) ; const MachineOperand * Src0Mods = getNamedOperand ( MI , <STR_LIT> :: src0_modifiers ) ; const MachineOperand * Src1 = getNamedOperand ( MI , <STR_LIT> :: src1 ) ; const MachineOperand * Src1Mods = getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) ; const MachineOperand * Src2 = getNamedOperand ( MI , <STR_LIT> :: src2 ) ; const MachineOperand * Clamp = getNamedOperand ( MI , <STR_LIT> :: clamp ) ; const MachineOperand * Omod = getNamedOperand ( MI , <STR_LIT> :: omod ) ; MachineInstrBuilder MIB ; if ( ! Src0Mods && ! Src1Mods && ! Clamp && ! Omod && ( ST . getConstantBusLimit ( Opc ) > <NUM_LIT> || ! Src0 -> isReg ( ) || ! RI . isSGPRReg ( MBB -> getParent ( ) -> getRegInfo ( ) , Src0 -> getReg ( ) ) ) ) { if ( auto Imm = getFoldableImm ( Src2 ) ) { unsigned NewOpc = IsFMA ? ( IsF16 ? <STR_LIT> : <STR_LIT> ) : ( IsF16 ? <STR_LIT> : <STR_LIT> ) ; if ( pseudoToMCOpcode ( NewOpc ) != - <NUM_LIT> ) { MIB = BuildMI ( * MBB , MI , MI . getDebugLoc ( ) , get ( NewOpc ) ) . add ( * Dst ) . add ( * Src0 ) . add ( * Src1 ) . addImm ( Imm ) ; updateLiveVariables ( LV , MI , * MIB ) ; return MIB ; } } unsigned NewOpc = IsFMA ? ( IsF16 ? <STR_LIT> : <STR_LIT> ) : ( IsF16 ? <STR_LIT> : <STR_LIT> ) ; if ( auto Imm = getFoldableImm ( Src1 ) ) { if ( pseudoToMCOpcode ( NewOpc ) != - <NUM_LIT> ) { MIB = BuildMI ( * MBB , MI , MI . getDebugLoc ( ) , get ( NewOpc ) ) . add ( * Dst ) . add ( * Src0 ) . addImm ( Imm ) . add ( * Src2 ) ; updateLiveVariables ( LV , MI , * MIB ) ; return MIB ;


<s> unsigned NumSubRegs = RegSize / <NUM_LIT> ; assert ( NumSubRegs % <NUM_LIT> == <NUM_LIT> && NumSubRegs <= <NUM_LIT> && <STR_LIT> ) ; for ( unsigned Idx = <NUM_LIT> ; Idx < NumSubRegs ; Idx += <NUM_LIT> ) { Register CurRegLo = MRI . createVirtualRegister ( & <STR_LIT> ) ; Register CurRegHi = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CurRegLo ) . addReg ( VRsrc , VRsrcUndef , TRI -> getSubRegFromChannel ( Idx ) ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CurRegHi ) . addReg ( VRsrc , VRsrcUndef , TRI -> getSubRegFromChannel ( Idx + <NUM_LIT> ) ) ; ReadlanePieces . push_back ( CurRegLo ) ; ReadlanePieces . push_back ( CurRegHi ) ; Register CurReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CurReg ) . addReg ( CurRegLo ) . addImm ( <STR_LIT> ) . addReg ( CurRegHi ) . addImm ( <STR_LIT> ) ; Register NewCondReg = MRI . createVirtualRegister ( BoolXExecRC ) ; auto Cmp = BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , NewCondReg ) . addReg ( CurReg ) ; if ( NumSubRegs <= <NUM_LIT> ) Cmp . addReg ( VRsrc ) ; else Cmp . addReg ( VRsrc , VRsrcUndef , TRI -> getSubRegFromChannel ( Idx , <NUM_LIT> ) ) ; if ( CondReg == <STR_LIT> ) CondReg = NewCondReg ; else { Register AndReg = MRI . createVirtualRegister ( BoolXExecRC ) ;

BuildMI ( LoopBB , I , DL , TII . get ( AndOpc ) , AndReg ) . addReg ( CondReg ) . addReg ( NewCondReg ) ;


<s> unsigned Exec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; unsigned SaveExecOpc = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; unsigned XorTermOpc = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; unsigned AndOpc = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; const auto * BoolXExecRC = TRI -> getRegClass ( <STR_LIT> ) ; MachineBasicBlock :: iterator I = LoopBB . begin ( ) ; SmallVector < Register , <NUM_LIT> > ReadlanePieces ; Register CondReg = <STR_LIT> ; Register VRsrc = Rsrc . getReg ( ) ; unsigned VRsrcUndef = getUndefRegState ( Rsrc . isUndef ( ) ) ; unsigned RegSize = TRI -> getRegSizeInBits ( Rsrc . getReg ( ) , MRI ) ; unsigned NumSubRegs = RegSize / <NUM_LIT> ; assert ( NumSubRegs % <NUM_LIT> == <NUM_LIT> && NumSubRegs <= <NUM_LIT> && <STR_LIT> ) ; for ( unsigned Idx = <NUM_LIT> ; Idx < NumSubRegs ; Idx += <NUM_LIT> ) { Register CurRegLo = MRI . createVirtualRegister ( & <STR_LIT> ) ; Register CurRegHi = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CurRegLo ) . addReg ( VRsrc , VRsrcUndef , TRI -> getSubRegFromChannel ( Idx ) ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CurRegHi ) . addReg ( VRsrc , VRsrcUndef , TRI -> getSubRegFromChannel ( Idx + <NUM_LIT> ) ) ; ReadlanePieces . push_back ( CurRegLo ) ; ReadlanePieces . push_back ( CurRegHi ) ; Register CurReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CurReg ) . addReg ( CurRegLo ) . addImm ( <STR_LIT> ) . addReg ( CurRegHi ) . addImm ( <STR_LIT> ) ; Register NewCondReg = MRI . createVirtualRegister ( BoolXExecRC ) ; auto Cmp = BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , NewCondReg ) . addReg ( CurReg ) ; if ( NumSubRegs <= <NUM_LIT> ) Cmp . addReg ( VRsrc ) ; else Cmp . addReg ( VRsrc , VRsrcUndef , TRI -> getSubRegFromChannel ( Idx , <NUM_LIT> ) ) ; if ( CondReg == <STR_LIT> ) CondReg = NewCondReg ; else { Register AndReg = MRI . createVirtualRegister ( BoolXExecRC ) ; BuildMI ( LoopBB , I , DL , TII . get ( AndOpc ) , AndReg ) . addReg ( CondReg ) . addReg ( NewCondReg ) ; CondReg = AndReg ; } } auto SRsrcRC = TRI -> getEquivalentSGPRClass ( MRI . getRegClass ( VRsrc ) ) ;

Register SRsrc = MRI . createVirtualRegister ( SRsrcRC ) ;


<s> return Size ; } if ( isVALU ( MI ) || isSALU ( MI ) ) { int Src0Idx = <STR_LIT> ( Opc , <STR_LIT> :: src0 ) ; if ( Src0Idx == - <NUM_LIT> ) return DescSize ; if ( isLiteralConstantLike ( MI . getOperand ( Src0Idx ) , Desc . OpInfo [ Src0Idx ] ) ) return isVOP3 ( MI ) ? <NUM_LIT> : ( DescSize + <NUM_LIT> ) ; int Src1Idx = <STR_LIT> ( Opc , <STR_LIT> :: src1 ) ; if ( Src1Idx == - <NUM_LIT> ) return DescSize ; if ( isLiteralConstantLike ( MI . getOperand ( Src1Idx ) , Desc . OpInfo [ Src1Idx ] ) ) return isVOP3 ( MI ) ? <NUM_LIT> : ( DescSize + <NUM_LIT> ) ; int Src2Idx = <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ; if ( Src2Idx == - <NUM_LIT> ) return DescSize ; if ( isLiteralConstantLike ( MI . getOperand ( Src2Idx ) , Desc . OpInfo [ Src2Idx ] ) ) return isVOP3 ( MI ) ? <NUM_LIT> : ( DescSize + <NUM_LIT> ) ; return DescSize ; } if ( isMIMG ( MI ) ) { int VAddr0Idx = <STR_LIT> ( Opc , <STR_LIT> :: vaddr0 ) ; if ( VAddr0Idx < <NUM_LIT> ) return <NUM_LIT> ;

int RSrcIdx = <STR_LIT> ( Opc , <STR_LIT> :: srsrc ) ;


<s> bool hasFPClamp ( uint16_t Opcode ) const { return get ( Opcode ) . TSFlags & <STR_LIT> ;


<s> return get ( Opcode ) . TSFlags & <STR_LIT> ;


<s> if ( MI . mayStore ( ) && isSMRD ( MI ) ) return true ; if ( MI . isReturn ( ) ) return true ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) return true ; if ( MI . isCall ( ) || MI . isInlineAsm ( ) ) return true ; if ( modifiesModeRegister ( MI ) ) return true ;

if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) return true ;


<s> bool <ISA_LIT>InstrInfo :: hasUnwantedEffectsWhenEXECEmpty ( const MachineInstr & MI ) const { unsigned Opcode = MI . getOpcode ( ) ; if ( MI . mayStore ( ) && isSMRD ( MI ) ) return true ; if ( MI . isReturn ( ) ) return true ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) return true ; if ( MI . isCall ( ) || MI . isInlineAsm ( ) ) return true ; if ( modifiesModeRegister ( MI ) ) return true ;


<s> switch ( Cond [ <NUM_LIT> ] . getImm ( ) ) { case VCCNZ : case VCCZ : { const MachineRegisterInfo & MRI = MBB . getParent ( ) -> getRegInfo ( ) ; const TargetRegisterClass * RC = MRI . getRegClass ( TrueReg ) ; assert ( MRI . getRegClass ( FalseReg ) == RC ) ; int NumInsts = <STR_LIT> ( RC -> getID ( ) ) / <NUM_LIT> ; CondCycles = TrueCycles = FalseCycles = NumInsts ; return RI . hasVGPRs ( RC ) && NumInsts <= <NUM_LIT> ; } case SCC_TRUE : case SCC_FALSE : { const MachineRegisterInfo & MRI = MBB . getParent ( ) -> getRegInfo ( ) ; const TargetRegisterClass * RC = MRI . getRegClass ( TrueReg ) ; assert ( MRI . getRegClass ( FalseReg ) == RC ) ; int NumInsts = <STR_LIT> ( RC -> getID ( ) ) / <NUM_LIT> ; if ( NumInsts % <NUM_LIT> == <NUM_LIT> ) NumInsts /= <NUM_LIT> ; CondCycles = TrueCycles = FalseCycles = NumInsts ; return RI . isSGPRClass ( RC ) ;


<s> break ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const MCInstrDesc & MovRelDesc = get ( <STR_LIT> ) ; Register VecReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; bool IsUndef = MI . getOperand ( <NUM_LIT> ) . isUndef ( ) ; unsigned SubReg = <STR_LIT> + MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; assert ( VecReg == MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MachineInstr * MovRel = BuildMI ( MBB , MI , DL , MovRelDesc ) . addReg ( RI . getSubReg ( VecReg , SubReg ) , RegState :: Undef ) . add ( MI . getOperand ( <NUM_LIT> ) ) . addReg ( VecReg , RegState :: ImplicitDefine ) . addReg ( VecReg , RegState :: Implicit | ( IsUndef ? RegState :: Undef : <NUM_LIT> ) ) ; const int ImpDefIdx = MovRelDesc . getNumOperands ( ) + MovRelDesc . getNumImplicitUses ( ) ; const int ImpUseIdx = ImpDefIdx + <NUM_LIT> ; MovRel -> tieOperands ( ImpDefIdx , ImpUseIdx ) ; MI . eraseFromParent ( ) ; break ; } case <STR_LIT> : { MachineFunction & MF = * MBB . getParent ( ) ; Register Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register RegLo = RI . getSubReg ( Reg , <STR_LIT> ) ; Register RegHi = RI . getSubReg ( Reg , <STR_LIT> ) ; MIBundleBuilder Bundler ( MBB , MI ) ; Bundler . append ( BuildMI ( MF , DL , get ( <STR_LIT> ) , Reg ) ) ; Bundler . append ( BuildMI ( MF , DL , get ( <STR_LIT> ) , RegLo ) . addReg ( RegLo ) . add ( MI . getOperand ( <NUM_LIT> ) ) ) ; MachineInstrBuilder MIB = BuildMI ( MF , DL , get ( <STR_LIT> ) , RegHi ) . addReg ( RegHi ) ; MIB . add ( MI . getOperand ( <NUM_LIT> ) ) ; Bundler . append ( MIB ) ; finalizeBundle ( MBB , Bundler . begin ( ) ) ; MI . eraseFromParent ( ) ; break ; } case <STR_LIT> : {

MI . setDesc ( get ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) ) ;


<s> BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addImm ( <NUM_LIT> ) . addReg ( TrueReg ) . addImm ( <NUM_LIT> ) . addReg ( FalseReg ) . addReg ( SReg ) ; break ; } case <ISA_LIT>InstrInfo :: EXECNZ : { Register SReg = MRI . createVirtualRegister ( BoolXExecRC ) ; Register SReg2 = MRI . createVirtualRegister ( RI . getBoolRC ( ) ) ; BuildMI ( MBB , I , DL , get ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , SReg2 ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , SReg ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addImm ( <NUM_LIT> ) . addReg ( FalseReg ) . addImm ( <NUM_LIT> ) . addReg ( TrueReg ) . addReg ( SReg ) ; break ; } case <ISA_LIT>InstrInfo :: EXECZ : { Register SReg = MRI . createVirtualRegister ( BoolXExecRC ) ; Register SReg2 = MRI . createVirtualRegister ( RI . getBoolRC ( ) ) ; BuildMI ( MBB , I , DL , get ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , SReg2 ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , SReg ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addImm ( <NUM_LIT> ) . addReg ( FalseReg ) . addImm ( <NUM_LIT> ) . addReg ( TrueReg ) . addReg ( SReg ) ; llvm_unreachable ( <STR_LIT> ) ; break ; } default : llvm_unreachable ( <STR_LIT> ) ; } } else {


<s> unsigned Align = FrameInfo . getObjectAlignment ( FrameIndex ) ; unsigned Size = FrameInfo . getObjectSize ( FrameIndex ) ; unsigned SpillSize = TRI -> getSpillSize ( * RC ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , Size , Align ) ; if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ; assert ( DestReg != <STR_LIT> && <STR_LIT> ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillRestoreOpcode ( SpillSize ) ) ; if ( Register :: isVirtualRegister ( DestReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( DestReg , & <STR_LIT> ) ; } if ( RI . spillSGPRToVGPR ( ) ) FrameInfo . setStackID ( FrameIndex , TargetStackID :: SGPRSpill ) ; BuildMI ( MBB , MI , DL , OpDesc , DestReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ; return ; } unsigned Opcode = RI . hasAGPRs ( RC ) ? getAGPRSpillRestoreOpcode ( SpillSize ) : getVGPRSpillRestoreOpcode ( SpillSize ) ; auto MIB = BuildMI ( MBB , MI , DL , get ( Opcode ) , DestReg ) ; if ( RI . hasAGPRs ( RC ) ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ;

Register Tmp = MRI . createVirtualRegister ( & <STR_LIT> ) ;


<s> unsigned SpillSize = TRI -> getSpillSize ( * RC ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , Size , Align ) ; if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ; assert ( DestReg != <STR_LIT> && <STR_LIT> ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillRestoreOpcode ( SpillSize ) ) ; if ( Register :: isVirtualRegister ( DestReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( DestReg , & <STR_LIT> ) ; } if ( RI . spillSGPRToVGPR ( ) ) FrameInfo . setStackID ( FrameIndex , TargetStackID :: SGPRSpill ) ; BuildMI ( MBB , MI , DL , OpDesc , DestReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ; return ; } unsigned Opcode = RI . hasAGPRs ( RC ) ? getAGPRSpillRestoreOpcode ( SpillSize ) : getVGPRSpillRestoreOpcode ( SpillSize ) ; auto MIB = BuildMI ( MBB , MI , DL , get ( Opcode ) , DestReg ) ; if ( RI . hasAGPRs ( RC ) ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; Register Tmp = MRI . createVirtualRegister ( & <STR_LIT> ) ; MIB . addReg ( Tmp , RegState :: Define ) ;


<s> if ( ! BaseOp1 . isReg ( ) || ! BaseOp2 . isReg ( ) ) return false ; if ( BaseOp1 . isIdenticalTo ( BaseOp2 ) ) return true ; if ( ! MI1 . hasOneMemOperand ( ) || ! MI2 . hasOneMemOperand ( ) ) return false ; auto MO1 = * MI1 . memoperands_begin ( ) ; auto MO2 = * MI2 . memoperands_begin ( ) ; if ( MO1 -> getAddrSpace ( ) != MO2 -> getAddrSpace ( ) ) return false ; auto Base1 = MO1 -> getValue ( ) ; auto Base2 = MO2 -> getValue ( ) ; if ( ! Base1 || ! Base2 ) return false ; const MachineFunction & MF = * MI1 . getParent ( ) -> getParent ( ) ; const DataLayout & DL = MF . getFunction ( ) . getParent ( ) -> getDataLayout ( ) ;

Base1 = GetUnderlyingObject ( Base1 , DL ) ;


<s> auto MO2 = * MI2 . memoperands_begin ( ) ; if ( MO1 -> getAddrSpace ( ) != MO2 -> getAddrSpace ( ) ) return false ; auto Base1 = MO1 -> getValue ( ) ; auto Base2 = MO2 -> getValue ( ) ; if ( ! Base1 || ! Base2 ) return false ; const MachineFunction & MF = * MI1 . getParent ( ) -> getParent ( ) ; const DataLayout & DL = MF . getFunction ( ) . getParent ( ) -> getDataLayout ( ) ; Base1 = GetUnderlyingObject ( Base1 , DL ) ; Base2 = GetUnderlyingObject ( Base2 , DL ) ;

if ( isa < UndefValue > ( Base1 ) || isa < UndefValue > ( Base2 ) ) return false ;


<s> DiagnosticInfoUnsupported IllegalCopy ( MF -> getFunction ( ) , <STR_LIT> , DL , DS_Error ) ; LLVMContext & C = MF -> getFunction ( ) . getContext ( ) ;


<s> if ( ! ST . hasFlatInstOffsets ( ) ) return false ; if ( ST . hasFlatSegmentOffsetBug ( ) && AddrSpace == <ISA_LIT>AS :: FLAT_ADDRESS ) return false ; unsigned N = <STR_LIT> ( ST , Signed ) ; return Signed ? isIntN ( N , Offset ) : isUIntN ( N , Offset ) ;


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :

return MI . getNumOperands ( ) == MI . getDesc ( ) . getNumOperands ( ) ;


<s> if ( IsSigned ) { int64_t D = <NUM_LIT> << ( NumBits - <NUM_LIT> ) ; RemainderOffset = ( COffsetVal / D ) * D ; ImmField = COffsetVal - RemainderOffset ; } else if ( COffsetVal >= <NUM_LIT> ) { ImmField = COffsetVal & maskTrailingOnes < uint64_t > ( NumBits ) ; RemainderOffset = COffsetVal - ImmField ;


<s> ImmField = COffsetVal - RemainderOffset ; } else if ( COffsetVal >= <NUM_LIT> ) { ImmField = COffsetVal & maskTrailingOnes < uint64_t > ( NumBits ) ; RemainderOffset = COffsetVal - ImmField ; }

assert ( isLegalFLATOffset ( ImmField , AddrSpace , IsSigned ) ) ;


<s> RS -> enterBasicBlockEnd ( MBB ) ; Register Scav = RS -> scavengeRegisterBackwards ( <STR_LIT> , MachineBasicBlock :: iterator ( GetPC ) , false , <NUM_LIT> , false ) ; if ( Scav ) { RS -> setRegUsed ( Scav ) ; MRI . replaceRegWith ( PCReg , Scav ) ; MRI . clearVirtRegs ( ) ; } else { const GCNSubtarget & ST = MF -> getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; TRI -> spillEmergencySGPR ( GetPC , RestoreBB , <STR_LIT> , RS ) ; MRI . replaceRegWith ( PCReg , <STR_LIT> ) ; MRI . clearVirtRegs ( ) ; } MCSymbol * DestLabel = Scav ? DestBB . getSymbol ( ) : RestoreBB . getSymbol ( ) ; auto * Offset = MCBinaryExpr :: createSub ( MCSymbolRefExpr :: create ( DestLabel , MCCtx ) , MCSymbolRefExpr :: create ( PostGetPCLabel , MCCtx ) , MCCtx ) ; auto * Mask = MCConstantExpr :: create ( <NUM_LIT> , MCCtx ) ; OffsetLo -> setVariableValue ( MCBinaryExpr :: createAnd ( Offset , Mask , MCCtx ) ) ; auto * ShAmt = MCConstantExpr :: create ( <NUM_LIT> , MCCtx ) ; OffsetHi -> setVariableValue ( MCBinaryExpr :: createAShr ( Offset , ShAmt , MCCtx ) ) ; return ;


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> return true ; } return false ; } if ( isMUBUF ( LdSt ) || isMTBUF ( LdSt ) ) { const MachineOperand * SOffset = getNamedOperand ( LdSt , <STR_LIT> :: soffset ) ; if ( SOffset && SOffset -> isReg ( ) ) { const MachineOperand * AddrReg = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ; if ( AddrReg && ! AddrReg -> isFI ( ) ) return false ; const MachineOperand * RSrc = getNamedOperand ( LdSt , <STR_LIT> :: srsrc ) ; const <ISA_LIT>MachineFunctionInfo * MFI = LdSt . getParent ( ) -> getParent ( ) -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( RSrc -> getReg ( ) != MFI -> getScratchRSrcReg ( ) ) return false ; const MachineOperand * OffsetImm = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; BaseOps . push_back ( SOffset ) ; Offset = OffsetImm -> getImm ( ) ; return true ; } const MachineOperand * AddrReg = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ; if ( ! AddrReg ) return false ; const MachineOperand * OffsetImm = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; BaseOp = AddrReg ; Offset = OffsetImm -> getImm ( ) ; if ( SOffset ) Offset += SOffset -> getImm ( ) ; if ( ! BaseOp -> isReg ( ) ) return false ; BaseOps . push_back ( BaseOp ) ; return true ; } if ( isSMRD ( LdSt ) ) { const MachineOperand * OffsetImm = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; if ( ! OffsetImm ) return false ; const MachineOperand * SBaseReg = getNamedOperand ( LdSt , <STR_LIT> :: sbase ) ; BaseOp = SBaseReg ; Offset = OffsetImm -> getImm ( ) ; if ( ! BaseOp -> isReg ( ) ) return false ; BaseOps . push_back ( BaseOp ) ; return true ; } if ( isFLAT ( LdSt ) ) { const MachineOperand * VAddr = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ; if ( VAddr ) { if ( getNamedOperand ( LdSt , <STR_LIT> :: saddr ) ) return false ; BaseOp = VAddr ; } else { BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: saddr ) ;


<s> if ( Src0 -> isReg ( ) && Src0 -> getReg ( ) == Reg ) { if ( ! Src1 -> isReg ( ) || RI . isSGPRClass ( MRI -> getRegClass ( Src1 -> getReg ( ) ) ) ) return false ; if ( ! Src2 -> isReg ( ) || RI . isSGPRClass ( MRI -> getRegClass ( Src2 -> getReg ( ) ) ) ) return false ; const int64_t Imm = DefMI . getOperand ( <NUM_LIT> ) . getImm ( ) ; UseMI . RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: omod ) ) ; UseMI . RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: clamp ) ) ; unsigned Src1Reg = Src1 -> getReg ( ) ; unsigned Src1SubReg = Src1 -> getSubReg ( ) ; Src0 -> setReg ( Src1Reg ) ; Src0 -> setSubReg ( Src1SubReg ) ; Src0 -> setIsKill ( Src1 -> isKill ( ) ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) UseMI . untieRegOperand ( <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ) ; Src1 -> ChangeToImmediate ( Imm ) ; removeModOperands ( UseMI ) ; UseMI . setDesc ( get ( IsF32 ? <STR_LIT> : <STR_LIT> ) ) ; bool DeleteDef = MRI -> hasOneNonDBGUse ( Reg ) ; if ( DeleteDef ) DefMI . eraseFromParent ( ) ; return true ; } if ( Src2 -> isReg ( ) && Src2 -> getReg ( ) == Reg ) { if ( ! Src0 -> isImm ( ) && ( Src0 -> isReg ( ) && RI . isSGPRClass ( MRI -> getRegClass ( Src0 -> getReg ( ) ) ) ) ) return false ; if ( ! Src1 -> isReg ( ) || RI . isSGPRClass ( MRI -> getRegClass ( Src1 -> getReg ( ) ) ) ) return false ; const int64_t Imm = DefMI . getOperand ( <NUM_LIT> ) . getImm ( ) ; UseMI . RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: omod ) ) ; UseMI . RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: clamp ) ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) UseMI . untieRegOperand ( <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ) ; Src2 -> ChangeToImmediate ( Imm ) ; removeModOperands ( UseMI ) ; UseMI . setDesc ( get ( IsF32 ? <STR_LIT> : <STR_LIT> ) ) ; bool DeleteDef = MRI -> hasOneNonDBGUse ( Reg ) ; if ( DeleteDef ) DefMI . eraseFromParent ( ) ; return true ; } }


<s> <ISA_LIT>MachineFunctionInfo * MFI = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineFrameInfo & FrameInfo = MF -> getFrameInfo ( ) ; DebugLoc DL = MBB . findDebugLoc ( MI ) ; unsigned Size = FrameInfo . getObjectSize ( FrameIndex ) ; unsigned Align = FrameInfo . getObjectAlignment ( FrameIndex ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOStore , Size , Align ) ; unsigned SpillSize = TRI -> getSpillSize ( * RC ) ; if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillSaveOpcode ( SpillSize ) ) ; if ( TargetRegisterInfo :: isVirtualRegister ( SrcReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( SrcReg , & <STR_LIT> ) ; } MachineInstrBuilder Spill = BuildMI ( MBB , MI , DL , OpDesc ) . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getFrameOffsetReg ( ) , RegState :: Implicit ) ; if ( ST . hasScalarStores ( ) ) { Spill . addReg ( <STR_LIT> , RegState :: ImplicitDefine ) ;


<s> MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOStore , Size , Align ) ; unsigned SpillSize = TRI -> getSpillSize ( * RC ) ; if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillSaveOpcode ( SpillSize ) ) ; if ( TargetRegisterInfo :: isVirtualRegister ( SrcReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( SrcReg , & <STR_LIT> ) ; } MachineInstrBuilder Spill = BuildMI ( MBB , MI , DL , OpDesc ) . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getFrameOffsetReg ( ) , RegState :: Implicit ) ; if ( ST . hasScalarStores ( ) ) { Spill . addReg ( <STR_LIT> , RegState :: ImplicitDefine ) ; } return ;


<s> return ; } if ( ! ST . isVGPRSpillingEnabled ( MF -> getFunction ( ) ) ) { LLVMContext & Ctx = MF -> getFunction ( ) . getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) ; return ; } assert ( RI . hasVGPRs ( RC ) && <STR_LIT> ) ; unsigned Opcode = getVGPRSpillRestoreOpcode ( SpillSize ) ;

BuildMI ( MBB , MI , DL , get ( Opcode ) , DestReg ) . addFrameIndex ( FrameIndex ) . addReg ( MFI -> getScratchRSrcReg ( ) ) . addReg ( MFI -> getFrameOffsetReg ( ) ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ;


<s> void <ISA_LIT>InstrInfo :: loadRegFromStackSlot ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator MI , unsigned DestReg , int FrameIndex , const TargetRegisterClass * RC , const TargetRegisterInfo * TRI ) const { MachineFunction * MF = MBB . getParent ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineFrameInfo & FrameInfo = MF -> getFrameInfo ( ) ; DebugLoc DL = MBB . findDebugLoc ( MI ) ; unsigned Align = FrameInfo . getObjectAlignment ( FrameIndex ) ; unsigned Size = FrameInfo . getObjectSize ( FrameIndex ) ; unsigned SpillSize = TRI -> getSpillSize ( * RC ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , Size , Align ) ;


<s> case <STR_LIT> : Count = <NUM_LIT> ; break ; } SmallVector < unsigned , <NUM_LIT> > ResultRegs ; for ( unsigned i = <NUM_LIT> ; i < Count ; ++ i ) { unsigned ResultReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; MachineInstr & NewMI = * BuildMI ( MBB , MII , DL , get ( NewOpcode ) , ResultReg ) . addReg ( Offset . getReg ( ) ) . addReg ( Rsrc . getReg ( ) ) . addImm ( <NUM_LIT> ) . addImm ( i << <NUM_LIT> ) . addImm ( Glc . getImm ( ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( * Inst . memoperands_begin ( ) ) ; auto & NewDestOp = NewMI . getOperand ( <NUM_LIT> ) ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; i ++ ) ResultRegs . push_back ( buildExtractSubReg ( MII , MRI , NewDestOp , & <STR_LIT> , RI . getSubRegFromChannel ( i ) , & <STR_LIT> ) ) ;


<s> break ; } SmallVector < unsigned , <NUM_LIT> > ResultRegs ; for ( unsigned i = <NUM_LIT> ; i < Count ; ++ i ) { unsigned ResultReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; MachineInstr & NewMI = * BuildMI ( MBB , MII , DL , get ( NewOpcode ) , ResultReg ) . addReg ( Offset . getReg ( ) ) . addReg ( Rsrc . getReg ( ) ) . addImm ( <NUM_LIT> ) . addImm ( i << <NUM_LIT> ) . addImm ( Glc . getImm ( ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( * Inst . memoperands_begin ( ) ) ; auto & NewDestOp = NewMI . getOperand ( <NUM_LIT> ) ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; i ++ ) ResultRegs . push_back ( buildExtractSubReg ( MII , MRI , NewDestOp , & <STR_LIT> , RI . getSubRegFromChannel ( i ) , & <STR_LIT> ) ) ;


<s> unsigned <ISA_LIT>InstrInfo :: getInstSizeInBytes ( const MachineInstr & MI ) const { unsigned Opc = MI . getOpcode ( ) ; const MCInstrDesc & Desc = getMCOpcodeFromPseudo ( Opc ) ; unsigned DescSize = Desc . getSize ( ) ; if ( DescSize != <NUM_LIT> && DescSize != <NUM_LIT> ) return DescSize ; if ( isFixedSize ( MI ) ) return DescSize ; if ( isVALU ( MI ) || isSALU ( MI ) ) { int Src0Idx = <STR_LIT> ( Opc , <STR_LIT> :: src0 ) ; if ( Src0Idx == - <NUM_LIT> ) return <NUM_LIT> ;

if ( isLiteralConstantLike ( MI . getOperand ( Src0Idx ) , Desc . OpInfo [ Src0Idx ] ) ) return <NUM_LIT> ;


<s> unsigned <ISA_LIT>InstrInfo :: getInstSizeInBytes ( const MachineInstr & MI ) const { unsigned Opc = MI . getOpcode ( ) ; const MCInstrDesc & Desc = getMCOpcodeFromPseudo ( Opc ) ; unsigned DescSize = Desc . getSize ( ) ; if ( DescSize != <NUM_LIT> && DescSize != <NUM_LIT> ) return DescSize ; if ( isFixedSize ( MI ) ) return DescSize ; if ( isVALU ( MI ) || isSALU ( MI ) ) { int Src0Idx = <STR_LIT> ( Opc , <STR_LIT> :: src0 ) ; if ( Src0Idx == - <NUM_LIT> ) return <NUM_LIT> ; if ( isLiteralConstantLike ( MI . getOperand ( Src0Idx ) , Desc . OpInfo [ Src0Idx ] ) ) return <NUM_LIT> ; int Src1Idx = <STR_LIT> ( Opc , <STR_LIT> :: src1 ) ; if ( Src1Idx == - <NUM_LIT> ) return <NUM_LIT> ; if ( isLiteralConstantLike ( MI . getOperand ( Src1Idx ) , Desc . OpInfo [ Src1Idx ] ) ) return <NUM_LIT> ; return <NUM_LIT> ; } if ( DescSize == <NUM_LIT> ) return <NUM_LIT> ; switch ( Opc ) { case TargetOpcode :: IMPLICIT_DEF : case TargetOpcode :: KILL : case TargetOpcode :: DBG_VALUE : case TargetOpcode :: EH_LABEL : return <NUM_LIT> ;


<s> case <STR_LIT> : case <STR_LIT> : { if ( isUInt < <NUM_LIT> > ( Imm ) ) { int16_t Trunc = static_cast < int16_t > ( Imm ) ; return ST . has16BitInsts ( ) && <STR_LIT> ( Trunc , ST . hasInv2PiInlineImm ( ) ) ; } if ( ! ( Imm & <NUM_LIT> ) ) { return ST . has16BitInsts ( ) && <STR_LIT> ( Imm >> <NUM_LIT> , ST . hasInv2PiInlineImm ( ) ) ; } uint32_t Trunc = static_cast < uint32_t > ( Imm ) ; return <STR_LIT> ( Trunc , ST . hasInv2PiInlineImm ( ) ) ; } default : llvm_unreachable ( <STR_LIT> ) ; }


<s> if ( MI . memoperands_empty ( ) ) return true ; for ( const MachineMemOperand * MMO : MI . memoperands ( ) ) { if ( MMO -> getAddrSpace ( ) == ST . get<ISA_LIT>AS ( ) . FLAT_ADDRESS ) return true ; }


<s> int64_t Offset0 , Offset1 ; if ( getMemOpBaseRegImmOfs ( MIa , BaseReg0 , Offset0 , & RI ) && getMemOpBaseRegImmOfs ( MIb , BaseReg1 , Offset1 , & RI ) ) { assert ( MIa -> hasOneMemOperand ( ) && MIb -> hasOneMemOperand ( ) && <STR_LIT> ) ; unsigned Width0 = ( * MIa -> memoperands_begin ( ) ) -> getSize ( ) ; unsigned Width1 = ( * MIb -> memoperands_begin ( ) ) -> getSize ( ) ; if ( BaseReg0 == BaseReg1 && offsetsDoNotOverlap ( Width0 , Offset0 , Width1 , Offset1 ) ) { return true ; }


<s> assert ( MIa -> hasOneMemOperand ( ) && MIb -> hasOneMemOperand ( ) && <STR_LIT> ) ; unsigned Width0 = ( * MIa -> memoperands_begin ( ) ) -> getSize ( ) ; unsigned Width1 = ( * MIb -> memoperands_begin ( ) ) -> getSize ( ) ; if ( BaseReg0 == BaseReg1 && offsetsDoNotOverlap ( Width0 , Offset0 , Width1 , Offset1 ) ) { return true ;


<s> if ( hasModifiersSet ( * UseMI , <STR_LIT> :: src0_modifiers ) || hasModifiersSet ( * UseMI , <STR_LIT> :: src1_modifiers ) || hasModifiersSet ( * UseMI , <STR_LIT> :: src2_modifiers ) ) { return false ; } MachineOperand * Src0 = getNamedOperand ( * UseMI , <STR_LIT> :: src0 ) ; MachineOperand * Src1 = getNamedOperand ( * UseMI , <STR_LIT> :: src1 ) ; MachineOperand * Src2 = getNamedOperand ( * UseMI , <STR_LIT> :: src2 ) ; if ( Src0 -> isReg ( ) && Src0 -> getReg ( ) == Reg ) { if ( ! Src1 -> isReg ( ) || RI . isSGPRClass ( MRI -> getRegClass ( Src1 -> getReg ( ) ) ) ) return false ; if ( ! Src2 -> isReg ( ) || RI . isSGPRClass ( MRI -> getRegClass ( Src2 -> getReg ( ) ) ) ) return false ; const int64_t Imm = DefMI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; UseMI -> RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: omod ) ) ; UseMI -> RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: clamp ) ) ; unsigned Src1Reg = Src1 -> getReg ( ) ; unsigned Src1SubReg = Src1 -> getSubReg ( ) ; Src0 -> setReg ( Src1Reg ) ; Src0 -> setSubReg ( Src1SubReg ) ; Src0 -> setIsKill ( Src1 -> isKill ( ) ) ; if ( Opc == <STR_LIT> ) { UseMI -> untieRegOperand ( <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ) ; } Src1 -> ChangeToImmediate ( Imm ) ; removeModOperands ( * UseMI ) ; UseMI -> setDesc ( get ( <STR_LIT> ) ) ; bool DeleteDef = MRI -> hasOneNonDBGUse ( Reg ) ;

if ( DeleteDef ) DefMI -> eraseFromParent ( ) ;


<s> uint8_t Offset0 = Offset0Imm -> getImm ( ) ; uint8_t Offset1 = Offset1Imm -> getImm ( ) ; if ( Offset1 > Offset0 && Offset1 - Offset0 == <NUM_LIT> ) { unsigned EltSize ; if ( LdSt -> mayLoad ( ) ) EltSize = getOpRegClass ( * LdSt , <NUM_LIT> ) -> getSize ( ) / <NUM_LIT> ; else { assert ( LdSt -> mayStore ( ) ) ; int Data0Idx = <STR_LIT> ( Opc , <STR_LIT> :: data0 ) ; EltSize = getOpRegClass ( * LdSt , Data0Idx ) -> getSize ( ) ; } if ( isStride64 ( Opc ) ) EltSize *= <NUM_LIT> ; const MachineOperand * AddrReg = getNamedOperand ( * LdSt , <STR_LIT> :: addr ) ; BaseReg = AddrReg -> getReg ( ) ; Offset = EltSize * Offset0 ; return true ;


<s> uint64_t Rsrc23 = getDefaultRsrcDataFormat ( ) | <STR_LIT> | <NUM_LIT> ; uint64_t EltSizeValue = Log2_32 ( ST . getMaxPrivateElementSize ( ) ) - <NUM_LIT> ; Rsrc23 |= ( EltSizeValue << <STR_LIT> ) ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) Rsrc23 &= ~ <STR_LIT> ; return Rsrc23 ;


<s> uint64_t Rsrc23 = getDefaultRsrcDataFormat ( ) | <STR_LIT> | <NUM_LIT> ;

uint64_t EltSizeValue = Log2_32 ( ST . getMaxPrivateElementSize ( ) ) - <NUM_LIT> ;


<s> if ( RI . isSGPRClass ( RC ) ) { unsigned Opcode = getSGPRSpillRestoreOpcode ( RC -> getSize ( ) ) ; BuildMI ( MBB , MI , DL , get ( Opcode ) , DestReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) ; return ; } if ( ! ST . isVGPRSpillingEnabled ( MFI ) ) { LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) ; return ; } assert ( RI . hasVGPRs ( RC ) && <STR_LIT> ) ; unsigned Opcode = getVGPRSpillRestoreOpcode ( RC -> getSize ( ) ) ; BuildMI ( MBB , MI , DL , get ( Opcode ) , DestReg ) . addFrameIndex ( FrameIndex ) . addReg ( MFI -> getScratchRSrcReg ( ) ) . addReg ( MFI -> getScratchWaveOffsetReg ( ) ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ;


<s> unsigned Size = FrameInfo -> getObjectSize ( FrameIndex ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , Size , Align ) ; if ( RI . isSGPRClass ( RC ) ) { unsigned Opcode = getSGPRSpillRestoreOpcode ( RC -> getSize ( ) ) ; BuildMI ( MBB , MI , DL , get ( Opcode ) , DestReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) ; return ;


<s> if ( isSMRD ( * FirstLdSt ) && isSMRD ( * SecondLdSt ) ) return true ;

return ( isMUBUF ( * FirstLdSt ) || isMTBUF ( * FirstLdSt ) ) && ( isMUBUF ( * SecondLdSt ) || isMTBUF ( * SecondLdSt ) ) ;


<s> bool <ISA_LIT>InstrInfo :: shouldClusterLoads ( MachineInstr * FirstLdSt , MachineInstr * SecondLdSt , unsigned NumLoads ) const { if ( NumLoads > <NUM_LIT> ) return false ;

if ( isDS ( * FirstLdSt ) && isDS ( * SecondLdSt ) ) return true ;


<s> MFI -> setHasSpilledSGPRs ( ) ; unsigned Opcode = getSGPRSpillSaveOpcode ( RC -> getSize ( ) ) ; BuildMI ( MBB , MI , DL , get ( Opcode ) ) . addReg ( SrcReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) ; return ; } if ( ! ST . isVGPRSpillingEnabled ( MFI ) ) { LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ) . addReg ( SrcReg ) ; return ; } assert ( RI . hasVGPRs ( RC ) && <STR_LIT> ) ;

unsigned Opcode = getVGPRSpillSaveOpcode ( RC -> getSize ( ) ) ;


<s> const MachineOperand & MO = MI -> getOperand ( OpIdx ) ; if ( usesConstantBus ( MRI , MO , getOpSize ( Opcode , OpIdx ) ) ) { if ( MO . isReg ( ) ) { if ( MO . getReg ( ) != SGPRUsed ) ++ ConstantBusCount ; SGPRUsed = MO . getReg ( ) ; } else { ++ ConstantBusCount ; } } } if ( ConstantBusCount > <NUM_LIT> ) { ErrInfo = <STR_LIT> ; return false ; } } if ( Desc . getOpcode ( ) == <STR_LIT> || Desc . getOpcode ( ) == <STR_LIT> ) { const MachineOperand & Src0 = MI -> getOperand ( Src0Idx ) ; const MachineOperand & Src1 = MI -> getOperand ( Src1Idx ) ; const MachineOperand & Src2 = MI -> getOperand ( Src2Idx ) ; if ( Src0 . isReg ( ) && Src1 . isReg ( ) && Src2 . isReg ( ) ) { if ( ! compareMachineOp ( Src0 , Src1 ) && ! compareMachineOp ( Src0 , Src2 ) ) { ErrInfo = <STR_LIT> ; return false ; } } } if ( ! isGenericOpcode ( Opcode ) && ! isSALU ( Opcode ) && ! isSMRD ( Opcode ) ) { const MachineOperand * Exec = MI -> findRegisterUseOperand ( <STR_LIT> ) ; if ( ! Exec || ! Exec -> isImplicit ( ) ) { ErrInfo = <STR_LIT> ; return false ; } } return true ;


<s> void <ISA_LIT>InstrInfo :: addSCCDefUsersToVALUWorklist ( MachineInstr & SCCDefInst , SetVectorType & Worklist ) const { for ( MachineInstr & MI : make_range ( MachineBasicBlock :: iterator ( SCCDefInst ) , SCCDefInst . getParent ( ) -> end ( ) ) ) { if ( MI . findRegisterDefOperandIdx ( <STR_LIT> , false , false , & RI ) != - <NUM_LIT> ) return ; if ( MI . findRegisterUseOperandIdx ( <STR_LIT> , false , & RI ) != - <NUM_LIT> ) Worklist . insert ( & MI ) ;


<s> MachineBasicBlock * MBB = MI . getParent ( ) ; ; MachineInstrBuilder Inst32 = BuildMI ( * MBB , MI , MI . getDebugLoc ( ) , get ( Op32 ) ) ; int Op32DstIdx = <STR_LIT> ( Op32 , <STR_LIT> :: vdst ) ; if ( Op32DstIdx != - <NUM_LIT> ) { Inst32 . add ( MI . getOperand ( <NUM_LIT> ) ) ; } else { assert ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && <STR_LIT> ) ; } Inst32 . add ( * getNamedOperand ( MI , <STR_LIT> :: src0 ) ) ; const MachineOperand * Src1 = getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 ) Inst32 . add ( * Src1 ) ; const MachineOperand * Src2 = getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( Src2 ) { int Op32Src2Idx = <STR_LIT> ( Op32 , <STR_LIT> :: src2 ) ; if ( Op32Src2Idx != - <NUM_LIT> ) { Inst32 . add ( * Src2 ) ; } else { copyFlagsToImplicitVCC ( * Inst32 , * Src2 ) ; }


<s> } RS -> enterBasicBlock ( Entry ) ; unsigned STmp0 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; unsigned STmp1 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp0 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Z ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Y ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( STmp1 ) . addReg ( STmp0 ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp1 ) . addReg ( TIDIGXReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp0 ) . addReg ( TIDIGYReg ) . addReg ( TIDReg ) ; getAddNoCarry ( Entry , Insert , DL , TIDReg ) . addReg ( TIDReg ) . addReg ( TIDIGZReg ) ; } else { BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( - <NUM_LIT> ) . addImm ( <NUM_LIT> ) ;

BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( - <NUM_LIT> ) . addReg ( TIDReg ) ;


<s> BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp0 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Z ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Y ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( STmp1 ) . addReg ( STmp0 ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp1 ) . addReg ( TIDIGXReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp0 ) . addReg ( TIDIGYReg ) . addReg ( TIDReg ) ; getAddNoCarry ( Entry , Insert , DL , TIDReg ) . addReg ( TIDReg ) . addReg ( TIDIGZReg ) ; } else { BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( - <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( - <NUM_LIT> ) . addReg ( TIDReg ) ; }

BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( <NUM_LIT> ) . addReg ( TIDReg ) ;


<s> if ( Src1 && ( ! Src1 -> isReg ( ) || ! RI . isVGPR ( MRI , Src1 -> getReg ( ) ) || hasModifiersSet ( MI , <STR_LIT> :: src1_modifiers ) ) ) return false ; if ( hasModifiersSet ( MI , <STR_LIT> :: src0_modifiers ) ) return false ; if ( ! hasVALU32BitEncoding ( MI . getOpcode ( ) ) ) return false ; return ! hasModifiersSet ( MI , <STR_LIT> :: omod ) && ! hasModifiersSet ( MI , <STR_LIT> :: clamp ) ;


<s> unsigned Width0 = ( * MIa . memoperands_begin ( ) ) -> getSize ( ) ; unsigned Width1 = ( * MIb . memoperands_begin ( ) ) -> getSize ( ) ; if ( offsetsDoNotOverlap ( Width0 , Offset0 , Width1 , Offset1 ) ) { return true ;


<s> } unsigned Width0 = ( * MIa . memoperands_begin ( ) ) -> getSize ( ) ; unsigned Width1 = ( * MIb . memoperands_begin ( ) ) -> getSize ( ) ;

if ( offsetsDoNotOverlap ( Width0 , Offset0 , Width1 , Offset1 ) ) {


<s> MachineInstr * <ISA_LIT>InstrInfo :: convertToThreeAddress ( MachineFunction :: iterator & MBB , MachineInstr & MI , LiveVariables * LV ) const { unsigned Opc = MI . getOpcode ( ) ; bool IsF16 = false ; bool IsFMA = Opc == <STR_LIT> || Opc == <STR_LIT> ; switch ( Opc ) { default : return nullptr ; case <STR_LIT> : IsF16 = true ; LLVM_FALLTHROUGH ; case <STR_LIT> : case <STR_LIT> : break ; case <STR_LIT> : IsF16 = true ; LLVM_FALLTHROUGH ; case <STR_LIT> : case <STR_LIT> : { int Src0Idx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: src0 ) ; const MachineOperand * Src0 = & MI . getOperand ( Src0Idx ) ; if ( ! Src0 -> isReg ( ) && ! Src0 -> isImm ( ) ) return nullptr ; if ( Src0 -> isImm ( ) && ! isInlineConstant ( MI , Src0Idx , * Src0 ) ) return nullptr ; break ; } } const MachineOperand * Dst = getNamedOperand ( MI , <STR_LIT> :: vdst ) ; const MachineOperand * Src0 = getNamedOperand ( MI , <STR_LIT> :: src0 ) ; const MachineOperand * Src0Mods = getNamedOperand ( MI , <STR_LIT> :: src0_modifiers ) ; const MachineOperand * Src1 = getNamedOperand ( MI , <STR_LIT> :: src1 ) ; const MachineOperand * Src1Mods = getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) ; const MachineOperand * Src2 = getNamedOperand ( MI , <STR_LIT> :: src2 ) ; const MachineOperand * Clamp = getNamedOperand ( MI , <STR_LIT> :: clamp ) ; const MachineOperand * Omod = getNamedOperand ( MI , <STR_LIT> :: omod ) ; if ( ! IsFMA && ! Src0Mods && ! Src1Mods && ! Clamp && ! Omod && ( ! Src0 -> isReg ( ) || ! RI . isSGPRReg ( MBB -> getParent ( ) -> getRegInfo ( ) , Src0 -> getReg ( ) ) ) ) { if ( auto Imm = getFoldableImm ( Src2 ) ) { return BuildMI ( * MBB , MI , MI . getDebugLoc ( ) , get ( IsF16 ? <STR_LIT> : <STR_LIT> ) ) . add ( * Dst ) . add ( * Src0 ) . add ( * Src1 ) . addImm ( Imm ) ;


<s> const MachineOperand * Src0 = & MI . getOperand ( Src0Idx ) ; if ( ! Src0 -> isReg ( ) && ! Src0 -> isImm ( ) ) return nullptr ; if ( Src0 -> isImm ( ) && ! isInlineConstant ( MI , Src0Idx , * Src0 ) ) return nullptr ; break ; } } const MachineOperand * Dst = getNamedOperand ( MI , <STR_LIT> :: vdst ) ; const MachineOperand * Src0 = getNamedOperand ( MI , <STR_LIT> :: src0 ) ; const MachineOperand * Src0Mods = getNamedOperand ( MI , <STR_LIT> :: src0_modifiers ) ; const MachineOperand * Src1 = getNamedOperand ( MI , <STR_LIT> :: src1 ) ; const MachineOperand * Src1Mods = getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) ; const MachineOperand * Src2 = getNamedOperand ( MI , <STR_LIT> :: src2 ) ; const MachineOperand * Clamp = getNamedOperand ( MI , <STR_LIT> :: clamp ) ; const MachineOperand * Omod = getNamedOperand ( MI , <STR_LIT> :: omod ) ; if ( ! IsFMA && ! Src0Mods && ! Src1Mods && ! Clamp && ! Omod && ( ! Src0 -> isReg ( ) || ! RI . isSGPRReg ( MBB -> getParent ( ) -> getRegInfo ( ) , Src0 -> getReg ( ) ) ) ) { if ( auto Imm = getFoldableImm ( Src2 ) ) { return BuildMI ( * MBB , MI , MI . getDebugLoc ( ) , get ( IsF16 ? <STR_LIT> : <STR_LIT> ) ) . add ( * Dst ) . add ( * Src0 ) . add ( * Src1 ) . addImm ( Imm ) ; } if ( auto Imm = getFoldableImm ( Src1 ) ) { return BuildMI ( * MBB , MI , MI . getDebugLoc ( ) , get ( IsF16 ? <STR_LIT> : <STR_LIT> ) ) . add ( * Dst ) . add ( * Src0 ) . addImm ( Imm ) . add ( * Src2 ) ; } if ( auto Imm = getFoldableImm ( Src0 ) ) { if ( isOperandLegal ( MI , <STR_LIT> ( <STR_LIT> , <STR_LIT> :: src0 ) , Src1 ) ) return BuildMI ( * MBB , MI , MI . getDebugLoc ( ) , get ( IsF16 ? <STR_LIT> : <STR_LIT> ) ) . add ( * Dst ) . add ( * Src1 ) . addImm ( Imm ) . add ( * Src2 ) ;


<s> static void copyFlagsToImplicitVCC ( MachineInstr & MI , const MachineOperand & Orig ) {

for ( MachineOperand & Use : MI . implicit_operands ( ) ) {


<s> for ( MachineOperand & Use : MI . implicit_operands ( ) ) {

if ( Use . isUse ( ) && Use . getReg ( ) == <STR_LIT> ) {


<s> unsigned SRsrcSub1 = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrcSub2 = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrcSub3 = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrc = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub0 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub1 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub2 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub3 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrc ) . addReg ( SRsrcSub0 ) . addImm ( <STR_LIT> ) . addReg ( SRsrcSub1 ) . addImm ( <STR_LIT> ) . addReg ( SRsrcSub2 ) . addImm ( <STR_LIT> ) . addReg ( SRsrcSub3 ) . addImm ( <STR_LIT> ) ; Rsrc . setReg ( SRsrc ) ; Rsrc . setIsKill ( true ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CondReg0 ) . addReg ( SRsrc , <NUM_LIT> , <STR_LIT> ) . addReg ( VRsrc , <NUM_LIT> , <STR_LIT> ) ;

BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CondReg1 ) . addReg ( SRsrc , <NUM_LIT> , <STR_LIT> ) . addReg ( VRsrc , <NUM_LIT> , <STR_LIT> ) ;


<s> unsigned SRsrcSub3 = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrc = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub0 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub1 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub2 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub3 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrc ) . addReg ( SRsrcSub0 ) . addImm ( <STR_LIT> ) . addReg ( SRsrcSub1 ) . addImm ( <STR_LIT> ) . addReg ( SRsrcSub2 ) . addImm ( <STR_LIT> ) . addReg ( SRsrcSub3 ) . addImm ( <STR_LIT> ) ; Rsrc . setReg ( SRsrc ) ; Rsrc . setIsKill ( true ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CondReg0 ) . addReg ( SRsrc , <NUM_LIT> , <STR_LIT> ) . addReg ( VRsrc , <NUM_LIT> , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CondReg1 ) . addReg ( SRsrc , <NUM_LIT> , <STR_LIT> ) . addReg ( VRsrc , <NUM_LIT> , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , AndCond ) . addReg ( CondReg0 ) . addReg ( CondReg1 ) ; MRI . setSimpleHint ( SaveExec , AndCond ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SaveExec ) . addReg ( AndCond , RegState :: Kill ) ; I = LoopBB . end ( ) ;

BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( SaveExec ) ;


<s> static std :: tuple < unsigned , unsigned > extractRsrcPtr ( const <ISA_LIT>InstrInfo & TII , MachineInstr & MI , MachineOperand & Rsrc ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; MachineFunction & MF = * MBB . getParent ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; unsigned RsrcPtr = TII . buildExtractSubReg ( MI , MRI , Rsrc , & <STR_LIT> , <STR_LIT> , & <STR_LIT> ) ; unsigned Zero64 = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrcFormatLo = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrcFormatHi = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned NewSRsrc = MRI . createVirtualRegister ( & <STR_LIT> ) ;

uint64_t RsrcDataFormat = TII . getDefaultRsrcDataFormat ( ) ;


<s> static std :: tuple < unsigned , unsigned > extractRsrcPtr ( const <ISA_LIT>InstrInfo & TII , MachineInstr & MI , MachineOperand & Rsrc ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; MachineFunction & MF = * MBB . getParent ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; unsigned RsrcPtr = TII . buildExtractSubReg ( MI , MRI , Rsrc , & <STR_LIT> , <STR_LIT> , & <STR_LIT> ) ; unsigned Zero64 = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrcFormatLo = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrcFormatHi = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned NewSRsrc = MRI . createVirtualRegister ( & <STR_LIT> ) ; uint64_t RsrcDataFormat = TII . getDefaultRsrcDataFormat ( ) ; BuildMI ( MBB , MI , MI . getDebugLoc ( ) , TII . get ( <STR_LIT> ) , Zero64 ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , MI , MI . getDebugLoc ( ) , TII . get ( <STR_LIT> ) , SRsrcFormatLo ) . addImm ( RsrcDataFormat & <NUM_LIT> ) ; BuildMI ( MBB , MI , MI . getDebugLoc ( ) , TII . get ( <STR_LIT> ) , SRsrcFormatHi ) . addImm ( RsrcDataFormat >> <NUM_LIT> ) ;

BuildMI ( MBB , MI , MI . getDebugLoc ( ) , TII . get ( <STR_LIT> ) , NewSRsrc ) . addReg ( Zero64 ) . addImm ( <STR_LIT> ) . addReg ( SRsrcFormatLo ) . addImm ( <STR_LIT> ) . addReg ( SRsrcFormatHi ) . addImm ( <STR_LIT> ) ;


<s> if ( Src0Idx == - <NUM_LIT> ) return false ; int Src1Idx = <STR_LIT> ( Opc , <STR_LIT> :: src1 ) ; if ( Src1Idx == - <NUM_LIT> ) return false ; return fixCommutedOpIndices ( SrcOpIdx0 , SrcOpIdx1 , Src0Idx , Src1Idx ) ;


<s> if ( ! Desc . isCommutable ( ) ) return false ; unsigned Opc = Desc . getOpcode ( ) ; int Src0Idx = <STR_LIT> ( Opc , <STR_LIT> :: src0 ) ; if ( Src0Idx == - <NUM_LIT> ) return false ; int Src1Idx = <STR_LIT> ( Opc , <STR_LIT> :: src1 ) ; if ( Src1Idx == - <NUM_LIT> ) return false ;

return fixCommutedOpIndices ( SrcOpIdx0 , SrcOpIdx1 , Src0Idx , Src1Idx ) ;