<s> MachineBasicBlock :: iterator Insert = Entry . front ( ) ; DebugLoc DL = Insert -> getDebugLoc ( ) ; TIDReg = RI . findUnusedRegister ( MF -> getRegInfo ( ) , & <STR_LIT> , * MF ) ; if ( TIDReg == <STR_LIT> ) return TIDReg ; if ( ! <STR_LIT> ( MF -> getFunction ( ) -> getCallingConv ( ) ) && WorkGroupSize > WavefrontSize ) { unsigned TIDIGXReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: WORKGROUP_ID_X ) ; unsigned TIDIGYReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: WORKGROUP_ID_Y ) ; unsigned TIDIGZReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: WORKGROUP_ID_Z ) ; unsigned InputPtrReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: KERNARG_SEGMENT_PTR ) ; for ( unsigned Reg : { TIDIGXReg , TIDIGYReg , TIDIGZReg } ) { if ( ! Entry . isLiveIn ( Reg ) ) Entry . addLiveIn ( Reg ) ; } RS -> enterBasicBlock ( Entry ) ; unsigned STmp0 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; unsigned STmp1 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp0 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Z ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Y ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( STmp1 ) . addReg ( STmp0 ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp1 ) . addReg ( TIDIGXReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp0 ) . addReg ( TIDIGYReg ) . addReg ( TIDReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( TIDReg ) . addReg ( TIDIGZReg ) ; } else { BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( - <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( - <NUM_LIT> ) . addReg ( TIDReg ) ; } BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( <NUM_LIT> ) . addReg ( TIDReg ) ; MFI -> setTIDReg ( TIDReg ) ; } unsigned LDSOffset = MFI -> LDSSize + ( FrameOffset * WorkGroupSize ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , TmpReg ) . addImm ( LDSOffset ) .


<s> reportIllegalCopy ( this , MBB , MI , DL , DestReg , SrcReg , KillSrc ) ; return ; } BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) ; return ; } if ( DestReg == <STR_LIT> ) { assert ( <STR_LIT> . contains ( SrcReg ) ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) . addImm ( <NUM_LIT> ) ; return ; } unsigned EltSize = <NUM_LIT> ; unsigned Opcode = <STR_LIT> ; if ( RI . isSGPRClass ( RC ) ) { if ( ! ( RI . getRegSizeInBits ( * RC ) % <NUM_LIT> ) ) { Opcode = <STR_LIT> ; EltSize = <NUM_LIT> ; } else { Opcode = <STR_LIT> ; EltSize = <NUM_LIT> ; } if ( ! RI . isSGPRClass ( RI . getPhysRegClass ( SrcReg ) ) ) { reportIllegalCopy ( this , MBB , MI , DL , DestReg , SrcReg , KillSrc ) ; return ; } } ArrayRef < int16_t > SubIndices = RI . getRegSplitParts ( RC , EltSize ) ; bool Forward = RI . getHWRegIndex ( DestReg ) <= RI . getHWRegIndex ( SrcReg ) ; for ( unsigned Idx = <NUM_LIT> ; Idx < SubIndices . size ( ) ; ++ Idx ) { unsigned SubIdx ; if ( Forward ) SubIdx = SubIndices [ Idx ] ; else SubIdx = SubIndices [ SubIndices . size ( ) - Idx - <NUM_LIT> ] ; MachineInstrBuilder Builder = BuildMI ( MBB , MI , DL , get ( Opcode ) , RI . getSubReg ( DestReg , SubIdx ) ) ; Builder . addReg ( RI . getSubReg ( SrcReg , SubIdx ) ) ; if ( Idx == <NUM_LIT> ) Builder . addReg ( DestReg , RegState :: Define | RegState


<s> if ( UseMI -> getParent ( ) != DefBB ) return true ; } else { int NumUse = <NUM_LIT> ; const int MaxUseScan = <NUM_LIT> ; for ( auto & UseInst : MRI . use_nodbg_instructions ( VReg ) ) { if ( UseInst . getParent ( ) != DefBB ) return true ; if ( NumUse ++ > MaxUseScan ) return true ; } } const int MaxInstScan = <NUM_LIT> ; int NumScan = <NUM_LIT> ; auto E = UseMI ? UseMI -> getIterator ( ) : DefBB -> end ( ) ; for ( auto I = std :: next ( DefMI . getIterator ( ) ) ; I != E ; ++ I ) { if ( I -> isDebugInstr ( ) ) continue ; if ( NumScan ++ > MaxInstScan ) return true ; if ( I -> modifiesRegister ( <STR_LIT> , TRI ) ) return true ; } return


<s> if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillSaveOpcode ( SpillSize ) ) ; if ( TargetRegisterInfo :: isVirtualRegister ( SrcReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( SrcReg , & <STR_LIT> ) ; } MachineInstrBuilder Spill = BuildMI ( MBB , MI , DL , OpDesc ) . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ; FrameInfo . setStackID ( FrameIndex , TargetStackID :: SGPRSpill ) ; if ( ST . hasScalarStores ( ) ) { Spill . addReg ( <STR_LIT> , RegState :: ImplicitDefine | RegState :: Dead ) ; } return ; } assert ( RI . hasVGPRs (


<s> unsigned NumSubRegs = RegSize / <NUM_LIT> ; assert ( NumSubRegs % <NUM_LIT> == <NUM_LIT> && NumSubRegs <= <NUM_LIT> && <STR_LIT> ) ; for ( unsigned Idx = <NUM_LIT> ; Idx < NumSubRegs ; Idx += <NUM_LIT> ) { Register CurRegLo = MRI . createVirtualRegister ( & <STR_LIT> ) ; Register CurRegHi = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CurRegLo ) . addReg ( VRsrc , VRsrcUndef , TRI -> getSubRegFromChannel ( Idx ) ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CurRegHi ) . addReg ( VRsrc , VRsrcUndef , TRI -> getSubRegFromChannel ( Idx + <NUM_LIT> ) ) ; ReadlanePieces . push_back ( CurRegLo ) ; ReadlanePieces . push_back ( CurRegHi ) ; Register CurReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CurReg ) . addReg ( CurRegLo ) . addImm ( <STR_LIT> ) . addReg ( CurRegHi ) . addImm ( <STR_LIT> ) ; Register NewCondReg = MRI . createVirtualRegister ( BoolXExecRC ) ; auto Cmp = BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , NewCondReg ) . addReg ( CurReg ) ; if ( NumSubRegs <= <NUM_LIT> ) Cmp . addReg ( VRsrc ) ; else Cmp . addReg ( VRsrc , VRsrcUndef , TRI -> getSubRegFromChannel ( Idx , <NUM_LIT> ) ) ; if ( CondReg == <STR_LIT> ) CondReg = NewCondReg ; else { Register AndReg = MRI . createVirtualRegister ( BoolXExecRC ) ; BuildMI ( LoopBB , I , DL , TII . get ( AndOpc )

, AndReg ) . addReg ( CondReg ) . addReg ( NewCondReg ) ;


<s> unsigned Exec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; unsigned SaveExecOpc = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; unsigned XorTermOpc = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; unsigned AndOpc = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; const auto * BoolXExecRC = TRI -> getRegClass ( <STR_LIT> ) ; MachineBasicBlock :: iterator I = LoopBB . begin ( ) ; SmallVector < Register , <NUM_LIT> > ReadlanePieces ; Register CondReg = <STR_LIT> ; Register VRsrc = Rsrc . getReg ( ) ; unsigned VRsrcUndef = getUndefRegState ( Rsrc . isUndef ( ) ) ; unsigned RegSize = TRI -> getRegSizeInBits ( Rsrc . getReg ( ) , MRI ) ; unsigned NumSubRegs = RegSize / <NUM_LIT> ; assert ( NumSubRegs % <NUM_LIT> == <NUM_LIT> && NumSubRegs <= <NUM_LIT> && <STR_LIT> ) ; for ( unsigned Idx = <NUM_LIT> ; Idx < NumSubRegs ; Idx += <NUM_LIT> ) { Register CurRegLo = MRI . createVirtualRegister ( & <STR_LIT> ) ; Register CurRegHi = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CurRegLo ) . addReg ( VRsrc , VRsrcUndef , TRI -> getSubRegFromChannel ( Idx ) ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CurRegHi ) . addReg ( VRsrc , VRsrcUndef , TRI -> getSubRegFromChannel ( Idx + <NUM_LIT> ) ) ; ReadlanePieces . push_back ( CurRegLo ) ; ReadlanePieces . push_back ( CurRegHi ) ; Register CurReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CurReg ) . addReg ( CurRegLo ) . addImm ( <STR_LIT> ) . addReg ( CurRegHi ) . addImm ( <STR_LIT> ) ; Register NewCondReg = MRI . createVirtualRegister ( BoolXExecRC ) ; auto Cmp = BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , NewCondReg ) . addReg ( CurReg ) ; if ( NumSubRegs <= <NUM_LIT> ) Cmp . addReg ( VRsrc ) ; else Cmp . addReg ( VRsrc , VRsrcUndef , TRI -> getSubRegFromChannel ( Idx , <NUM_LIT> ) ) ; if ( CondReg == <STR_LIT> ) CondReg = NewCondReg ; else { Register AndReg = MRI . createVirtualRegister ( BoolXExecRC ) ; BuildMI ( LoopBB , I , DL , TII . get ( AndOpc ) , AndReg ) . addReg ( CondReg ) . addReg ( NewCondReg ) ; CondReg = AndReg ; } } auto SRsrcRC = TRI -> getEquivalentSGPRClass ( MRI . getRegClass ( VRsrc ) ) ; Register SRsrc = MRI .


<s> return Size ; } if ( isVALU ( MI ) || isSALU ( MI ) ) { int Src0Idx = <STR_LIT> ( Opc , <STR_LIT> :: src0 ) ; if ( Src0Idx == - <NUM_LIT> ) return DescSize ; if ( isLiteralConstantLike ( MI . getOperand ( Src0Idx ) , Desc . OpInfo [ Src0Idx ] ) ) return isVOP3 ( MI ) ? <NUM_LIT> : ( DescSize + <NUM_LIT> ) ; int Src1Idx = <STR_LIT> ( Opc , <STR_LIT> :: src1 ) ; if ( Src1Idx == - <NUM_LIT> ) return DescSize ; if ( isLiteralConstantLike ( MI . getOperand ( Src1Idx ) , Desc . OpInfo [ Src1Idx ] ) ) return isVOP3 ( MI ) ? <NUM_LIT> : ( DescSize + <NUM_LIT> ) ; int Src2Idx = <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ; if ( Src2Idx == - <NUM_LIT> ) return DescSize ; if ( isLiteralConstantLike ( MI . getOperand ( Src2Idx ) , Desc . OpInfo [ Src2Idx ] ) ) return isVOP3 ( MI ) ? <NUM_LIT> : ( DescSize + <NUM_LIT> ) ; return DescSize ; } if ( isMIMG ( MI ) ) { int VAddr0Idx = <STR_LIT> ( Opc , <STR_LIT> :: vaddr0 ) ; if ( VAddr0Idx < <NUM_LIT> ) return <NUM_LIT> ; int RSrcIdx = <STR_LIT> ( Opc , <STR_LIT>


<s> if ( MI . mayStore ( ) && isSMRD ( MI ) ) return true ; if ( MI . isReturn ( ) ) return true ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) return true ; if ( MI . isCall ( ) || MI . isInlineAsm ( ) ) return true ; if ( modifiesModeRegister ( MI ) ) return true ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT>


<s> bool <ISA_LIT>InstrInfo :: hasUnwantedEffectsWhenEXECEmpty ( const MachineInstr & MI ) const { unsigned Opcode = MI . getOpcode ( ) ; if ( MI . mayStore ( ) && isSMRD ( MI ) ) return true ; if ( MI . isReturn ( ) ) return true ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) return true ; if ( MI . isCall ( ) || MI . isInlineAsm ( ) ) return true ; if ( modifiesModeRegister ( MI ) ) return true ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) return


<s> break ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const MCInstrDesc & MovRelDesc = get ( <STR_LIT> ) ; Register VecReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; bool IsUndef = MI . getOperand ( <NUM_LIT> ) . isUndef ( ) ; unsigned SubReg = <STR_LIT> + MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; assert ( VecReg == MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MachineInstr * MovRel = BuildMI ( MBB , MI , DL , MovRelDesc ) . addReg ( RI . getSubReg ( VecReg , SubReg ) , RegState :: Undef ) . add ( MI . getOperand ( <NUM_LIT> ) ) . addReg ( VecReg , RegState :: ImplicitDefine ) . addReg ( VecReg , RegState :: Implicit | ( IsUndef ? RegState :: Undef : <NUM_LIT> ) ) ; const int ImpDefIdx = MovRelDesc . getNumOperands ( ) + MovRelDesc . getNumImplicitUses ( ) ; const int ImpUseIdx = ImpDefIdx + <NUM_LIT> ; MovRel -> tieOperands ( ImpDefIdx , ImpUseIdx ) ; MI . eraseFromParent ( ) ; break ; } case <STR_LIT> : { MachineFunction & MF = * MBB . getParent ( ) ; Register Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register RegLo = RI . getSubReg ( Reg , <STR_LIT> ) ; Register RegHi = RI . getSubReg ( Reg , <STR_LIT> ) ; MIBundleBuilder Bundler ( MBB , MI ) ; Bundler . append ( BuildMI ( MF , DL , get ( <STR_LIT> ) , Reg ) ) ; Bundler . append ( BuildMI ( MF , DL , get ( <STR_LIT> ) , RegLo ) . addReg ( RegLo ) . add ( MI . getOperand ( <NUM_LIT> ) ) ) ; MachineInstrBuilder MIB = BuildMI ( MF , DL , get ( <STR_LIT> ) , RegHi ) . addReg ( RegHi ) ; MIB . add ( MI . getOperand ( <NUM_LIT> ) ) ; Bundler . append ( MIB ) ; finalizeBundle ( MBB , Bundler . begin ( ) ) ; MI . eraseFromParent ( ) ; break ; } case <STR_LIT> : { MI . setDesc ( get ( ST . isWave32 ( )


<s> BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addImm ( <NUM_LIT> ) . addReg ( TrueReg ) . addImm ( <NUM_LIT> ) . addReg ( FalseReg ) . addReg ( SReg ) ; break ; } case <ISA_LIT>InstrInfo :: EXECNZ : { Register SReg = MRI . createVirtualRegister ( BoolXExecRC ) ; Register SReg2 = MRI . createVirtualRegister ( RI . getBoolRC ( ) ) ; BuildMI ( MBB , I , DL , get ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , SReg2 ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , SReg ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addImm ( <NUM_LIT> ) . addReg ( FalseReg ) . addImm ( <NUM_LIT> ) . addReg ( TrueReg ) . addReg ( SReg ) ; break ; } case <ISA_LIT>InstrInfo :: EXECZ : { Register SReg = MRI . createVirtualRegister ( BoolXExecRC ) ; Register SReg2 = MRI . createVirtualRegister ( RI . getBoolRC ( ) ) ; BuildMI ( MBB , I , DL , get ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , SReg2 ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , SReg ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addImm ( <NUM_LIT> ) . addReg ( FalseReg ) . addImm ( <NUM_LIT> ) . addReg ( TrueReg ) . addReg ( SReg ) ; llvm_unreachable ( <STR_LIT> ) ; break ; } default : llvm_unreachable ( <STR_LIT> ) ; } } else { llvm_unreachable ( <STR_LIT>


<s> unsigned Align = FrameInfo . getObjectAlignment ( FrameIndex ) ; unsigned Size = FrameInfo . getObjectSize ( FrameIndex ) ; unsigned SpillSize = TRI -> getSpillSize ( * RC ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , Size , Align ) ; if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ; assert ( DestReg != <STR_LIT> && <STR_LIT> ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillRestoreOpcode ( SpillSize ) ) ; if ( Register :: isVirtualRegister ( DestReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( DestReg , & <STR_LIT> ) ; } if ( RI . spillSGPRToVGPR ( ) ) FrameInfo . setStackID ( FrameIndex , TargetStackID :: SGPRSpill ) ; BuildMI ( MBB , MI , DL , OpDesc , DestReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ; return ; } unsigned Opcode = RI . hasAGPRs ( RC ) ? getAGPRSpillRestoreOpcode ( SpillSize ) : getVGPRSpillRestoreOpcode ( SpillSize ) ; auto MIB = BuildMI ( MBB , MI , DL , get ( Opcode ) , DestReg ) ; if ( RI . hasAGPRs ( RC ) ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; Register Tmp = MRI . createVirtualRegister ( &


<s> if ( ! BaseOp1 . isReg ( ) || ! BaseOp2 . isReg ( ) ) return false ; if ( BaseOp1 . isIdenticalTo ( BaseOp2 ) ) return true ; if ( ! MI1 . hasOneMemOperand ( ) || ! MI2 . hasOneMemOperand ( ) ) return false ; auto MO1 = * MI1 . memoperands_begin ( ) ; auto MO2 = * MI2 . memoperands_begin ( ) ; if ( MO1 -> getAddrSpace ( ) != MO2 -> getAddrSpace ( ) ) return false ; auto Base1 = MO1 -> getValue ( ) ; auto Base2 = MO2 -> getValue ( ) ; if ( ! Base1 || ! Base2 ) return false ; const MachineFunction & MF = * MI1 . getParent ( ) -> getParent ( ) ; const DataLayout & DL = MF . getFunction ( ) . getParent ( ) -> getDataLayout ( ) ; Base1 = GetUnderlyingObject ( Base1


<s> auto MO2 = * MI2 . memoperands_begin ( ) ; if ( MO1 -> getAddrSpace ( ) != MO2 -> getAddrSpace ( ) ) return false ; auto Base1 = MO1 -> getValue ( ) ; auto Base2 = MO2 -> getValue ( ) ; if ( ! Base1 || ! Base2 ) return false ; const MachineFunction & MF = * MI1 . getParent ( ) -> getParent ( ) ; const DataLayout & DL = MF . getFunction ( ) . getParent ( ) -> getDataLayout ( ) ; Base1 = GetUnderlyingObject ( Base1 , DL ) ; Base2 = GetUnderlyingObject ( Base2 , DL ) ; if ( isa < UndefValue > ( Base1 ) || isa < UndefValue > ( Base2


<s> DiagnosticInfoUnsupported IllegalCopy ( MF -> getFunction ( ) , <STR_LIT> , DL , DS_Error ) ; LLVMContext & C = MF -> getFunction ( ) . getContext ( ) ; C . diagnose ( IllegalCopy


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return MI . getNumOperands ( ) == MI .


<s> ImmField = COffsetVal - RemainderOffset ; } else if ( COffsetVal >= <NUM_LIT> ) { ImmField = COffsetVal & maskTrailingOnes < uint64_t > ( NumBits ) ; RemainderOffset = COffsetVal - ImmField ; } assert ( isLegalFLATOffset ( ImmField , AddrSpace


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case


<s> if ( Src0 -> isReg ( ) && Src0 -> getReg ( ) == Reg ) { if ( ! Src1 -> isReg ( ) || RI . isSGPRClass ( MRI -> getRegClass ( Src1 -> getReg ( ) ) ) ) return false ; if ( ! Src2 -> isReg ( ) || RI . isSGPRClass ( MRI -> getRegClass ( Src2 -> getReg ( ) ) ) ) return false ; const int64_t Imm = DefMI . getOperand ( <NUM_LIT> ) . getImm ( ) ; UseMI . RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: omod ) ) ; UseMI . RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: clamp ) ) ; unsigned Src1Reg = Src1 -> getReg ( ) ; unsigned Src1SubReg = Src1 -> getSubReg ( ) ; Src0 -> setReg ( Src1Reg ) ; Src0 -> setSubReg ( Src1SubReg ) ; Src0 -> setIsKill ( Src1 -> isKill ( ) ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) UseMI . untieRegOperand ( <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ) ; Src1 -> ChangeToImmediate ( Imm ) ; removeModOperands ( UseMI ) ; UseMI . setDesc ( get ( IsF32 ? <STR_LIT> : <STR_LIT> ) ) ; bool DeleteDef = MRI -> hasOneNonDBGUse ( Reg ) ; if ( DeleteDef ) DefMI . eraseFromParent ( ) ; return true ; } if ( Src2 -> isReg ( ) && Src2 -> getReg ( ) == Reg ) { if ( ! Src0 -> isImm ( ) && ( Src0 -> isReg ( ) && RI . isSGPRClass ( MRI -> getRegClass ( Src0 -> getReg ( ) ) ) ) ) return false ; if ( ! Src1 -> isReg ( ) || RI . isSGPRClass ( MRI -> getRegClass ( Src1 -> getReg ( ) ) ) ) return false ; const int64_t Imm = DefMI . getOperand ( <NUM_LIT> ) . getImm ( ) ; UseMI . RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: omod ) ) ; UseMI . RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: clamp ) ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) UseMI . untieRegOperand ( <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ) ; Src2 -> ChangeToImmediate ( Imm ) ; removeModOperands ( UseMI ) ; UseMI . setDesc ( get ( IsF32 ? <STR_LIT> : <STR_LIT> ) ) ; bool DeleteDef = MRI -> hasOneNonDBGUse ( Reg ) ; if ( DeleteDef ) DefMI . eraseFromParent ( ) ; return true ; } } return


<s> return ; } if ( ! ST . isVGPRSpillingEnabled ( MF -> getFunction ( ) ) ) { LLVMContext & Ctx = MF -> getFunction ( ) . getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) ; return ; } assert ( RI . hasVGPRs ( RC ) && <STR_LIT> ) ; unsigned Opcode = getVGPRSpillRestoreOpcode ( SpillSize ) ; BuildMI ( MBB , MI , DL , get ( Opcode ) , DestReg ) . addFrameIndex ( FrameIndex ) . addReg ( MFI -> getScratchRSrcReg ( ) ) . addReg ( MFI ->

getFrameOffsetReg ( ) ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ;


<s> void <ISA_LIT>InstrInfo :: loadRegFromStackSlot ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator MI , unsigned DestReg , int FrameIndex , const TargetRegisterClass * RC , const TargetRegisterInfo * TRI ) const { MachineFunction * MF = MBB . getParent ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineFrameInfo & FrameInfo = MF -> getFrameInfo ( ) ; DebugLoc DL = MBB . findDebugLoc ( MI ) ; unsigned Align = FrameInfo . getObjectAlignment ( FrameIndex ) ; unsigned Size = FrameInfo . getObjectSize ( FrameIndex ) ; unsigned SpillSize = TRI -> getSpillSize ( * RC ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , Size , Align ) ; if ( RI . isSGPRClass ( RC )


<s> unsigned <ISA_LIT>InstrInfo :: getInstSizeInBytes ( const MachineInstr & MI ) const { unsigned Opc = MI . getOpcode ( ) ; const MCInstrDesc & Desc = getMCOpcodeFromPseudo ( Opc ) ; unsigned DescSize = Desc . getSize ( ) ; if ( DescSize != <NUM_LIT> && DescSize != <NUM_LIT> ) return DescSize ; if ( isFixedSize ( MI ) ) return DescSize ; if ( isVALU ( MI ) || isSALU ( MI ) ) { int Src0Idx = <STR_LIT> ( Opc , <STR_LIT> :: src0 ) ; if ( Src0Idx == - <NUM_LIT> ) return <NUM_LIT> ; if ( isLiteralConstantLike ( MI . getOperand ( Src0Idx ) , Desc . OpInfo


<s> unsigned <ISA_LIT>InstrInfo :: getInstSizeInBytes ( const MachineInstr & MI ) const { unsigned Opc = MI . getOpcode ( ) ; const MCInstrDesc & Desc = getMCOpcodeFromPseudo ( Opc ) ; unsigned DescSize = Desc . getSize ( ) ; if ( DescSize != <NUM_LIT> && DescSize != <NUM_LIT> ) return DescSize ; if ( isFixedSize ( MI ) ) return DescSize ; if ( isVALU ( MI ) || isSALU ( MI ) ) { int Src0Idx = <STR_LIT> ( Opc , <STR_LIT> :: src0 ) ; if ( Src0Idx == - <NUM_LIT> ) return <NUM_LIT> ; if ( isLiteralConstantLike ( MI . getOperand ( Src0Idx ) , Desc . OpInfo [ Src0Idx ] ) ) return <NUM_LIT> ; int Src1Idx = <STR_LIT> ( Opc , <STR_LIT> :: src1 ) ; if ( Src1Idx == - <NUM_LIT> ) return <NUM_LIT> ; if ( isLiteralConstantLike ( MI . getOperand ( Src1Idx ) , Desc . OpInfo [ Src1Idx ] ) ) return <NUM_LIT> ; return <NUM_LIT> ; } if ( DescSize == <NUM_LIT> ) return <NUM_LIT> ; switch ( Opc ) { case TargetOpcode :: IMPLICIT_DEF : case TargetOpcode :: KILL : case TargetOpcode :: DBG_VALUE : case TargetOpcode :: EH_LABEL : return <NUM_LIT> ; case TargetOpcode :: BUNDLE


<s> if ( MI . memoperands_empty ( ) ) return true ; for ( const MachineMemOperand * MMO : MI . memoperands ( ) ) { if ( MMO -> getAddrSpace ( ) == ST . get<ISA_LIT>AS ( ) . FLAT_ADDRESS ) return true ; } return false


<s> if ( hasModifiersSet ( * UseMI , <STR_LIT> :: src0_modifiers ) || hasModifiersSet ( * UseMI , <STR_LIT> :: src1_modifiers ) || hasModifiersSet ( * UseMI , <STR_LIT> :: src2_modifiers ) ) { return false ; } MachineOperand * Src0 = getNamedOperand ( * UseMI , <STR_LIT> :: src0 ) ; MachineOperand * Src1 = getNamedOperand ( * UseMI , <STR_LIT> :: src1 ) ; MachineOperand * Src2 = getNamedOperand ( * UseMI , <STR_LIT> :: src2 ) ; if ( Src0 -> isReg ( ) && Src0 -> getReg ( ) == Reg ) { if ( ! Src1 -> isReg ( ) || RI . isSGPRClass ( MRI -> getRegClass ( Src1 -> getReg ( ) ) ) ) return false ; if ( ! Src2 -> isReg ( ) || RI . isSGPRClass ( MRI -> getRegClass ( Src2 -> getReg ( ) ) ) ) return false ; const int64_t Imm = DefMI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; UseMI -> RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: omod ) ) ; UseMI -> RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: clamp ) ) ; unsigned Src1Reg = Src1 -> getReg ( ) ; unsigned Src1SubReg = Src1 -> getSubReg ( ) ; Src0 -> setReg ( Src1Reg ) ; Src0 -> setSubReg ( Src1SubReg ) ; Src0 -> setIsKill ( Src1 -> isKill ( ) ) ; if ( Opc == <STR_LIT> ) { UseMI -> untieRegOperand ( <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ) ; } Src1 -> ChangeToImmediate ( Imm ) ; removeModOperands ( * UseMI ) ; UseMI -> setDesc ( get ( <STR_LIT> ) ) ; bool DeleteDef = MRI -> hasOneNonDBGUse ( Reg ) ; if ( DeleteDef ) DefMI -> eraseFromParent


<s> uint64_t Rsrc23 = getDefaultRsrcDataFormat ( ) | <STR_LIT> | <NUM_LIT> ; uint64_t EltSizeValue = Log2_32 ( ST . getMaxPrivateElementSize ( ) )


<s> if ( isSMRD ( * FirstLdSt ) && isSMRD ( * SecondLdSt ) ) return true ; return ( isMUBUF ( * FirstLdSt ) || isMTBUF ( * FirstLdSt ) ) && ( isMUBUF ( * SecondLdSt ) || isMTBUF


<s> bool <ISA_LIT>InstrInfo :: shouldClusterLoads ( MachineInstr * FirstLdSt , MachineInstr * SecondLdSt , unsigned NumLoads ) const { if ( NumLoads > <NUM_LIT> ) return false ; if ( isDS ( * FirstLdSt ) && isDS ( * SecondLdSt ) )


<s> MFI -> setHasSpilledSGPRs ( ) ; unsigned Opcode = getSGPRSpillSaveOpcode ( RC -> getSize ( ) ) ; BuildMI ( MBB , MI , DL , get ( Opcode ) ) . addReg ( SrcReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) ; return ; } if ( ! ST . isVGPRSpillingEnabled ( MFI ) ) { LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ) . addReg ( SrcReg ) ; return ; } assert ( RI . hasVGPRs ( RC ) && <STR_LIT> ) ; unsigned Opcode = getVGPRSpillSaveOpcode ( RC -> getSize


<s> } RS -> enterBasicBlock ( Entry ) ; unsigned STmp0 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; unsigned STmp1 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp0 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Z ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Y ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( STmp1 ) . addReg ( STmp0 ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp1 ) . addReg ( TIDIGXReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp0 ) . addReg ( TIDIGYReg ) . addReg ( TIDReg ) ; getAddNoCarry ( Entry , Insert , DL , TIDReg ) . addReg ( TIDReg ) . addReg ( TIDIGZReg ) ; } else { BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( - <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) .

addImm ( - <NUM_LIT> ) . addReg ( TIDReg ) ;


<s> BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp0 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Z ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Y ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( STmp1 ) . addReg ( STmp0 ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp1 ) . addReg ( TIDIGXReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp0 ) . addReg ( TIDIGYReg ) . addReg ( TIDReg ) ; getAddNoCarry ( Entry , Insert , DL , TIDReg ) . addReg ( TIDReg ) . addReg ( TIDIGZReg ) ; } else { BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( - <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( - <NUM_LIT> ) . addReg ( TIDReg ) ; } BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( <NUM_LIT>


<s> } unsigned Width0 = ( * MIa . memoperands_begin ( ) ) -> getSize ( ) ; unsigned Width1 = ( * MIb . memoperands_begin ( ) ) -> getSize ( ) ; if ( offsetsDoNotOverlap ( Width0 , Offset0 , Width1 , Offset1


<s> static void copyFlagsToImplicitVCC ( MachineInstr & MI , const MachineOperand & Orig ) { for ( MachineOperand & Use : MI .


<s> for ( MachineOperand & Use : MI . implicit_operands ( ) ) { if ( Use . isUse ( ) && Use . getReg


<s> unsigned SRsrcSub1 = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrcSub2 = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrcSub3 = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrc = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub0 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub1 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub2 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub3 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrc ) . addReg ( SRsrcSub0 ) . addImm ( <STR_LIT> ) . addReg ( SRsrcSub1 ) . addImm ( <STR_LIT> ) . addReg ( SRsrcSub2 ) . addImm ( <STR_LIT> ) . addReg ( SRsrcSub3 ) . addImm ( <STR_LIT> ) ; Rsrc . setReg ( SRsrc ) ; Rsrc . setIsKill ( true ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CondReg0 ) . addReg ( SRsrc , <NUM_LIT> , <STR_LIT> ) . addReg ( VRsrc , <NUM_LIT> , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CondReg1 ) . addReg ( SRsrc , <NUM_LIT> , <STR_LIT> ) . addReg (


<s> unsigned SRsrcSub3 = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrc = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub0 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub1 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub2 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub3 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrc ) . addReg ( SRsrcSub0 ) . addImm ( <STR_LIT> ) . addReg ( SRsrcSub1 ) . addImm ( <STR_LIT> ) . addReg ( SRsrcSub2 ) . addImm ( <STR_LIT> ) . addReg ( SRsrcSub3 ) . addImm ( <STR_LIT> ) ; Rsrc . setReg ( SRsrc ) ; Rsrc . setIsKill ( true ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CondReg0 ) . addReg ( SRsrc , <NUM_LIT> , <STR_LIT> ) . addReg ( VRsrc , <NUM_LIT> , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CondReg1 ) . addReg ( SRsrc , <NUM_LIT> , <STR_LIT> ) . addReg ( VRsrc , <NUM_LIT> , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , AndCond ) . addReg ( CondReg0 ) . addReg ( CondReg1 ) ; MRI . setSimpleHint ( SaveExec , AndCond ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SaveExec ) . addReg ( AndCond , RegState :: Kill ) ; I = LoopBB . end ( ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , <STR_LIT>

) . addReg ( <STR_LIT> ) . addReg ( SaveExec ) ;


<s> static std :: tuple < unsigned , unsigned > extractRsrcPtr ( const <ISA_LIT>InstrInfo & TII , MachineInstr & MI , MachineOperand & Rsrc ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; MachineFunction & MF = * MBB . getParent ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; unsigned RsrcPtr = TII . buildExtractSubReg ( MI , MRI , Rsrc , & <STR_LIT> , <STR_LIT> , & <STR_LIT> ) ; unsigned Zero64 = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrcFormatLo = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrcFormatHi = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned NewSRsrc = MRI . createVirtualRegister ( & <STR_LIT> ) ; uint64_t RsrcDataFormat = TII


<s> static std :: tuple < unsigned , unsigned > extractRsrcPtr ( const <ISA_LIT>InstrInfo & TII , MachineInstr & MI , MachineOperand & Rsrc ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; MachineFunction & MF = * MBB . getParent ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; unsigned RsrcPtr = TII . buildExtractSubReg ( MI , MRI , Rsrc , & <STR_LIT> , <STR_LIT> , & <STR_LIT> ) ; unsigned Zero64 = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrcFormatLo = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrcFormatHi = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned NewSRsrc = MRI . createVirtualRegister ( & <STR_LIT> ) ; uint64_t RsrcDataFormat = TII . getDefaultRsrcDataFormat ( ) ; BuildMI ( MBB , MI , MI . getDebugLoc ( ) , TII . get ( <STR_LIT> ) , Zero64 ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , MI , MI . getDebugLoc ( ) , TII . get ( <STR_LIT> ) , SRsrcFormatLo ) . addImm ( RsrcDataFormat & <NUM_LIT> ) ; BuildMI ( MBB , MI , MI . getDebugLoc ( ) , TII . get ( <STR_LIT> ) , SRsrcFormatHi ) . addImm ( RsrcDataFormat >> <NUM_LIT> ) ; BuildMI ( MBB , MI , MI . getDebugLoc ( ) , TII . get ( <STR_LIT> ) , NewSRsrc ) . addReg ( Zero64 ) . addImm ( <STR_LIT> ) . addReg ( SRsrcFormatLo ) . addImm

( <STR_LIT> ) . addReg ( SRsrcFormatHi ) . addImm ( <STR_LIT> ) ;


<s> if ( ! Desc . isCommutable ( ) ) return false ; unsigned Opc = Desc . getOpcode ( ) ; int Src0Idx = <STR_LIT> ( Opc , <STR_LIT> :: src0 ) ; if ( Src0Idx == - <NUM_LIT> ) return false ; int Src1Idx = <STR_LIT> ( Opc , <STR_LIT> :: src1 ) ; if ( Src1Idx == - <NUM_LIT> ) return false ; return fixCommutedOpIndices ( SrcOpIdx0 , SrcOpIdx1 , Src0Idx , Src1Idx


<s> case PseudoSourceValue :: GOT : case PseudoSourceValue :: JumpTable : case PseudoSourceValue :: GlobalValueCallEntry : case PseudoSourceValue :: ExternalSymbolCallEntry : case PseudoSourceValue


<s> case PseudoSourceValue :: ConstantPool : case PseudoSourceValue :: GOT : case PseudoSourceValue :: JumpTable : case PseudoSourceValue :: GlobalValueCallEntry : case PseudoSourceValue :: ExternalSymbolCallEntry : case PseudoSourceValue :: TargetCustom : return <ISA_LIT>AS :: CONSTANT_ADDRESS


<s> if ( isLiteralConstantLike ( MI . getOperand ( Src1Idx ) , Desc . OpInfo [ Src1Idx ] ) ) return DescSize + <NUM_LIT> ; int Src2Idx = <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ; if ( Src2Idx == - <NUM_LIT> ) return DescSize ; if ( isLiteralConstantLike ( MI . getOperand ( Src2Idx ) , Desc . OpInfo [ Src2Idx ] ) ) return DescSize + <NUM_LIT> ; return DescSize ; } switch ( Opc ) { case TargetOpcode :: IMPLICIT_DEF : case TargetOpcode :: KILL : case TargetOpcode :: DBG_VALUE : case TargetOpcode :: EH_LABEL : return <NUM_LIT> ; case TargetOpcode :: BUNDLE : return getInstBundleSize ( MI ) ; case TargetOpcode :: INLINEASM : { const MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; const char * AsmStr = MI . getOperand (


<s> unsigned Opc = LdSt . getOpcode ( ) ; if ( isDS ( LdSt ) ) { const MachineOperand * OffsetImm = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; if ( OffsetImm ) { BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: addr ) ; Offset = OffsetImm -> getImm ( ) ; assert ( BaseOp -> isReg ( ) && <STR_LIT> <STR_LIT> ) ; return true ; } const MachineOperand * Offset0Imm = getNamedOperand ( LdSt , <STR_LIT> :: offset0 ) ; const MachineOperand * Offset1Imm = getNamedOperand ( LdSt , <STR_LIT> :: offset1 ) ; uint8_t Offset0 = Offset0Imm -> getImm ( ) ; uint8_t Offset1 = Offset1Imm -> getImm ( ) ; if ( Offset1 > Offset0 && Offset1 - Offset0 == <NUM_LIT> ) { unsigned EltSize ; if ( LdSt . mayLoad ( ) ) EltSize = TRI -> getRegSizeInBits ( * getOpRegClass ( LdSt , <NUM_LIT> ) ) / <NUM_LIT> ; else { assert ( LdSt . mayStore ( ) ) ; int Data0Idx = <STR_LIT> ( Opc , <STR_LIT> :: data0 ) ; EltSize = TRI -> getRegSizeInBits ( * getOpRegClass ( LdSt , Data0Idx ) ) / <NUM_LIT> ; } if ( isStride64 ( Opc ) ) EltSize *= <NUM_LIT> ; BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: addr ) ; Offset = EltSize * Offset0 ; assert ( BaseOp -> isReg ( ) && <STR_LIT> <STR_LIT> ) ; return true ; } return false ; } if ( isMUBUF ( LdSt ) || isMTBUF ( LdSt ) ) { const MachineOperand * SOffset = getNamedOperand ( LdSt , <STR_LIT> :: soffset ) ; if ( SOffset && SOffset -> isReg ( ) ) return false ; MachineOperand * AddrReg = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ; if ( ! AddrReg ) return false ; const MachineOperand * OffsetImm = getNamedOperand ( LdSt


<s> static TargetInstrInfo :: RegSubRegPair getRegOrUndef ( const MachineOperand & RegOpnd ) { assert ( RegOpnd . isReg ( ) ) ; return RegOpnd . isUndef ( ) ? TargetInstrInfo :: RegSubRegPair


<s> for ( unsigned I = <NUM_LIT> , E = ( MI . getNumOperands ( ) - <NUM_LIT> ) / <NUM_LIT> ; I < E ; ++ I ) if ( MI . getOperand ( <NUM_LIT> + <NUM_LIT> * I + <NUM_LIT> ) . getImm ( ) == SubReg ) { auto & RegOp = MI . getOperand ( <NUM_LIT> + <NUM_LIT> * I ) ; return getRegOrUndef ( RegOp ) ; } return TargetInstrInfo :: RegSubRegPair


<s> if ( Op1 . isUndef ( ) ) return nullptr ; RSR = getRegSubRegPair ( Op1 ) ; DefInst = MRI . getVRegDef ( RSR . Reg ) ; } break ; } default : if ( followSubRegDef ( * MI , RSR ) ) { if ( ! RSR . Reg ) return nullptr ; DefInst = MRI . getVRegDef ( RSR . Reg ) ; } } if ( ! DefInst


<s> bool <ISA_LIT>InstrInfo :: hasUnwantedEffectsWhenEXECEmpty ( const MachineInstr & MI ) const { unsigned Opcode = MI . getOpcode ( ) ; if ( MI . mayStore ( ) && isSMRD


<s> if ( MI . mayStore ( ) && isSMRD ( MI ) ) return true ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) return true ; if ( MI . isInlineAsm ( ) ) return true ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) return true ; return false


<s> bool <ISA_LIT>InstrInfo :: isAlwaysGDS ( uint16_t Opcode ) const { return Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ||


<s> bool <ISA_LIT>InstrInfo :: isAlwaysGDS ( uint16_t Opcode ) const { return Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT>


<s> int Idx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: sbase ) ; if ( Idx == - <NUM_LIT> ) return false ; const auto RCID = MI . getDesc ( ) . OpInfo [ Idx


<s> case <STR_LIT> : { int32_t Trunc = static_cast < int32_t > ( Imm ) ; return <STR_LIT> ( Trunc , ST . hasInv2PiInlineImm ( ) ) ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ( MO . getImm ( ) , ST . hasInv2PiInlineImm ( ) ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( isInt < <NUM_LIT> > ( Imm ) || isUInt < <NUM_LIT> > ( Imm ) ) { int16_t Trunc = static_cast < int16_t > ( Imm ) ; return ST . has16BitInsts ( ) && <STR_LIT> ( Trunc , ST . hasInv2PiInlineImm ( ) ) ; } return false ; } case <STR_LIT> : case <STR_LIT> : { if ( isUInt < <NUM_LIT> > ( Imm ) ) { int16_t Trunc = static_cast < int16_t > ( Imm ) ; return ST . has16BitInsts ( ) && <STR_LIT> ( Trunc , ST . hasInv2PiInlineImm ( ) ) ; } if ( ! ( Imm &


<s> if ( ! Addr || ! Addr -> isFI ( ) ) return <STR_LIT> ; assert ( ! MI . memoperands_empty ( ) && ( * MI . memoperands_begin ( ) ) -> getAddrSpace ( ) == <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; FrameIndex = Addr -> getIndex ( ) ; return getNamedOperand ( MI , <STR_LIT> :: vdata )


<s> void <ISA_LIT>InstrInfo :: legalizeOperandsSMRD ( MachineRegisterInfo & MRI , MachineInstr & MI ) const { MachineOperand * SBase = getNamedOperand ( MI , <STR_LIT> :: sbase ) ; if ( SBase && ! RI . isSGPRClass ( MRI . getRegClass ( SBase -> getReg ( ) ) ) ) { unsigned SGPR = readlaneVGPRToSGPR ( SBase -> getReg ( ) ,


<s> MachineFrameInfo & FrameInfo = MF -> getFrameInfo ( ) ; const DebugLoc & DL = MBB . findDebugLoc ( MI ) ; unsigned Align = FrameInfo . getObjectAlignment ( FrameIndex ) ; unsigned Size = FrameInfo . getObjectSize ( FrameIndex ) ; unsigned SpillSize = TRI -> getSpillSize ( * RC ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , Size , Align ) ; if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillRestoreOpcode ( SpillSize ) ) ; if ( TargetRegisterInfo :: isVirtualRegister ( DestReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( DestReg , & <STR_LIT> ) ; } FrameInfo . setStackID ( FrameIndex , <STR_LIT> ) ; MachineInstrBuilder Spill = BuildMI ( MBB , MI , DL , OpDesc , DestReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getFrameOffsetReg ( ) , RegState :: Implicit ) ; if ( ST . hasScalarStores ( ) ) { Spill . addReg ( <STR_LIT> , RegState :: ImplicitDefine | RegState :: Dead ) ; } return ; } assert ( RI . hasVGPRs ( RC ) && <STR_LIT> ) ; unsigned Opcode = getVGPRSpillRestoreOpcode ( SpillSize ) ; BuildMI ( MBB , MI , DL , get ( Opcode ) , DestReg ) . addFrameIndex ( FrameIndex ) . addReg ( MFI -> getScratchRSrcReg ( ) )

. addReg ( MFI -> getFrameOffsetReg ( ) ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ;


<s> MachineFunction & MF = * MBB . getParent ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; MachineBasicBlock :: iterator I ( & MI ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; unsigned SaveExec = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII . get ( <STR_LIT> ) , SaveExec ) . addReg ( <STR_LIT> ) ; for ( auto & MO : MI . uses ( ) ) { if ( MO . isReg ( ) && MO . isUse ( ) ) { MRI . clearKillFlags ( MO . getReg ( ) ) ; } } MachineBasicBlock * LoopBB = MF . CreateMachineBasicBlock ( ) ; MachineBasicBlock * RemainderBB = MF . CreateMachineBasicBlock ( ) ; MachineFunction :: iterator MBBI ( MBB ) ; ++ MBBI ; MF . insert ( MBBI , LoopBB ) ; MF . insert ( MBBI , RemainderBB ) ; LoopBB -> addSuccessor ( LoopBB ) ; LoopBB -> addSuccessor


<s> MachineBasicBlock & MBB = * Inst . getParent ( ) ; MachineRegisterInfo & MRI = MBB . getParent ( ) -> getRegInfo ( ) ; MachineBasicBlock :: iterator MII = Inst ; const DebugLoc & DL = Inst . getDebugLoc ( ) ; MachineOperand & Dest = Inst . getOperand ( <NUM_LIT> ) ; MachineOperand & Src0 = Inst . getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = Inst . getOperand ( <NUM_LIT> ) ; if ( ST . hasDLInsts ( ) ) { unsigned NewDest = MRI . createVirtualRegister ( & <STR_LIT> ) ; legalizeGenericOperand ( MBB , MII , & <STR_LIT> , Src0 , MRI , DL ) ; legalizeGenericOperand ( MBB , MII , & <STR_LIT> , Src1 , MRI , DL ) ; BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , NewDest ) . add ( Src0 ) . add ( Src1 ) ; MRI . replaceRegWith ( Dest . getReg ( ) , NewDest ) ; addUsersToMoveToVALUWorklist ( NewDest , MRI , Worklist ) ; } else { bool Src0IsSGPR = Src0 . isReg ( ) && RI . isSGPRClass ( MRI . getRegClass ( Src0 . getReg ( ) ) ) ; bool Src1IsSGPR = Src1 . isReg ( ) && RI . isSGPRClass ( MRI . getRegClass ( Src1 . getReg ( ) ) ) ; MachineInstr * Not = nullptr ; MachineInstr * Xor = nullptr ; unsigned Temp = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned NewDest = MRI . createVirtualRegister ( &


<s> bool <ISA_LIT>InstrInfo :: mayAccessFlatAddressSpace ( const MachineInstr & MI ) const { if ( ! isFLAT ( MI ) ) return false ; if ( MI . memoperands_empty ( ) ) return true ; for ( const MachineMemOperand * MMO : MI . memoperands ( ) ) { if ( MMO -> getAddrSpace ( ) == <ISA_LIT>AS :: FLAT_ADDRESS


<s> if ( MI . memoperands_empty ( ) ) return true ; for ( const MachineMemOperand * MMO : MI . memoperands ( ) ) { if ( MMO -> getAddrSpace ( ) == <ISA_LIT>AS :: FLAT_ADDRESS ) return true ; } return false


<s> if ( ! MI1 . hasOneMemOperand ( ) || ! MI2 . hasOneMemOperand ( ) ) return false ; auto MO1 = * MI1 . memoperands_begin ( ) ; auto MO2 = * MI2 . memoperands_begin ( ) ; if ( MO1 -> getAddrSpace ( ) != MO2 -> getAddrSpace ( ) ) return false ; auto Base1 = MO1 -> getValue ( ) ; auto Base2 = MO2 -> getValue ( ) ; if ( ! Base1 || ! Base2 ) return false ; const MachineFunction & MF = * MI1 . getParent ( ) ->


<s> assert ( Opc == <STR_LIT> || Opc == <STR_LIT> ) ; unsigned NewOpc = Opc == <STR_LIT> ? <STR_LIT> : <STR_LIT> ; assert ( Inst . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) ; Inst . RemoveOperand ( <NUM_LIT> ) ; Inst . setDesc ( get ( NewOpc ) ) ; Inst . addImplicitDefUseOperands ( * MBB . getParent ( ) ) ; MRI . replaceRegWith ( OldDstReg , ResultReg ) ; legalizeOperands ( Inst , MDT ) ; addUsersToMoveToVALUWorklist ( ResultReg , MRI


<s> case <STR_LIT> : if ( ST . hasDLInsts ( ) ) splitScalar64BitBinaryOp ( Worklist , Inst , <STR_LIT> , MDT ) ; else splitScalar64BitXnor ( Worklist , Inst , MDT ) ; Inst . eraseFromParent ( ) ; continue ; case <STR_LIT> : splitScalar64BitBinaryOp ( Worklist , Inst , <STR_LIT> , MDT ) ; Inst . eraseFromParent ( ) ; continue ; case <STR_LIT> : splitScalar64BitBinaryOp ( Worklist , Inst , <STR_LIT> , MDT ) ; Inst . eraseFromParent ( ) ; continue ; case <STR_LIT> : splitScalar64BitUnaryOp ( Worklist , Inst , <STR_LIT> ) ; Inst . eraseFromParent ( ) ; continue ; case <STR_LIT> : splitScalar64BitBCNT ( Worklist , Inst ) ; Inst . eraseFromParent ( ) ; continue ; case <STR_LIT> : splitScalar64BitBFE ( Worklist , Inst ) ; Inst . eraseFromParent ( ) ; continue ; case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : lowerScalarAbs ( Worklist , Inst ) ; Inst . eraseFromParent ( ) ; continue ; case <STR_LIT> : case <STR_LIT> : BuildMI ( * MBB , Inst , Inst . getDebugLoc ( ) , get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) ; break ; case <STR_LIT> : case <STR_LIT> : llvm_unreachable ( <STR_LIT> ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : movePackToVALU ( Worklist , MRI , Inst ) ; Inst . eraseFromParent ( ) ; continue ; case <STR_LIT> : lowerScalarXnor ( Worklist , Inst ) ; Inst . eraseFromParent ( ) ; continue ; case <STR_LIT> : splitScalarNotBinop ( Worklist , Inst , <STR_LIT> ) ; Inst . eraseFromParent ( ) ; continue ; case <STR_LIT> : splitScalarNotBinop ( Worklist , Inst , <STR_LIT> ) ; Inst . eraseFromParent ( ) ; continue ; case <STR_LIT> : splitScalarBinOpN2 ( Worklist , Inst , <STR_LIT> ) ; Inst . eraseFromParent ( ) ; continue ; case <STR_LIT>


<s> <ISA_LIT>EncodingFamily Gen = subtargetEncodingFamily ( ST ) ; if ( ( get ( Opcode ) . TSFlags & <STR_LIT> ) != <NUM_LIT> && ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 ) Gen = <STR_LIT> ; if ( get ( Opcode ) . TSFlags & <STR_LIT> ) Gen = ST . getGeneration ( ) == <ISA_LIT>Subtarget :: GFX9 ? <STR_LIT> : <STR_LIT> ; if ( ST . hasUnpackedD16VMem ( ) && ( get ( Opcode ) . TSFlags & <STR_LIT> ) ) Gen = <STR_LIT> ; int MCOp = <STR_LIT> ( Opcode , Gen ) ; if ( MCOp == - <NUM_LIT> ) return Opcode ; if ( MCOp == ( uint16_t )


<s> unsigned <ISA_LIT>InstrInfo :: readlaneVGPRToSGPR ( unsigned SrcReg , MachineInstr & UseMI , MachineRegisterInfo & MRI ) const { const TargetRegisterClass * VRC = MRI . getRegClass ( SrcReg ) ; const TargetRegisterClass * SRC = RI . getEquivalentSGPRClass ( VRC ) ; unsigned DstReg = MRI . createVirtualRegister ( SRC ) ; unsigned SubRegs = RI . getRegSizeInBits ( * VRC ) / <NUM_LIT> ; if ( SubRegs == <NUM_LIT> ) { BuildMI ( * UseMI . getParent ( ) , UseMI , UseMI . getDebugLoc ( ) , get ( <STR_LIT> ) , DstReg ) . addReg ( SrcReg ) ; return DstReg ; } SmallVector < unsigned , <NUM_LIT> > SRegs ; for ( unsigned i = <NUM_LIT> ; i < SubRegs ; ++ i ) { unsigned SGPR = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * UseMI . getParent ( ) , UseMI , UseMI . getDebugLoc ( ) , get ( <STR_LIT> ) , SGPR ) . addReg ( SrcReg , <NUM_LIT> , RI . getSubRegFromChannel ( i ) ) ; SRegs . push_back ( SGPR ) ; } MachineInstrBuilder MIB = BuildMI ( * UseMI . getParent ( ) , UseMI , UseMI .

getDebugLoc ( ) , get ( <STR_LIT> ) , DstReg ) ;


<s> const MachineOperand * FirstDst = nullptr ; const MachineOperand * SecondDst = nullptr ; if ( ( isMUBUF ( FirstLdSt ) && isMUBUF ( SecondLdSt ) ) || ( isMTBUF ( FirstLdSt ) && isMTBUF ( SecondLdSt ) ) || ( isFLAT ( FirstLdSt ) && isFLAT ( SecondLdSt ) ) ) { const unsigned MaxGlobalLoadCluster = <NUM_LIT> ; if ( NumLoads > MaxGlobalLoadCluster ) return false ; FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> :: vdata ) ; if ( ! FirstDst ) FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> :: vdst ) ; SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: vdata ) ; if ( ! SecondDst ) SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: vdst ) ; } else if ( isSMRD ( FirstLdSt ) && isSMRD ( SecondLdSt ) ) { FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> :: sdst ) ; SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: sdst ) ; } else if ( isDS ( FirstLdSt ) &&


<s> bool <ISA_LIT>InstrInfo :: shouldScheduleLoadsNear ( SDNode * Load0 , SDNode * Load1 , int64_t Offset0 , int64_t Offset1 , unsigned NumLoads ) const { assert ( Offset1 > Offset0 && <STR_LIT>


<s> const TargetRegisterClass * Src1RC = MRI . getRegClass ( Src1 . getReg ( ) ) ; const TargetRegisterClass * Src0SubRC = RI . getSubRegClass ( Src0RC , <STR_LIT> ) ; const TargetRegisterClass * Src1SubRC = RI . getSubRegClass ( Src1RC , <STR_LIT> ) ; MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm ( MII , MRI , Src0 , Src0RC , <STR_LIT> , Src0SubRC ) ; MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm ( MII , MRI , Src1 , Src1RC , <STR_LIT> , Src1SubRC ) ; MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm ( MII , MRI , Src0 , Src0RC , <STR_LIT> , Src0SubRC ) ; MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm ( MII , MRI , Src1 , Src1RC , <STR_LIT> , Src1SubRC ) ; unsigned LoOpc = IsAdd ? <STR_LIT> : <STR_LIT> ; MachineInstr * LoHalf = BuildMI ( MBB , MII , DL , get ( LoOpc ) , DestSub0 ) . addReg ( CarryReg , RegState :: Define ) . add ( SrcReg0Sub0 ) . add ( SrcReg1Sub0 ) ; unsigned HiOpc = IsAdd ? <STR_LIT> : <STR_LIT> ; MachineInstr * HiHalf = BuildMI ( MBB , MII , DL , get ( HiOpc ) , DestSub1 ) . addReg ( DeadCarryReg , RegState :: Define | RegState :: Dead ) . add ( SrcReg0Sub1 ) . add ( SrcReg1Sub1 ) . addReg ( CarryReg , RegState :: Kill ) ; BuildMI ( MBB , MII , DL , get ( TargetOpcode :: REG_SEQUENCE ) , FullDestReg ) . addReg ( DestSub0 ) . addImm ( <STR_LIT> ) . addReg ( DestSub1 ) . addImm ( <STR_LIT> ) ; MRI . replaceRegWith ( Dest . getReg ( )


<s> const DebugLoc & DL = Inst . getDebugLoc ( ) ; MachineOperand & Dest = Inst . getOperand ( <NUM_LIT> ) ; MachineOperand & Src = Inst . getOperand ( <NUM_LIT> ) ; const MCInstrDesc & InstDesc = get ( <STR_LIT> ) ; const TargetRegisterClass * SrcRC = Src . isReg ( ) ? MRI . getRegClass ( Src . getReg ( ) ) : & <STR_LIT> ; unsigned MidReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned ResultReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; const TargetRegisterClass * SrcSubRC = RI . getSubRegClass (


<s> MachineOperand & Dest = Inst . getOperand ( <NUM_LIT> ) ; uint32_t Imm = Inst . getOperand ( <NUM_LIT> ) . getImm ( ) ; uint32_t Offset = Imm & <NUM_LIT> ; uint32_t BitWidth = ( Imm & <NUM_LIT> ) >> <NUM_LIT> ; ( void ) Offset ; assert ( Inst . getOpcode ( ) == <STR_LIT> && BitWidth <= <NUM_LIT> && Offset == <NUM_LIT> && <STR_LIT> ) ; if ( BitWidth < <NUM_LIT> ) { unsigned MidRegLo = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned MidRegHi = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned ResultReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , MidRegLo ) . addReg ( Inst . getOperand ( <NUM_LIT> ) . getReg ( ) , <NUM_LIT> , <STR_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( BitWidth ) ; BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , MidRegHi ) . addImm ( <NUM_LIT> ) . addReg


<s> Op1 = & Src0 ; } BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , Interm ) . add ( * Op0 ) ; unsigned NewDest = MRI . createVirtualRegister ( DestRC ) ; MachineInstr & Xor = * BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , NewDest ) . addReg ( Interm ) . add ( * Op1 ) ; MRI . replaceRegWith ( Dest . getReg ( ) , NewDest ) ; Worklist . insert ( &


<s> MachineOperand & Src1 = Inst . getOperand ( <NUM_LIT> ) ; unsigned NewDest = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned Interm = MRI . createVirtualRegister ( & <STR_LIT> ) ; MachineInstr & Not = * BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , Interm ) . add ( Src1 ) ; MachineInstr & Op = * BuildMI ( MBB , MII , DL , get ( Opcode ) , NewDest ) . add ( Src0 ) . addReg ( Interm ) ; Worklist . insert ( &


<s> MachineOperand & Dest = Inst . getOperand ( <NUM_LIT> ) ; MachineOperand & Src0 = Inst . getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = Inst . getOperand ( <NUM_LIT> ) ; unsigned NewDest = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned Interm = MRI . createVirtualRegister ( & <STR_LIT> ) ; MachineInstr & Op = * BuildMI ( MBB , MII , DL , get ( Opcode ) , Interm ) . add ( Src0 ) . add ( Src1 ) ; MachineInstr & Not = * BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , NewDest ) . addReg ( Interm ) ; Worklist . insert ( & Op ) ; Worklist . insert ( & Not


<s> MachineOperand & Src0 = Inst . getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = Inst . getOperand ( <NUM_LIT> ) ; unsigned NewDest = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned Interm = MRI . createVirtualRegister ( & <STR_LIT> ) ; MachineInstr & Op = * BuildMI ( MBB , MII , DL , get ( Opcode ) , Interm ) . add ( Src0 ) . add ( Src1 ) ; MachineInstr & Not = * BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , NewDest ) . addReg ( Interm ) ; Worklist . insert ( & Op ) ; Worklist . insert ( &


<s> MachineFunction * MF = MBB . getParent ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineFrameInfo & FrameInfo = MF -> getFrameInfo ( ) ; const DebugLoc & DL = MBB . findDebugLoc ( MI ) ; unsigned Size = FrameInfo . getObjectSize ( FrameIndex ) ; unsigned Align = FrameInfo . getObjectAlignment ( FrameIndex ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOStore , Size , Align ) ; unsigned SpillSize = TRI -> getSpillSize ( * RC ) ; if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillSaveOpcode ( SpillSize ) ) ; if ( TargetRegisterInfo :: isVirtualRegister ( SrcReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( SrcReg , & <STR_LIT> ) ; } MachineInstrBuilder Spill = BuildMI ( MBB , MI , DL , OpDesc ) . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getFrameOffsetReg ( ) , RegState :: Implicit ) ; FrameInfo . setStackID ( FrameIndex , <STR_LIT>


<s> MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOStore , Size , Align ) ; unsigned SpillSize = TRI -> getSpillSize ( * RC ) ; if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillSaveOpcode ( SpillSize ) ) ; if ( TargetRegisterInfo :: isVirtualRegister ( SrcReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( SrcReg , & <STR_LIT> ) ; } MachineInstrBuilder Spill = BuildMI ( MBB , MI , DL , OpDesc ) . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI

-> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getFrameOffsetReg ( ) , RegState :: Implicit ) ;


<s> static <ISA_LIT>EncodingFamily subtargetEncodingFamily ( const GCNSubtarget & ST ) { switch ( ST . getGeneration


<s> } if ( Src2 -> isReg ( ) && Src2 -> getReg ( ) == Reg ) { bool Src0Inlined = false ; if ( Src0 -> isReg ( ) ) { MachineInstr * Def = MRI -> getUniqueVRegDef ( Src0 -> getReg ( ) ) ; if ( Def && Def -> isMoveImmediate ( ) && isInlineConstant ( Def -> getOperand ( <NUM_LIT> ) ) && MRI -> hasOneUse ( Src0 -> getReg ( ) ) ) { Src0 -> ChangeToImmediate ( Def -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ; Src0Inlined = true ; } else if ( ( Register :: isPhysicalRegister ( Src0 -> getReg ( ) ) && ( ST . getConstantBusLimit ( Opc ) <= <NUM_LIT> && RI . isSGPRClass ( RI . getPhysRegClass ( Src0 -> getReg ( ) ) ) ) ) || ( Register :: isVirtualRegister ( Src0 -> getReg ( ) ) && ( ST . getConstantBusLimit ( Opc ) <= <NUM_LIT> && RI . isSGPRClass ( MRI -> getRegClass ( Src0 -> getReg ( ) ) ) ) ) ) return false ; } if ( Src1 -> isReg ( ) && ! Src0Inlined ) { MachineInstr * Def = MRI -> getUniqueVRegDef ( Src1 -> getReg ( ) ) ; if ( Def && Def -> isMoveImmediate ( ) && isInlineConstant ( Def -> getOperand ( <NUM_LIT> ) ) && MRI -> hasOneUse ( Src1 -> getReg ( ) ) && commuteInstruction ( UseMI ) ) { Src0 -> ChangeToImmediate ( Def -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ; } else if ( ( Register :: isPhysicalRegister ( Src1 -> getReg ( ) ) && RI . isSGPRClass ( RI . getPhysRegClass ( Src1 -> getReg ( ) ) ) ) || ( Register :: isVirtualRegister ( Src1 -> getReg ( ) ) && RI . isSGPRClass ( MRI -> getRegClass ( Src1 -> getReg ( ) ) ) ) ) return false ; } unsigned NewOpc = IsFMA ? ( IsF32 ? <STR_LIT> : <STR_LIT> ) : ( IsF32 ? <STR_LIT> : <STR_LIT> ) ; if ( pseudoToMCOpcode ( NewOpc ) == - <NUM_LIT> ) return false ; const int64_t Imm = ImmOp -> getImm ( ) ; UseMI . RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: omod ) ) ; UseMI . RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: clamp ) ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) UseMI . untieRegOperand ( <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ) ; Src2 -> ChangeToImmediate ( Imm ) ; removeModOperands (


<s> auto E = UseMI ? UseMI -> getIterator ( ) : DefBB -> end ( ) ; for ( auto I = std :: next ( DefMI . getIterator ( ) ) ; I != E ; ++ I ) { if ( I -> isDebugInstr ( ) ) continue ; if ( NumScan ++ > MaxInstScan ) return true ; if ( I -> modifiesRegister ( <STR_LIT> ,


<s> return NeedsCFI ? <STR_LIT> : <STR_LIT> ; case <NUM_LIT> : return NeedsCFI ? <STR_LIT> : <STR_LIT> ; case <NUM_LIT> : return NeedsCFI ? <STR_LIT> : <STR_LIT> ; case <NUM_LIT> : return NeedsCFI ? <STR_LIT> : <STR_LIT> ; case <NUM_LIT> : return NeedsCFI ? <STR_LIT> : <STR_LIT> ; case <NUM_LIT> : return NeedsCFI ? <STR_LIT> : <STR_LIT> ; case <NUM_LIT> : return NeedsCFI ? <STR_LIT> :


<s> return NeedsCFI ? <STR_LIT> : <STR_LIT> ; case <NUM_LIT> : return NeedsCFI ? <STR_LIT> : <STR_LIT> ; case <NUM_LIT> : return NeedsCFI ? <STR_LIT> : <STR_LIT> ; case <NUM_LIT> : return NeedsCFI ? <STR_LIT> : <STR_LIT> ; case <NUM_LIT> : return NeedsCFI ? <STR_LIT> : <STR_LIT> ; case <NUM_LIT>


<s> storeRegToStackSlotImpl ( MBB , MI , SrcReg , isKill , FrameIndex , RC , TRI


<s> void <ISA_LIT>InstrInfo :: storeRegToStackSlot ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator MI , Register SrcReg , bool isKill ,

int FrameIndex , const TargetRegisterClass * RC , const TargetRegisterInfo * TRI ) const {


<s> void <ISA_LIT>InstrInfo :: storeRegToStackSlotCFI ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator MI , Register SrcReg , bool isKill , int FrameIndex ,

const TargetRegisterClass * RC , const TargetRegisterInfo * TRI ) const {


<s> storeRegToStackSlotImpl ( MBB , MI , SrcReg , isKill , FrameIndex , RC ,


<s> MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOStore , FrameInfo . getObjectSize ( FrameIndex ) , FrameInfo . getObjectAlign ( FrameIndex ) ) ; unsigned SpillSize = TRI -> getSpillSize ( * RC ) ; if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ; assert ( SrcReg != <STR_LIT> && <STR_LIT> ) ; assert ( SrcReg != <STR_LIT> && SrcReg != <STR_LIT> && SrcReg != <STR_LIT> && <STR_LIT> ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillSaveOpcode ( SpillSize , NeedsCFI ) ) ; if ( SrcReg . isVirtual ( ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( SrcReg , & <STR_LIT> ) ; } BuildMI ( MBB , MI , DL , OpDesc ) . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addMemOperand

( MMO ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ;


<s> int CommutedOpcode = commuteOpcode ( * MI ) ; if ( CommutedOpcode == - <NUM_LIT> ) return nullptr ; int Src0Idx = <STR_LIT> ( MI -> getOpcode ( ) , <STR_LIT> :: src0 ) ; MachineOperand & Src0 = MI -> getOperand ( Src0Idx ) ; if ( ! Src0 . isReg ( ) ) return nullptr ; int Src1Idx = <STR_LIT> ( MI -> getOpcode ( ) , <STR_LIT> :: src1 ) ; if ( ( OpIdx0 != static_cast < unsigned > ( Src0Idx ) || OpIdx1 != static_cast < unsigned > ( Src1Idx ) ) && ( OpIdx0 != static_cast < unsigned > ( Src1Idx ) || OpIdx1 != static_cast < unsigned > ( Src0Idx ) ) ) return nullptr ; MachineOperand & Src1 = MI -> getOperand


<s> const MCInstrDesc & InstDesc = get ( MI -> getOpcode ( ) ) ; const MCOperandInfo & OpInfo = InstDesc . OpInfo [ OpIdx ] ; const TargetRegisterClass * DefinedRC = OpInfo . RegClass != - <NUM_LIT> ? RI . getRegClass ( OpInfo . RegClass ) : nullptr ; if ( ! MO ) MO = & MI -> getOperand ( OpIdx ) ; if ( isVALU ( * MI ) && usesConstantBus ( MRI , * MO , DefinedRC -> getSize ( ) ) ) { unsigned SGPRUsed = MO -> isReg ( ) ? MO -> getReg ( ) : ( unsigned ) <STR_LIT> ; for ( unsigned i = <NUM_LIT> , e = MI -> getNumOperands ( ) ; i != e ; ++ i ) { if ( i == OpIdx ) continue ; const MachineOperand & Op = MI -> getOperand ( i ) ; if ( Op . isReg ( ) && Op . getReg ( ) != SGPRUsed && usesConstantBus ( MRI , Op , getOpSize ( * MI , i ) ) ) { return false ; } } } if ( MO -> isReg (


<s> moveSMRDToVALU ( Inst , MRI , Worklist ) ; continue ; } break ; case <STR_LIT> : splitScalar64BitBinaryOp ( Worklist , Inst , <STR_LIT> ) ; Inst -> eraseFromParent ( ) ; continue ; case <STR_LIT> : splitScalar64BitBinaryOp ( Worklist , Inst , <STR_LIT> ) ; Inst -> eraseFromParent ( ) ; continue ; case <STR_LIT> : splitScalar64BitBinaryOp ( Worklist , Inst , <STR_LIT> ) ; Inst -> eraseFromParent ( ) ; continue ; case <STR_LIT> : splitScalar64BitUnaryOp ( Worklist , Inst , <STR_LIT> ) ; Inst -> eraseFromParent ( ) ; continue ; case <STR_LIT> : splitScalar64BitBCNT ( Worklist , Inst ) ; Inst -> eraseFromParent ( ) ; continue ; case <STR_LIT> : { splitScalar64BitBFE ( Worklist , Inst ) ; Inst -> eraseFromParent ( ) ; continue ; } case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : case <STR_LIT> : llvm_unreachable ( <STR_LIT> ) ; } if ( NewOpcode == <STR_LIT> ) { legalizeOperands ( Inst ) ; continue ; } const MCInstrDesc & NewDesc = get ( NewOpcode ) ; Inst -> setDesc ( NewDesc ) ; for ( unsigned i = Inst -> getNumOperands ( ) - <NUM_LIT> ; i > <NUM_LIT> ; -- i ) { MachineOperand & Op = Inst -> getOperand ( i ) ; if ( Op . isReg ( ) && Op . getReg ( ) == <STR_LIT> ) Inst -> RemoveOperand ( i ) ; } if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { unsigned Size = ( Opcode == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; Inst -> addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ; Inst -> addOperand ( MachineOperand :: CreateImm ( Size ) ) ; } else if ( Opcode == <STR_LIT> ) { Inst -> addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ; } Inst -> addImplicitDefUseOperands ( * Inst -> getParent ( )


<s> break ; } } const MachineOperand * Dst = getNamedOperand ( MI , <STR_LIT> :: vdst ) ; const MachineOperand * Src0 = getNamedOperand ( MI , <STR_LIT> :: src0 ) ; const MachineOperand * Src0Mods = getNamedOperand ( MI , <STR_LIT> :: src0_modifiers ) ; const MachineOperand * Src1 = getNamedOperand ( MI , <STR_LIT> :: src1 ) ; const MachineOperand * Src1Mods = getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) ; const MachineOperand * Src2 = getNamedOperand ( MI , <STR_LIT> :: src2 ) ; const MachineOperand * Clamp = getNamedOperand ( MI , <STR_LIT> :: clamp ) ; const MachineOperand * Omod = getNamedOperand ( MI , <STR_LIT> :: omod ) ; MachineInstrBuilder MIB ; MachineBasicBlock & MBB = * MI . getParent ( ) ; if ( ! Src0Mods && ! Src1Mods && ! Clamp && ! Omod && ! IsF64 && ( ST . getConstantBusLimit ( Opc ) > <NUM_LIT> || ! Src0 -> isReg ( ) || ! RI . isSGPRReg ( MBB . getParent ( ) -> getRegInfo ( ) , Src0 -> getReg ( ) ) ) ) { int64_t Imm ; if ( getFoldableImm ( Src2 , Imm ) ) { unsigned NewOpc = IsFMA ? ( IsF16 ? <STR_LIT> : <STR_LIT> ) : ( IsF16 ? <STR_LIT> : <STR_LIT> ) ; if ( pseudoToMCOpcode ( NewOpc ) != - <NUM_LIT> ) { MIB = BuildMI ( MBB , MI , MI . getDebugLoc ( ) , get ( NewOpc ) ) . add ( * Dst ) . add ( * Src0 ) . add ( * Src1 ) . addImm ( Imm ) ; updateLiveVariables ( LV , MI , * MIB ) ; if ( LIS ) LIS -> ReplaceMachineInstrInMaps ( MI , * MIB ) ; return MIB ; } } unsigned NewOpc = IsFMA ? ( IsF16 ? <STR_LIT> : <STR_LIT> ) : ( IsF16 ? <STR_LIT> : <STR_LIT> ) ; if ( getFoldableImm ( Src1 , Imm ) ) { if ( pseudoToMCOpcode ( NewOpc ) != - <NUM_LIT> ) { MIB = BuildMI ( MBB , MI , MI . getDebugLoc ( ) , get ( NewOpc ) ) . add ( * Dst ) . add ( * Src0 ) . addImm ( Imm ) . add ( * Src2 ) ; updateLiveVariables ( LV , MI , * MIB ) ; if ( LIS ) LIS -> ReplaceMachineInstrInMaps ( MI , * MIB ) ; return MIB


<s> const MachineOperand * Src0 = & MI . getOperand ( Src0Idx ) ; if ( ! Src0 -> isReg ( ) && ! Src0 -> isImm ( ) ) return nullptr ; if ( Src0 -> isImm ( ) && ! isInlineConstant ( MI , Src0Idx , * Src0 ) ) return nullptr ; break ; } } const MachineOperand * Dst = getNamedOperand ( MI , <STR_LIT> :: vdst ) ; const MachineOperand * Src0 = getNamedOperand ( MI , <STR_LIT> :: src0 ) ; const MachineOperand * Src0Mods = getNamedOperand ( MI , <STR_LIT> :: src0_modifiers ) ; const MachineOperand * Src1 = getNamedOperand ( MI , <STR_LIT> :: src1 ) ; const MachineOperand * Src1Mods = getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) ; const MachineOperand * Src2 = getNamedOperand ( MI , <STR_LIT> :: src2 ) ; const MachineOperand * Clamp = getNamedOperand ( MI , <STR_LIT> :: clamp ) ; const MachineOperand * Omod = getNamedOperand ( MI , <STR_LIT> :: omod ) ; MachineInstrBuilder MIB ; MachineBasicBlock & MBB = * MI . getParent ( ) ; if ( ! Src0Mods && ! Src1Mods && ! Clamp && ! Omod && ! IsF64 && ( ST . getConstantBusLimit ( Opc ) > <NUM_LIT> || ! Src0 -> isReg ( ) || ! RI . isSGPRReg ( MBB . getParent ( ) -> getRegInfo ( ) , Src0 -> getReg ( ) ) ) ) { int64_t Imm ; if ( getFoldableImm ( Src2 , Imm ) ) { unsigned NewOpc = IsFMA ? ( IsF16 ? <STR_LIT> : <STR_LIT> ) : ( IsF16 ? <STR_LIT> : <STR_LIT> ) ; if ( pseudoToMCOpcode ( NewOpc ) != - <NUM_LIT> ) { MIB = BuildMI ( MBB , MI , MI . getDebugLoc ( ) , get ( NewOpc ) ) . add ( * Dst ) . add ( * Src0 ) . add ( * Src1 ) . addImm ( Imm ) ; updateLiveVariables ( LV , MI , * MIB ) ; if ( LIS ) LIS -> ReplaceMachineInstrInMaps ( MI , * MIB ) ; return MIB ; } } unsigned NewOpc = IsFMA ? ( IsF16 ? <STR_LIT> : <STR_LIT> ) : ( IsF16 ? <STR_LIT> : <STR_LIT> ) ; if ( getFoldableImm ( Src1 , Imm ) ) { if ( pseudoToMCOpcode ( NewOpc ) != - <NUM_LIT> ) { MIB = BuildMI ( MBB , MI , MI . getDebugLoc ( ) , get ( NewOpc ) ) . add ( * Dst ) . add ( * Src0 ) . addImm ( Imm ) . add ( * Src2 ) ; updateLiveVariables ( LV , MI , * MIB ) ; if ( LIS ) LIS -> ReplaceMachineInstrInMaps ( MI , * MIB ) ; return MIB ; } } if ( getFoldableImm ( Src0 , Imm


<s> MRI . constrainRegClass ( DestReg , & <STR_LIT> ) ; } if ( RI . spillSGPRToVGPR ( ) ) FrameInfo . setStackID ( FrameIndex , TargetStackID :: SGPRSpill ) ; BuildMI ( MBB , MI , DL , OpDesc , DestReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ; return ; } unsigned Opcode = RI . hasAGPRs ( RC ) ? getAGPRSpillRestoreOpcode ( SpillSize ) : getVGPRSpillRestoreOpcode ( SpillSize ) ; auto MIB = BuildMI ( MBB , MI , DL


<s> const MCInstrDesc & OpDesc = get ( getSGPRSpillSaveOpcode ( SpillSize ) ) ; if ( Register :: isVirtualRegister ( SrcReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( SrcReg , & <STR_LIT> ) ; } BuildMI ( MBB , MI , DL , OpDesc ) . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ; if ( RI . spillSGPRToVGPR ( ) ) FrameInfo . setStackID ( FrameIndex , TargetStackID :: SGPRSpill ) ; return ; } unsigned Opcode = RI . hasAGPRs ( RC ) ? getAGPRSpillSaveOpcode ( SpillSize ) : getVGPRSpillSaveOpcode ( SpillSize ) ; MFI -> setHasSpilledVGPRs ( )


<s> return MO . getReg ( ) == <STR_LIT> && MO . isImplicit ( ) && isVALU ( * MO .

getParent ( ) ) && ! readsExecAsData ( * MO . getParent ( ) ) ;


<s> assert ( SrcReg != <STR_LIT> && <STR_LIT> ) ; assert ( SrcReg != <STR_LIT> && SrcReg != <STR_LIT> && SrcReg != <STR_LIT> && <STR_LIT> ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillSaveOpcode ( SpillSize ) ) ; if ( SrcReg . isVirtual ( ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( SrcReg , & <STR_LIT> ) ; } BuildMI ( MBB , MI , DL , OpDesc ) . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ; if ( RI . spillSGPRToVGPR ( ) ) FrameInfo . setStackID ( FrameIndex , TargetStackID :: SGPRSpill ) ; return ; } unsigned Opcode = RI . isAGPRClass ( RC ) ? getAGPRSpillSaveOpcode ( SpillSize ) : getVGPRSpillSaveOpcode ( SpillSize ) ; MFI -> setHasSpilledVGPRs ( ) ; BuildMI ( MBB , MI , DL , get ( Opcode ) ) . addReg ( SrcReg , getKillRegState ( isKill ) ) .

addFrameIndex ( FrameIndex ) . addReg ( MFI -> getStackPtrOffsetReg ( ) ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ;


<s> unsigned STmp0 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; unsigned STmp1 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp0 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Z ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Y ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( STmp1 ) . addReg ( STmp0 ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp1 ) . addReg ( TIDIGXReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp0 ) . addReg ( TIDIGYReg ) . addReg ( TIDReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( TIDReg ) . addReg ( TIDIGZReg ) ; } else


<s> if ( BaseReg1 == BaseReg2 ) return true ; if ( ! MI1 . hasOneMemOperand ( ) || ! MI2 . hasOneMemOperand ( ) ) return false ; auto MO1 = * MI1 . memoperands_begin ( ) ; auto MO2 = * MI2 . memoperands_begin ( ) ; if ( MO1 -> getAddrSpace ( ) != MO2 -> getAddrSpace ( ) ) return false ; auto Base1 = MO1 -> getValue ( ) ; auto Base2 = MO2 -> getValue ( ) ; if ( ! Base1 || ! Base2 ) return false ; const MachineFunction & MF = * MI1 . getParent ( ) -> getParent ( ) ; const DataLayout & DL = MF . getFunction ( )


<s> auto MO2 = * MI2 . memoperands_begin ( ) ; if ( MO1 -> getAddrSpace ( ) != MO2 -> getAddrSpace ( ) ) return false ; auto Base1 = MO1 -> getValue ( ) ; auto Base2 = MO2 -> getValue ( ) ; if ( ! Base1 || ! Base2 ) return false ; const MachineFunction & MF = * MI1 . getParent ( ) -> getParent ( ) ; const DataLayout & DL = MF . getFunction ( ) -> getParent ( ) -> getDataLayout ( ) ; Base1 = GetUnderlyingObject ( Base1 , DL ) ; Base2 = GetUnderlyingObject ( Base1 , DL


<s> DebugLoc DL = MBB . findDebugLoc ( MI ) ; assert ( SrcReg != MFI -> getStackPtrOffsetReg ( ) && SrcReg != MFI -> getFrameOffsetReg ( ) && SrcReg != MFI -> getScratchWaveOffsetReg ( ) ) ; unsigned Size = FrameInfo . getObjectSize ( FrameIndex ) ; unsigned Align = FrameInfo . getObjectAlignment ( FrameIndex ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOStore , Size , Align ) ; unsigned SpillSize = TRI -> getSpillSize ( * RC ) ; if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillSaveOpcode ( SpillSize ) ) ; if ( TargetRegisterInfo :: isVirtualRegister ( SrcReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( SrcReg , & <STR_LIT> ) ; } MachineInstrBuilder Spill = BuildMI ( MBB , MI , DL , OpDesc ) . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getFrameOffsetReg ( ) , RegState :: Implicit ) ; FrameInfo . setStackID ( FrameIndex , <NUM_LIT> ) ; if ( ST . hasScalarStores ( ) ) { Spill . addReg ( <STR_LIT> , RegState :: ImplicitDefine | RegState :: Dead ) ; } return ; } if ( ! ST . isVGPRSpillingEnabled ( * MF -> getFunction ( ) ) ) { LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ; BuildMI ( MBB , MI , DL , get (


<s> assert ( <STR_LIT> . contains ( SrcReg ) ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) . addImm ( <NUM_LIT> ) ; return ; } unsigned EltSize = <NUM_LIT> ; unsigned Opcode = <STR_LIT> ; if ( RI . isSGPRClass ( RC ) ) { if ( RI . getRegSizeInBits ( * RC ) > <NUM_LIT> ) { Opcode = <STR_LIT> ; EltSize = <NUM_LIT> ; } else { Opcode = <STR_LIT> ; EltSize = <NUM_LIT> ; } if ( ! RI . isSGPRClass ( RI . getPhysRegClass ( SrcReg ) ) ) { reportIllegalCopy ( this , MBB , MI , DL , DestReg , SrcReg , KillSrc ) ; return ; } } ArrayRef < int16_t > SubIndices = RI . getRegSplitParts ( RC , EltSize ) ; bool Forward = RI . getHWRegIndex ( DestReg ) <= RI . getHWRegIndex ( SrcReg ) ; for ( unsigned Idx = <NUM_LIT> ; Idx < SubIndices . size ( ) ; ++ Idx ) { unsigned SubIdx ; if ( Forward ) SubIdx = SubIndices [ Idx ] ; else SubIdx = SubIndices [ SubIndices . size ( ) - Idx - <NUM_LIT> ] ; MachineInstrBuilder Builder = BuildMI ( MBB , MI , DL , get ( Opcode ) , RI .


<s> void <ISA_LIT>InstrInfo :: storeRegToStackSlot ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator MI , unsigned SrcReg , bool isKill , int FrameIndex , const TargetRegisterClass * RC , const TargetRegisterInfo * TRI ) const { MachineFunction * MF = MBB . getParent ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineFrameInfo & FrameInfo = MF -> getFrameInfo ( ) ; DebugLoc DL = MBB . findDebugLoc ( MI ) ; unsigned Size = FrameInfo . getObjectSize ( FrameIndex ) ; unsigned Align = FrameInfo . getObjectAlignment ( FrameIndex ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOStore , Size , Align ) ; unsigned SpillSize = TRI -> getSpillSize ( * RC ) ; if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillSaveOpcode ( SpillSize ) ) ; if ( TargetRegisterInfo :: isVirtualRegister ( SrcReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( SrcReg , & <STR_LIT> ) ; } MachineInstrBuilder Spill = BuildMI ( MBB , MI , DL , OpDesc ) . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getScratchWaveOffsetReg ( ) , RegState :: Implicit ) ; if ( ST . hasScalarStores ( ) ) { Spill . addReg ( <STR_LIT> , RegState :: ImplicitDefine ) ; } return ; } if ( ! ST . isVGPRSpillingEnabled ( * MF -> getFunction ( ) ) ) { LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ) . addReg ( SrcReg ) ; return ; } assert ( RI . hasVGPRs ( RC ) &&


<s> int Src0Idx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: src0 ) ; const MachineOperand * Src0 = & MI . getOperand ( Src0Idx ) ; if ( Src0 -> isImm ( ) && ! isInlineConstant ( MI , Src0Idx , * Src0 ) ) return nullptr ; break ; } } const MachineOperand * Dst = getNamedOperand ( MI , <STR_LIT> :: vdst ) ; const MachineOperand * Src0 = getNamedOperand ( MI , <STR_LIT> :: src0 ) ; const MachineOperand * Src1 = getNamedOperand ( MI , <STR_LIT> :: src1 ) ; const MachineOperand * Src2 = getNamedOperand ( MI , <STR_LIT> :: src2 ) ; return BuildMI ( * MBB , MI , MI . getDebugLoc ( ) , get ( IsF16 ? <STR_LIT> : <STR_LIT> ) ) . addOperand ( * Dst ) . addImm ( <NUM_LIT> ) . addOperand ( * Src0 ) . addImm ( <NUM_LIT> )

. addOperand ( * Src1 ) . addImm ( <NUM_LIT> ) . addOperand ( * Src2 ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ;


<s> } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const MCInstrDesc & MovRelDesc = get ( <STR_LIT> ) ; unsigned VecReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; bool IsUndef = MI . getOperand ( <NUM_LIT> ) . isUndef ( ) ; unsigned SubReg = <STR_LIT> + MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; assert ( VecReg == MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MachineInstr * MovRel = BuildMI ( MBB , MI , DL , MovRelDesc ) . addReg ( RI . getSubReg ( VecReg , SubReg ) , RegState :: Undef ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) . addReg ( VecReg , RegState :: ImplicitDefine ) . addReg ( VecReg , RegState :: Implicit | ( IsUndef ? RegState :: Undef : <NUM_LIT> ) ) ; const int ImpDefIdx = MovRelDesc . getNumOperands ( ) + MovRelDesc . getNumImplicitUses ( ) ; const int ImpUseIdx = ImpDefIdx + <NUM_LIT> ; MovRel -> tieOperands ( ImpDefIdx , ImpUseIdx ) ; MI . eraseFromParent ( ) ; break ; } case <STR_LIT> : { MachineFunction & MF = * MBB . getParent ( ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned RegLo = RI . getSubReg ( Reg , <STR_LIT> ) ; unsigned RegHi = RI . getSubReg ( Reg , <STR_LIT> ) ; MIBundleBuilder Bundler ( MBB , MI ) ; Bundler . append ( BuildMI ( MF , DL , get ( <STR_LIT> ) , Reg ) ) ; Bundler . append ( BuildMI ( MF , DL , get ( <STR_LIT> ) , RegLo ) . addReg ( RegLo ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ) ; MachineInstrBuilder MIB = BuildMI ( MF , DL , get ( <STR_LIT> ) , RegHi ) . addReg ( RegHi ) ; if ( MI . getOperand ( <NUM_LIT> ) . getTargetFlags ( ) == <ISA_LIT>InstrInfo :: MO_NONE ) MIB


<s> unsigned <ISA_LIT>InstrInfo :: insertBranch ( MachineBasicBlock & MBB , MachineBasicBlock * TBB , MachineBasicBlock * FBB , ArrayRef < MachineOperand > Cond , const DebugLoc & DL , int * BytesAdded ) const { if ( ! FBB && Cond . empty ( ) ) { BuildMI ( & MBB , DL , get ( <STR_LIT> ) ) . addMBB ( TBB ) ; if ( BytesAdded ) * BytesAdded = <NUM_LIT> ; return <NUM_LIT> ; } assert ( TBB && Cond [ <NUM_LIT> ] . isImm ( ) ) ; unsigned Opcode = getBranchOpcode ( static_cast < BranchPredicate > ( Cond [ <NUM_LIT> ] . getImm ( ) ) ) ; if ( ! FBB ) { Cond [ <NUM_LIT> ] . isUndef ( ) ; MachineInstr * CondBr = BuildMI ( & MBB , DL , get


<s> void <ISA_LIT>InstrInfo :: legalizeGenericOperand ( MachineBasicBlock & InsertMBB , MachineBasicBlock :: iterator I , const TargetRegisterClass * DstRC , MachineOperand & Op , MachineRegisterInfo & MRI , const DebugLoc & DL ) const { unsigned OpReg = Op . getReg ( ) ; unsigned OpSubReg = Op . getSubReg ( ) ; const TargetRegisterClass * OpRC = RI . getSubClassWithSubReg ( RI . getRegClassForReg ( MRI , OpReg ) , OpSubReg ) ; if ( DstRC == OpRC ) return ; unsigned DstReg = MRI . createVirtualRegister ( DstRC ) ; MachineInstr * Copy = BuildMI ( InsertMBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addOperand ( Op ) ; Op . setReg ( DstReg ) ; Op . setSubReg ( <NUM_LIT> ) ; MachineInstr * Def = MRI . getVRegDef ( OpReg ) ; if ( ! Def ) return ; if ( Def -> isMoveImmediate ( ) ) FoldImmediate ( * Copy ,


<s> FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> :: vdata ) ; SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: vdata ) ; } else if ( isSMRD ( FirstLdSt ) && isSMRD ( SecondLdSt ) ) { FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> :: sdst ) ; SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: sdst ) ; } else if ( isDS ( FirstLdSt ) && isDS


<s> if ( isMUBUF ( LdSt ) || isMTBUF ( LdSt ) ) { const MachineOperand * SOffset = getNamedOperand ( LdSt , <STR_LIT> :: soffset ) ; if ( SOffset && SOffset -> isReg ( ) ) { const MachineOperand * AddrReg = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ; if ( AddrReg && ! AddrReg -> isFI ( ) ) return false ; const MachineOperand * RSrc = getNamedOperand ( LdSt , <STR_LIT> :: srsrc ) ; const <ISA_LIT>MachineFunctionInfo * MFI = LdSt . getParent ( ) -> getParent ( ) -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( RSrc -> getReg ( ) != MFI -> getScratchRSrcReg ( ) ) return false ; const MachineOperand * OffsetImm = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; BaseOps . push_back ( RSrc ) ; BaseOps . push_back ( SOffset ) ; Offset = OffsetImm -> getImm ( ) ; MOp = getNamedOperand ( LdSt , <STR_LIT> :: vdst ) ; if ( ! MOp ) MOp = getNamedOperand ( LdSt , <STR_LIT> :: vdata ) ; Width = getOperandSizeInBytes ( LdSt , MOp ) ; return true ; } BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: srsrc ) ; if ( ! BaseOp ) return false ; BaseOps . push_back ( BaseOp ) ; BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ; if ( BaseOp ) BaseOps . push_back ( BaseOp ) ; const MachineOperand * OffsetImm = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; Offset = OffsetImm -> getImm ( ) ; if ( SOffset ) Offset += SOffset -> getImm ( ) ; MOp = getNamedOperand ( LdSt , <STR_LIT> :: vdst ) ; if ( ! MOp ) MOp = getNamedOperand ( LdSt , <STR_LIT> :: vdata ) ; Width = getOperandSizeInBytes ( LdSt , MOp ) ; return true ; } if ( isSMRD ( LdSt ) ) { BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: sbase


<s> unsigned <ISA_LIT>InstrInfo :: getOperandSizeInBytes ( const MachineInstr & LdSt , const MachineOperand * MOp ) const { assert ( MOp && <STR_LIT> ) ; const MachineRegisterInfo & MRI = LdSt . getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; const Register Reg = MOp -> getReg ( ) ; const TargetRegisterClass * DstRC = Register :: isVirtualRegister ( Reg ) ? MRI . getRegClass ( Reg ) : RI . getPhysRegClass ( Reg ) ; return ( RI . getRegSizeInBits ( *


<s> bool isLDSDIR ( uint16_t Opcode ) const { return get ( Opcode ) . TSFlags


<s> bool <ISA_LIT>InstrInfo :: isSchedulingBoundary ( const MachineInstr & MI , const MachineBasicBlock * MBB , const MachineFunction & MF ) const { if ( MI . isTerminator ( ) || MI . isPosition ( ) ) return true ; return MI . modifiesRegister ( <STR_LIT> , & RI ) || MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || changesVGPRIndexingMode


<s> bool <ISA_LIT>InstrInfo :: isSchedulingBoundary ( const MachineInstr & MI , const MachineBasicBlock * MBB , const MachineFunction & MF ) const { if ( MI . isTerminator ( ) || MI .


<s> if ( ! memOpsHaveSameBasePtr ( FirstLdSt , BaseOps1 , SecondLdSt , BaseOps2 ) ) return false ; const MachineOperand * FirstDst = nullptr ; const MachineOperand * SecondDst = nullptr ; if ( ( isMUBUF ( FirstLdSt ) && isMUBUF ( SecondLdSt ) ) || ( isMTBUF ( FirstLdSt ) && isMTBUF ( SecondLdSt ) ) || ( isFLAT ( FirstLdSt ) && isFLAT ( SecondLdSt ) ) ) { const unsigned MaxGlobalLoadCluster = <NUM_LIT> ; if ( NumLoads > MaxGlobalLoadCluster ) return false ; FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> :: vdata ) ; if ( ! FirstDst ) FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> :: vdst ) ; SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: vdata ) ; if ( ! SecondDst ) SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: vdst ) ; } else if ( isSMRD ( FirstLdSt ) && isSMRD ( SecondLdSt ) ) { FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> ::


<s> int RSrcIdx = <STR_LIT> ( Opc , <STR_LIT> :: srsrc ) ; return <NUM_LIT> + <NUM_LIT> * ( ( RSrcIdx - VAddr0Idx + <NUM_LIT> ) / <NUM_LIT> ) ; } switch ( Opc ) { case TargetOpcode :: BUNDLE : return getInstBundleSize ( MI ) ; case TargetOpcode :: INLINEASM : case TargetOpcode :: INLINEASM_BR : { const MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; const char * AsmStr = MI . getOperand ( <NUM_LIT> ) . getSymbolName ( ) ; return getInlineAsmLength ( AsmStr , * MF -> getTarget ( ) . getMCAsmInfo


<s> const MachineOperand * Src0Mods = getNamedOperand ( MI , <STR_LIT> :: src0_modifiers ) ; const MachineOperand * Src1 = getNamedOperand ( MI , <STR_LIT> :: src1 ) ; const MachineOperand * Src1Mods = getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) ; const MachineOperand * Src2 = getNamedOperand ( MI , <STR_LIT> :: src2 ) ; const MachineOperand * Clamp = getNamedOperand ( MI , <STR_LIT> :: clamp ) ; const MachineOperand * Omod = getNamedOperand ( MI , <STR_LIT> :: omod ) ; MachineInstrBuilder MIB ; if ( ! Src0Mods && ! Src1Mods && ! Clamp && ! Omod && ! IsF64 && ( ST . getConstantBusLimit ( Opc ) > <NUM_LIT> || ! Src0 -> isReg ( ) || ! RI . isSGPRReg ( MBB -> getParent ( ) -> getRegInfo ( ) , Src0 -> getReg ( ) ) ) ) { int64_t Imm ; if ( getFoldableImm ( Src2 , Imm ) ) { unsigned NewOpc = IsFMA ? ( IsF16 ? <STR_LIT> : <STR_LIT> ) : ( IsF16 ? <STR_LIT> : <STR_LIT> ) ; if ( pseudoToMCOpcode ( NewOpc ) != - <NUM_LIT> ) { MIB = BuildMI ( * MBB , MI , MI . getDebugLoc ( ) , get ( NewOpc ) ) . add ( * Dst ) . add ( * Src0 ) . add ( * Src1 ) . addImm ( Imm ) ; updateLiveVariables ( LV , MI , * MIB ) ; return MIB ; } } unsigned NewOpc = IsFMA ? ( IsF16 ? <STR_LIT> : <STR_LIT> ) : ( IsF16 ? <STR_LIT> : <STR_LIT> ) ; if ( getFoldableImm ( Src1 , Imm ) ) { if ( pseudoToMCOpcode ( NewOpc ) != - <NUM_LIT> ) { MIB = BuildMI ( * MBB , MI , MI . getDebugLoc ( ) , get ( NewOpc ) ) . add ( * Dst ) . add ( * Src0 ) . addImm ( Imm ) . add ( * Src2 ) ; updateLiveVariables ( LV , MI , * MIB ) ; return MIB ; } } if ( getFoldableImm ( Src0 , Imm ) ) { if ( pseudoToMCOpcode ( NewOpc ) != - <NUM_LIT> && isOperandLegal ( MI , <STR_LIT> ( NewOpc ,


<s> if ( CmpValue != ExpectedValue ) { if ( ! IsReversable ) return false ; IsReversedCC = CmpValue == ( ExpectedValue ^ Mask ) ; if ( ! IsReversedCC ) return false ; } Register DefReg = Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( IsReversedCC && ! MRI -> hasOneNonDBGUse ( DefReg ) ) return false ; for ( auto I = std :: next ( Def -> getIterator ( ) ) , E = CmpInstr . getIterator ( ) ; I != E ; ++ I ) { if ( I -> modifiesRegister ( <STR_LIT> , & RI ) || I -> killsRegister ( <STR_LIT> , & RI ) ) return false ; } MachineOperand * SccDef = Def -> findRegisterDefOperand ( <STR_LIT> ) ; SccDef -> setIsDead ( false ) ; CmpInstr . eraseFromParent ( ) ; if ( ! MRI -> use_nodbg_empty ( DefReg ) ) { assert ( ! IsReversedCC ) ; return true ; } MachineBasicBlock * MBB = Def -> getParent ( ) ; unsigned NewOpc = ( SrcSize == <NUM_LIT> ) ? IsReversedCC ? <STR_LIT> : <STR_LIT> : IsReversedCC ? <STR_LIT> : <STR_LIT> ; BuildMI ( * MBB , Def , Def -> getDebugLoc ( ) , get ( NewOpc ) ) . add ( * SrcOp ) . addImm ( Log2_64 ( Mask ) ) ; Def -> eraseFromParent ( ) ; return true ; } ; switch ( CmpInstr . getOpcode ( ) ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return optimizeCmpAnd ( <NUM_LIT> , <NUM_LIT> , true , false ) ; case <STR_LIT> : case <STR_LIT> : return optimizeCmpAnd ( <NUM_LIT> , <NUM_LIT> , false , false ) ; case <STR_LIT> : case <STR_LIT> : return optimizeCmpAnd ( <NUM_LIT> , <NUM_LIT> , false , true ) ; case <STR_LIT>


<s> Register Undef = MRI . createVirtualRegister ( IsAGPR ? & <STR_LIT> : & <STR_LIT> ) ; BuildMI ( * BB , MI , DL , get ( <STR_LIT> ) , Undef ) ; Register NewVR = MRI . createVirtualRegister ( IsAGPR ? & <STR_LIT> : & <STR_LIT> ) ; BuildMI ( * BB , MI , DL , get ( <STR_LIT> ) , NewVR ) . addReg ( DataReg , <NUM_LIT> , Op . getSubReg ( ) ) . addImm ( <STR_LIT> ) . addReg ( Undef ) . addImm ( <STR_LIT> ) ; Op . setReg ( NewVR )


<s> int NumUse = <NUM_LIT> ; for ( auto & Use : MRI . use_nodbg_operands ( VReg ) ) { auto & UseInst = * Use . getParent ( ) ; if ( UseInst . getParent ( ) != DefBB || UseInst . isPHI ( ) ) return true ; if ( ++ NumUse > MaxUseScan ) return true ; } if ( NumUse == <NUM_LIT> ) return false ; const int MaxInstScan = <NUM_LIT> ; int NumInst = <NUM_LIT> ; for ( auto I = std :: next ( DefMI . getIterator ( ) ) ;


<s> assert ( UseMI . getOperand ( <NUM_LIT> ) . getReg ( ) . isVirtual ( ) ) ; } UseMI . setDesc ( get ( NewOpc ) ) ; UseMI . getOperand ( <NUM_LIT> ) . ChangeToImmediate ( Imm . getSExtValue ( ) ) ; UseMI . addImplicitDefUseOperands ( * UseMI . getParent ( ) -> getParent ( ) ) ; return true ; } if ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) { if ( hasAnyModifiersSet ( UseMI ) ) return false ; MachineOperand * Src0 = getNamedOperand ( UseMI , <STR_LIT> :: src0 ) ; if ( isInlineConstant ( UseMI , * Src0 , * ImmOp ) ) return false ; bool IsF32 = Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ; bool IsFMA = Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ; MachineOperand * Src1 = getNamedOperand ( UseMI , <STR_LIT> :: src1 ) ; MachineOperand * Src2 = getNamedOperand ( UseMI , <STR_LIT> :: src2 ) ; if ( Src0 -> isReg ( ) && Src0 -> getReg ( ) == Reg ) { if ( ! Src1 -> isReg ( ) || RI . isSGPRClass ( MRI -> getRegClass ( Src1 -> getReg ( ) ) ) ) return false ; if ( ! Src2 -> isReg ( ) || RI . isSGPRClass ( MRI -> getRegClass ( Src2 -> getReg ( ) ) ) ) return false ; unsigned NewOpc = IsFMA ? ( IsF32 ? <STR_LIT> : <STR_LIT> ) : ( IsF32 ? <STR_LIT> : <STR_LIT> ) ; if ( pseudoToMCOpcode ( NewOpc ) == - <NUM_LIT> ) return false ; const int64_t Imm = ImmOp -> getImm ( ) ; Register Src1Reg = Src1 -> getReg ( ) ; unsigned Src1SubReg = Src1 -> getSubReg ( ) ; Src0 -> setReg ( Src1Reg ) ; Src0 -> setSubReg ( Src1SubReg ) ; Src0 -> setIsKill ( Src1 -> isKill ( ) ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) UseMI . untieRegOperand ( <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ) ; Src1 -> ChangeToImmediate ( Imm ) ; removeModOperands ( UseMI ) ; UseMI . setDesc ( get ( NewOpc


<s> if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX10 ) { int64_t Format = ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX11 ? <STR_LIT> :: UFMT_32_FLOAT : <STR_LIT> :: UFMT_32_FLOAT ; return ( Format << <NUM_LIT> ) | ( <NUM_LIT> << <NUM_LIT> ) | ( <NUM_LIT> << <NUM_LIT> ) ; } uint64_t RsrcDataFormat = <STR_LIT> ; if ( ST . isAmdHsaOS ( ) ) { if ( ST . getGeneration ( ) <= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) RsrcDataFormat |= ( <NUM_LIT> << <NUM_LIT> ) ; if ( ST . getGeneration ( ) == <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) RsrcDataFormat


<s> int Data0Idx = <STR_LIT> ( Opc , <STR_LIT> :: data0 ) ; EltSize = TRI -> getRegSizeInBits ( * getOpRegClass ( LdSt , Data0Idx ) ) / <NUM_LIT> ; } if ( isStride64 ( Opc ) ) EltSize *= <NUM_LIT> ; BaseOps . push_back ( BaseOp ) ; Offset = EltSize * Offset0 ; DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: vdst ) ; if ( DataOpIdx == - <NUM_LIT> ) { DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: data0 ) ; Width = getOpSize ( LdSt , DataOpIdx ) ; DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: data1 ) ; Width += getOpSize ( LdSt , DataOpIdx ) ; } else { Width = getOpSize ( LdSt , DataOpIdx ) ; } } return true ; } if ( isMUBUF ( LdSt ) || isMTBUF ( LdSt ) ) { const MachineOperand * RSrc = getNamedOperand ( LdSt , <STR_LIT> :: srsrc ) ; if ( ! RSrc ) return false ; BaseOps . push_back ( RSrc ) ; BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ; if ( BaseOp && ! BaseOp -> isFI ( ) ) BaseOps . push_back ( BaseOp ) ; const MachineOperand * OffsetImm = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; Offset = OffsetImm -> getImm ( ) ; const MachineOperand * SOffset = getNamedOperand ( LdSt , <STR_LIT> :: soffset ) ; if ( SOffset ) { if ( SOffset -> isReg ( ) ) BaseOps . push_back ( SOffset ) ; else Offset += SOffset -> getImm ( ) ; } DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: vdst ) ; if ( DataOpIdx == - <NUM_LIT> ) DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: vdata ) ; if ( DataOpIdx == - <NUM_LIT> ) return false ; Width = getOpSize ( LdSt , DataOpIdx ) ; return true ; } if ( isMIMG ( LdSt ) ) { int SRsrcIdx = <STR_LIT> ( Opc , <STR_LIT> :: srsrc ) ; BaseOps . push_back ( & LdSt . getOperand (


<s> Width = getOpSize ( LdSt , DataOpIdx ) ; } } return true ; } if ( isMUBUF ( LdSt ) || isMTBUF ( LdSt ) ) { const MachineOperand * RSrc = getNamedOperand ( LdSt , <STR_LIT> :: srsrc ) ; if ( ! RSrc ) return false ; BaseOps . push_back ( RSrc ) ; BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ; if ( BaseOp && ! BaseOp -> isFI ( ) ) BaseOps . push_back ( BaseOp ) ; const MachineOperand * OffsetImm = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; Offset = OffsetImm -> getImm ( ) ; const MachineOperand * SOffset = getNamedOperand ( LdSt , <STR_LIT> :: soffset ) ; if ( SOffset ) { if ( SOffset -> isReg ( ) ) BaseOps . push_back ( SOffset ) ; else Offset += SOffset -> getImm ( ) ; } DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: vdst ) ; if ( DataOpIdx == - <NUM_LIT> ) DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: vdata ) ; if ( DataOpIdx == - <NUM_LIT> ) return false ; Width = getOpSize ( LdSt , DataOpIdx ) ; return true ; } if ( isMIMG ( LdSt ) ) { int SRsrcIdx = <STR_LIT> ( Opc , <STR_LIT> :: srsrc ) ; BaseOps . push_back ( & LdSt . getOperand ( SRsrcIdx ) ) ; int VAddr0Idx = <STR_LIT> ( Opc , <STR_LIT> :: vaddr0 ) ; if ( VAddr0Idx >= <NUM_LIT> ) { for ( int I = VAddr0Idx ; I < SRsrcIdx ; ++ I ) BaseOps . push_back ( & LdSt . getOperand ( I ) ) ; } else { BaseOps . push_back ( getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ) ; } Offset = <NUM_LIT> ; DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: vdata ) ; Width = getOpSize ( LdSt , DataOpIdx ) ; return true ; } if ( isSMRD ( LdSt ) ) { BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: sbase ) ; if ( ! BaseOp ) return false ; BaseOps . push_back ( BaseOp ) ; OffsetOp = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; Offset = OffsetOp ? OffsetOp -> getImm ( ) : <NUM_LIT> ; DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: sdst ) ; Width = getOpSize ( LdSt , DataOpIdx ) ; return true ; } if ( isFLAT ( LdSt ) ) { BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ; if ( BaseOp ) BaseOps . push_back ( BaseOp ) ; BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: saddr ) ; if ( BaseOp ) BaseOps . push_back ( BaseOp


<s> switch ( MI . getOpcode ( ) ) { default : if ( MI . isMetaInstruction ( ) ) return <NUM_LIT> ; return <NUM_LIT> ; case <STR_LIT> : return MI . getOperand ( <NUM_LIT> ) . getImm ( ) + <NUM_LIT> ; case <STR_LIT>


<s> return any_of ( ModifierOpNames , [ & ] ( unsigned Name )

{ return hasModifiersSet ( MI , Name ) ; } ) ;


<s> MachineOperand * SBase = getNamedOperand ( MI , <STR_LIT> :: sbase ) ; if ( SBase && ! RI . isSGPRClass ( MRI . getRegClass ( SBase -> getReg ( ) ) ) ) { Register SGPR = readlaneVGPRToSGPR ( SBase -> getReg ( ) , MI , MRI ) ; SBase -> setReg ( SGPR ) ; } MachineOperand * SOff = getNamedOperand ( MI , <STR_LIT> :: soffset ) ; if ( SOff && ! RI . isSGPRClass ( MRI . getRegClass ( SOff -> getReg (


<s> for ( unsigned Name : reverse ( ModifierOpNames ) ) MI . removeOperand ( <STR_LIT>


<s> default : return nullptr ; case <STR_LIT> : case <STR_LIT> : IsF16 = true ; LLVM_FALLTHROUGH ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : break ; case <STR_LIT> : case <STR_LIT> : IsF16 = true ; LLVM_FALLTHROUGH ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { int Src0Idx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: src0 ) ; const MachineOperand * Src0 = & MI . getOperand ( Src0Idx ) ; if ( ! Src0 -> isReg ( ) && ! Src0 -> isImm ( ) ) return nullptr ; if ( Src0 -> isImm ( ) && ! isInlineConstant ( MI , Src0Idx , * Src0 ) ) return nullptr ; break ; } } const MachineOperand * Dst = getNamedOperand ( MI , <STR_LIT> :: vdst ) ; const MachineOperand * Src0 = getNamedOperand ( MI , <STR_LIT> :: src0 ) ; const MachineOperand * Src0Mods = getNamedOperand ( MI , <STR_LIT> :: src0_modifiers ) ; const MachineOperand * Src1 = getNamedOperand ( MI , <STR_LIT> :: src1 ) ; const MachineOperand * Src1Mods = getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) ; const MachineOperand * Src2 = getNamedOperand ( MI , <STR_LIT> :: src2 ) ; const MachineOperand * Clamp = getNamedOperand ( MI , <STR_LIT> :: clamp ) ; const MachineOperand * Omod = getNamedOperand ( MI , <STR_LIT> :: omod ) ; MachineInstrBuilder MIB ; if ( ! Src0Mods && ! Src1Mods && ! Clamp && ! Omod && ! IsF64 && ( ST . getConstantBusLimit ( Opc ) > <NUM_LIT> || ! Src0 -> isReg ( ) || ! RI . isSGPRReg ( MBB -> getParent ( ) -> getRegInfo ( ) , Src0 -> getReg ( ) ) ) ) { if ( auto Imm = getFoldableImm ( Src2 ) ) { unsigned NewOpc = IsFMA ? ( IsF16 ? <STR_LIT> : <STR_LIT> ) : ( IsF16 ? <STR_LIT> : <STR_LIT> ) ; if ( pseudoToMCOpcode ( NewOpc ) != - <NUM_LIT> ) { MIB = BuildMI ( * MBB , MI , MI . getDebugLoc ( ) , get ( NewOpc ) ) . add ( * Dst ) . add ( * Src0 ) . add ( * Src1 ) . addImm ( Imm ) ; updateLiveVariables ( LV , MI , * MIB ) ; return MIB ; } } unsigned NewOpc = IsFMA ? ( IsF16 ? <STR_LIT> : <STR_LIT> ) : ( IsF16 ? <STR_LIT> : <STR_LIT> ) ; if ( auto Imm = getFoldableImm ( Src1 ) ) { if ( pseudoToMCOpcode ( NewOpc ) != - <NUM_LIT> ) { MIB = BuildMI ( * MBB , MI , MI . getDebugLoc ( ) , get ( NewOpc ) ) . add ( * Dst ) . add ( * Src0 ) . addImm ( Imm ) . add


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { unsigned NumOps = MI . getDesc ( ) . getNumOperands ( ) + MI . getDesc ( ) . getNumImplicitUses ( ) ; return MI . getNumOperands ( ) == NumOps ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case