<s> case MCOI :: OPERAND_REGISTER : if ( MI . getOperand ( i ) . isImm ( ) ) { ErrInfo = <STR_LIT> ; return false ; } break ; case <STR_LIT> : break ; case <STR_LIT> : if ( isLiteralConstant ( MI . getOperand ( i ) , RI . getRegClass ( RegClass ) -> getSize ( ) ) ) { ErrInfo = <STR_LIT> ; return false ; } break ; case MCOI :: OPERAND_IMMEDIATE : case <STR_LIT> : if ( ! MI . getOperand ( i ) . isImm ( ) && ! MI . getOperand ( i ) . isFI ( ) ) { ErrInfo = <STR_LIT> ; return false ; } default : continue ; } if ( ! MI . getOperand ( i ) . isReg ( ) ) continue ; if ( RegClass != - <NUM_LIT> ) { unsigned Reg = MI . getOperand ( i ) . getReg ( ) ; if ( Reg == <STR_LIT> || TargetRegisterInfo :: isVirtualRegister ( Reg ) ) continue ; const TargetRegisterClass * RC = RI . getRegClass ( RegClass ) ; if ( ! RC -> contains ( Reg ) ) { ErrInfo = <STR_LIT> ; return false ; } } } if ( isVOP1 ( MI ) || isVOP2 ( MI ) || isVOP3 ( MI ) || isVOPC ( MI ) ) { const int OpIndices [ ] = { Src0Idx , Src1Idx , Src2Idx } ; unsigned ConstantBusCount = <NUM_LIT> ; if ( <STR_LIT> ( Opcode , <STR_LIT> :: imm ) != - <NUM_LIT> ) ++ ConstantBusCount ; unsigned SGPRUsed = findImplicitSGPRRead ( MI ) ; if ( SGPRUsed != <STR_LIT> ) ++ ConstantBusCount ; for ( int OpIdx : OpIndices ) { if ( OpIdx == - <NUM_LIT> ) break ; const MachineOperand & MO = MI . getOperand ( OpIdx ) ; if ( usesConstantBus ( MRI , MO , getOpSize ( Opcode , OpIdx ) ) ) { if ( MO . isReg ( ) ) { if ( MO . getReg ( ) != SGPRUsed ) ++ ConstantBusCount ; SGPRUsed = MO . getReg ( ) ; } else { ++ ConstantBusCount ; } } } if ( ConstantBusCount > <NUM_LIT> ) { ErrInfo = <STR_LIT> ; return false ; } } if ( Desc . getOpcode ( ) == <STR_LIT> || Desc . getOpcode ( ) == <STR_LIT> ) { const MachineOperand & Src0 = MI . getOperand ( Src0Idx ) ; const MachineOperand & Src1 = MI . getOperand ( Src1Idx ) ; const MachineOperand & Src2 = MI . getOperand ( Src2Idx ) ; if ( Src0 . isReg ( ) && Src1 . isReg ( ) && Src2 . isReg ( ) ) { if ( ! compareMachineOp ( Src0 , Src1 ) && ! compareMachineOp ( Src0 , Src2 ) ) { ErrInfo = <STR_LIT> ; return false ; }


<s> case PseudoSourceValue :: ExternalSymbolCallEntry : case PseudoSourceValue :: TargetCustom : return ST . get<ISA_LIT>AS ( ) . CONSTANT_ADDRESS ; }

return ST . get<ISA_LIT>AS ( ) . FLAT_ADDRESS ;


<s> switch ( Kind ) { case PseudoSourceValue :: Stack : case PseudoSourceValue :: FixedStack : return ST . get<ISA_LIT>AS ( ) . PRIVATE_ADDRESS ; case PseudoSourceValue :: ConstantPool : case PseudoSourceValue :: GOT : case PseudoSourceValue :: JumpTable : case PseudoSourceValue :: GlobalValueCallEntry : case PseudoSourceValue :: ExternalSymbolCallEntry : case PseudoSourceValue :: TargetCustom : return ST . get<ISA_LIT>AS ( ) . CONSTANT_ADDRESS ; }


<s> const MCInstrDesc & OpDesc = get ( getSGPRSpillSaveOpcode ( SpillSize ) ) ; if ( Register :: isVirtualRegister ( SrcReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( SrcReg , & <STR_LIT> ) ; } BuildMI ( MBB , MI , DL , OpDesc ) . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ;

if ( RI . spillSGPRToVGPR ( ) ) FrameInfo . setStackID ( FrameIndex , TargetStackID :: SGPRSpill ) ;


<s> if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ; assert ( SrcReg != <STR_LIT> && <STR_LIT> ) ; assert ( SrcReg != <STR_LIT> && SrcReg != <STR_LIT> && SrcReg != <STR_LIT> && <STR_LIT> ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillSaveOpcode ( SpillSize ) ) ; if ( Register :: isVirtualRegister ( SrcReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( SrcReg , & <STR_LIT> ) ; } BuildMI ( MBB , MI , DL , OpDesc ) . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ; if ( RI . spillSGPRToVGPR ( ) ) FrameInfo . setStackID ( FrameIndex , TargetStackID :: SGPRSpill ) ; return ; } unsigned Opcode = RI . hasAGPRs ( RC ) ? getAGPRSpillSaveOpcode ( SpillSize ) : getVGPRSpillSaveOpcode ( SpillSize ) ; MFI -> setHasSpilledVGPRs ( ) ; BuildMI ( MBB , MI , DL , get ( Opcode ) ) . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addReg ( MFI -> getScratchRSrcReg ( ) ) . addReg ( MFI -> getStackPtrOffsetReg ( ) ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ;


<s> case PseudoSourceValue :: JumpTable : case PseudoSourceValue :: GlobalValueCallEntry : case PseudoSourceValue :: ExternalSymbolCallEntry : case PseudoSourceValue :: TargetCustom : return <ISA_LIT>AS :: CONSTANT_ADDRESS ;


<s> case PseudoSourceValue :: ExternalSymbolCallEntry : case PseudoSourceValue :: TargetCustom : return <ISA_LIT>AS :: CONSTANT_ADDRESS ; case PseudoSourceValue :: CapTable :


<s> if ( Register :: isVirtualRegister ( DestReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( DestReg , & <STR_LIT> ) ; } if ( RI . spillSGPRToVGPR ( ) ) FrameInfo . setStackID ( FrameIndex , TargetStackID :: SGPRSpill ) ; BuildMI ( MBB , MI , DL , OpDesc , DestReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ; return ; } unsigned Opcode = RI . hasAGPRs ( RC ) ? getAGPRSpillRestoreOpcode ( SpillSize ) : getVGPRSpillRestoreOpcode ( SpillSize ) ;

auto MIB = BuildMI ( MBB , MI , DL , get ( Opcode ) , DestReg ) ;


<s> MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , Size , Align ) ; if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ; assert ( DestReg != <STR_LIT> && <STR_LIT> ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillRestoreOpcode ( SpillSize ) ) ; if ( Register :: isVirtualRegister ( DestReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( DestReg , & <STR_LIT> ) ; }


<s> MFI -> setHasSpilledSGPRs ( ) ; assert ( SrcReg != <STR_LIT> && <STR_LIT> ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillSaveOpcode ( SpillSize ) ) ; if ( Register :: isVirtualRegister ( SrcReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( SrcReg , & <STR_LIT> ) ; } BuildMI ( MBB , MI , DL , OpDesc ) . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ; if ( RI . spillSGPRToVGPR ( ) ) FrameInfo . setStackID ( FrameIndex , TargetStackID :: SGPRSpill ) ; return ; } unsigned Opcode = RI . hasAGPRs ( RC ) ? getAGPRSpillSaveOpcode ( SpillSize ) : getVGPRSpillSaveOpcode ( SpillSize ) ; MFI -> setHasSpilledVGPRs ( ) ; auto MIB = BuildMI ( MBB , MI , DL , get ( Opcode ) ) ; if ( RI . hasAGPRs ( RC ) ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; Register Tmp = MRI . createVirtualRegister ( & <STR_LIT> ) ; MIB . addReg ( Tmp , RegState :: Define ) ; } MIB . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addReg ( MFI -> getScratchRSrcReg ( ) ) . addReg ( MFI -> getStackPtrOffsetReg ( ) ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ;


<s> if ( Src -> isImm ( ) ) Offset = Src -> getImm ( ) ; else if ( Src -> isCImm ( ) ) Offset = Src -> getCImm ( ) -> getZExtValue ( ) ; } if ( Offset && isLegalMUBUFImmOffset ( Offset ) ) VAddr = getNamedOperand ( * Add , <STR_LIT> :: src1 ) ; else Offset = <NUM_LIT> ; } BuildMI ( * MBB , Inst , Inst . getDebugLoc ( ) , get ( <STR_LIT> ) , VDst ) . add ( * VAddr ) . add ( * getNamedOperand ( Inst , <STR_LIT> :: sbase ) ) . addImm ( <NUM_LIT> ) . addImm ( Offset ) . addImm ( getNamedOperand ( Inst , <STR_LIT> :: glc ) -> getImm ( ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . setMemRefs ( Inst . memoperands_begin ( ) , Inst . memoperands_end ( ) ) ; MRI . replaceRegWith ( getNamedOperand ( Inst , <STR_LIT> :: sdst ) -> getReg ( ) , VDst ) ; addUsersToMoveToVALUWorklist ( VDst , MRI , Worklist ) ; Inst . eraseFromParent ( ) ; continue ; } } if ( NewOpcode == <STR_LIT> ) { legalizeOperands ( Inst ) ; continue ; } const MCInstrDesc & NewDesc = get ( NewOpcode ) ; Inst . setDesc ( NewDesc ) ; for ( unsigned i = Inst . getNumOperands ( ) - <NUM_LIT> ; i > <NUM_LIT> ; -- i ) { MachineOperand & Op = Inst . getOperand ( i ) ; if ( Op . isReg ( ) && Op . getReg ( ) == <STR_LIT> ) { Inst . RemoveOperand ( i ) ; addSCCDefUsersToVALUWorklist ( Inst , Worklist ) ; } } if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { unsigned Size = ( Opcode == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; Inst . addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ; Inst . addOperand ( MachineOperand :: CreateImm ( Size ) ) ; } else if ( Opcode == <STR_LIT> ) { Inst . addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ; } Inst . addImplicitDefUseOperands ( * Inst . getParent ( ) -> getParent ( ) ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { const MachineOperand & OffsetWidthOp = Inst . getOperand ( <NUM_LIT> ) ; assert ( OffsetWidthOp . isImm ( ) && <STR_LIT> ) ; uint32_t Imm = OffsetWidthOp . getImm ( ) ; uint32_t Offset = Imm & <NUM_LIT> ; uint32_t BitWidth = ( Imm & <NUM_LIT> ) >> <NUM_LIT> ; Inst . RemoveOperand ( <NUM_LIT> ) ; Inst . addOperand ( MachineOperand :: CreateImm ( Offset ) ) ; Inst . addOperand ( MachineOperand :: CreateImm ( BitWidth ) ) ; }

bool HasDst = Inst . getOperand ( <NUM_LIT> ) . isReg ( ) && Inst . getOperand ( <NUM_LIT> ) . isDef ( ) ;


<s> splitScalar64BitBinaryOp ( Worklist , Inst , <STR_LIT> ) ; Inst . eraseFromParent ( ) ; continue ; case <STR_LIT> : { unsigned VDst = MRI . createVirtualRegister ( & <STR_LIT> ) ; const MachineOperand * VAddr = getNamedOperand ( Inst , <STR_LIT> :: soff ) ; auto Add = MRI . getUniqueVRegDef ( VAddr -> getReg ( ) ) ; unsigned Offset = <NUM_LIT> ; if ( Add && Add -> getOpcode ( ) == <STR_LIT> ) { const MachineOperand * Src = getNamedOperand ( * Add , <STR_LIT> :: src0 ) ; if ( Src && Src -> isReg ( ) ) { auto Mov = MRI . getUniqueVRegDef ( Src -> getReg ( ) ) ; if ( Mov && Mov -> getOpcode ( ) == <STR_LIT> ) Src = & Mov -> getOperand ( <NUM_LIT> ) ; } if ( Src ) { if ( Src -> isImm ( ) ) Offset = Src -> getImm ( ) ; else if ( Src -> isCImm ( ) ) Offset = Src -> getCImm ( ) -> getZExtValue ( ) ; } if ( Offset && isLegalMUBUFImmOffset ( Offset ) ) VAddr = getNamedOperand ( * Add , <STR_LIT> :: src1 ) ; else Offset = <NUM_LIT> ; } BuildMI ( * MBB , Inst , Inst . getDebugLoc ( ) , get ( <STR_LIT> ) , VDst ) . add ( * VAddr ) . add ( * getNamedOperand ( Inst , <STR_LIT> :: sbase ) ) . addImm ( <NUM_LIT> ) . addImm ( Offset ) . addImm ( getNamedOperand ( Inst , <STR_LIT> :: glc ) -> getImm ( ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . setMemRefs ( Inst . memoperands_begin ( ) , Inst . memoperands_end ( ) ) ; MRI . replaceRegWith ( getNamedOperand ( Inst , <STR_LIT> :: sdst ) -> getReg ( ) , VDst ) ; addUsersToMoveToVALUWorklist ( VDst , MRI , Worklist ) ; Inst . eraseFromParent ( ) ; continue ; } } if ( NewOpcode == <STR_LIT> ) { legalizeOperands ( Inst ) ; continue ; } const MCInstrDesc & NewDesc = get ( NewOpcode ) ; Inst . setDesc ( NewDesc ) ; for ( unsigned i = Inst . getNumOperands ( ) - <NUM_LIT> ; i > <NUM_LIT> ; -- i ) { MachineOperand & Op = Inst . getOperand ( i ) ; if ( Op . isReg ( ) && Op . getReg ( ) == <STR_LIT> ) { Inst . RemoveOperand ( i ) ; addSCCDefUsersToVALUWorklist ( Inst , Worklist ) ; } } if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { unsigned Size = ( Opcode == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; Inst . addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ; Inst . addOperand ( MachineOperand :: CreateImm ( Size ) ) ; } else if ( Opcode == <STR_LIT> ) { Inst . addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ;


<s> return ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_3_64 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_7_64 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_15_64 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) || <STR_LIT> . contains ( SrcReg ) ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) ; return ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) || <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_1 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_2 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) || <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_3 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) || <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_7 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) || <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_15 ; } else { llvm_unreachable ( <STR_LIT> ) ; } if ( RI . getHWRegIndex ( DestReg ) <= RI . getHWRegIndex ( SrcReg ) ) Forward = true ; else Forward = false ; for ( unsigned Idx = <NUM_LIT> ; Idx < SubIndices . size ( ) ; ++ Idx ) { unsigned SubIdx ; if ( Forward ) SubIdx = SubIndices [ Idx ] ;

else SubIdx = SubIndices [ SubIndices . size ( ) - Idx - <NUM_LIT> ] ;


<s> BuildMI ( & MBB , DL , get ( <STR_LIT> ) ) . addMBB ( TBB ) ; return <NUM_LIT> ; } assert ( TBB && Cond [ <NUM_LIT> ] . isImm ( ) ) ; unsigned Opcode = getBranchOpcode ( static_cast < BranchPredicate > ( Cond [ <NUM_LIT> ] . getImm ( ) ) ) ; if ( ! FBB ) { BuildMI ( & MBB , DL , get ( Opcode ) ) . addMBB ( TBB ) ; return <NUM_LIT> ;


<s> <ISA_LIT>MachineFunctionInfo * MFI = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineFrameInfo * FrameInfo = MF -> getFrameInfo ( ) ; DebugLoc DL = MBB . findDebugLoc ( MI ) ; unsigned Size = FrameInfo -> getObjectSize ( FrameIndex ) ; unsigned Align = FrameInfo -> getObjectAlignment ( FrameIndex ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOStore , Size , Align ) ; if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( SrcReg ) && RC -> getSize ( ) == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( SrcReg , & <STR_LIT> ) ; } unsigned Opcode = getSGPRSpillSaveOpcode ( RC -> getSize ( ) ) ; BuildMI ( MBB , MI , DL , get ( Opcode ) ) . addReg ( SrcReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) ; return ; } if ( ! ST . isVGPRSpillingEnabled ( * MF -> getFunction ( ) ) ) { LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ) . addReg ( SrcReg ) ; return ; } assert ( RI . hasVGPRs ( RC ) && <STR_LIT> ) ; unsigned Opcode = getVGPRSpillSaveOpcode ( RC -> getSize ( ) ) ;


<s> if ( ST . hasAddNoCarry ( ) ) return BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DestReg ) ;

Register UnusedCarry = RS . scavengeRegister ( RI . getBoolRC ( ) , I , <NUM_LIT> , false ) ;


<s> if ( ! UnusedCarry . isValid ( ) ) return MachineInstrBuilder ( ) ;

return BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DestReg ) . addReg ( UnusedCarry , RegState :: Define | RegState :: Dead ) ;


<s> unsigned Opcode = getSGPRSpillRestoreOpcode ( RC -> getSize ( ) ) ; BuildMI ( MBB , MI , DL , get ( Opcode ) , DestReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) ; return ; } if ( ! ST . isVGPRSpillingEnabled ( MFI ) ) { LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) ; return ; } assert ( RI . hasVGPRs ( RC ) && <STR_LIT> ) ; unsigned Opcode = getVGPRSpillRestoreOpcode ( RC -> getSize ( ) ) ; BuildMI ( MBB , MI , DL , get ( Opcode ) , DestReg ) . addFrameIndex ( FrameIndex ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addMemOperand ( MMO ) ;


<s> unsigned Size = FrameInfo -> getObjectSize ( FrameIndex ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , Size , Align ) ; if ( RI . isSGPRClass ( RC ) ) { unsigned Opcode = getSGPRSpillRestoreOpcode ( RC -> getSize ( ) ) ; BuildMI ( MBB , MI , DL , get ( Opcode ) , DestReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) ; return ; } if ( ! ST . isVGPRSpillingEnabled ( MFI ) ) { LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) ; return ;


<s> return ; } if ( ! ST . isVGPRSpillingEnabled ( MFI ) ) { LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ) . addReg ( SrcReg ) ; return ; } assert ( RI . hasVGPRs ( RC ) && <STR_LIT> ) ; unsigned Opcode = getVGPRSpillSaveOpcode ( RC -> getSize ( ) ) ; MFI -> setHasSpilledVGPRs ( ) ;

BuildMI ( MBB , MI , DL , get ( Opcode ) ) . addReg ( SrcReg ) . addFrameIndex ( FrameIndex ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addMemOperand ( MMO ) ;


<s> case <STR_LIT> : if ( RI . hasVGPRs ( NewDstRC ) || NewDstRC == & <STR_LIT> ) return nullptr ; NewDstRC = RI . getEquivalentVGPRClass ( NewDstRC ) ; if ( ! NewDstRC ) return nullptr ; return NewDstRC ;


<s> if ( MI . mayStore ( ) && isSMRD ( MI ) ) return true ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) return true ; if ( MI . isCall ( ) || MI . isInlineAsm ( ) ) return true ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) return true ;


<s> unsigned OpSubReg = Op . getSubReg ( ) ; const TargetRegisterClass * OpRC = RI . getSubClassWithSubReg ( RI . getRegClassForReg ( MRI , OpReg ) , OpSubReg ) ; if ( DstRC == OpRC ) return ; unsigned DstReg = MRI . createVirtualRegister ( DstRC ) ; MachineInstr * Copy = BuildMI ( InsertMBB , I , DL , get ( <STR_LIT> ) , DstReg ) . add ( Op ) ; Op . setReg ( DstReg ) ; Op . setSubReg ( <NUM_LIT> ) ;

MachineInstr * Def = MRI . getVRegDef ( OpReg ) ;


<s> unsigned OpSubReg = Op . getSubReg ( ) ; const TargetRegisterClass * OpRC = RI . getSubClassWithSubReg ( RI . getRegClassForReg ( MRI , OpReg ) , OpSubReg ) ; if ( DstRC == OpRC ) return ; unsigned DstReg = MRI . createVirtualRegister ( DstRC ) ; MachineInstr * Copy = BuildMI ( InsertMBB , I , DL , get ( <STR_LIT> ) , DstReg ) . add ( Op ) ; Op . setReg ( DstReg ) ; Op . setSubReg ( <NUM_LIT> ) ; MachineInstr * Def = MRI . getVRegDef ( OpReg ) ; if ( ! Def ) return ;

if ( Def -> isMoveImmediate ( ) && DstRC != & <STR_LIT> ) FoldImmediate ( * Copy , * Def , OpReg , & MRI ) ;


<s> unsigned Opc = MI . getOpcode ( ) ; int VOP3Idx [ <NUM_LIT> ] = { <STR_LIT> ( Opc , <STR_LIT> :: src0 ) , <STR_LIT> ( Opc , <STR_LIT> :: src1 ) , <STR_LIT> ( Opc , <STR_LIT> :: src2 ) } ; int ConstantBusLimit = ST . getConstantBusLimit ( Opc ) ; int LiteralLimit = ST . hasVOP3Literal ( ) ? <NUM_LIT> : <NUM_LIT> ; SmallDenseSet < unsigned > SGPRsUsed ; unsigned SGPRReg = findUsedSGPR ( MI , VOP3Idx ) ; if ( SGPRReg != <STR_LIT> ) { SGPRsUsed . insert ( SGPRReg ) ; -- ConstantBusLimit ; } for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) { int Idx = VOP3Idx [ i ] ; if ( Idx == - <NUM_LIT> ) break ; MachineOperand & MO = MI . getOperand ( Idx ) ; if ( ! MO . isReg ( ) ) { if ( ! isLiteralConstantLike ( MO , get ( Opc ) . OpInfo [ Idx ] ) ) continue ; if ( LiteralLimit > <NUM_LIT> && ConstantBusLimit > <NUM_LIT> ) { -- LiteralLimit ; -- ConstantBusLimit ; continue ; } -- LiteralLimit ; -- ConstantBusLimit ; legalizeOpWithMove ( MI , Idx ) ;


<s> assert ( MI . getDesc ( ) . getNumImplicitDefs ( ) == <NUM_LIT> ) ; assert ( MI . getDesc ( ) . getNumImplicitUses ( ) == <NUM_LIT> ) ; return MI . getNumOperands ( ) == <NUM_LIT> ; default : return false ; }


<s> unsigned TIDIGXReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; unsigned TIDIGYReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; unsigned TIDIGZReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; unsigned InputPtrReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; for ( unsigned Reg : { TIDIGXReg , TIDIGYReg , TIDIGZReg } ) { if ( ! Entry . isLiveIn ( Reg ) ) Entry . addLiveIn ( Reg ) ; } RS -> enterBasicBlock ( Entry ) ; unsigned STmp0 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; unsigned STmp1 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp0 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Z ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Y ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( STmp1 ) . addReg ( STmp0 ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp1 ) . addReg ( TIDIGXReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp0 ) . addReg ( TIDIGYReg ) . addReg ( TIDReg ) ; getAddNoCarry ( Entry , Insert , DL , TIDReg ) . addReg ( TIDReg ) . addReg ( TIDIGZReg ) ;


<s> unsigned TIDIGZReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; unsigned InputPtrReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; for ( unsigned Reg : { TIDIGXReg , TIDIGYReg , TIDIGZReg } ) { if ( ! Entry . isLiveIn ( Reg ) ) Entry . addLiveIn ( Reg ) ; } RS -> enterBasicBlock ( Entry ) ; unsigned STmp0 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; unsigned STmp1 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp0 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Z ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Y ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( STmp1 ) . addReg ( STmp0 ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp1 ) . addReg ( TIDIGXReg ) ;

BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp0 ) . addReg ( TIDIGYReg ) . addReg ( TIDReg ) ;


<s> <ISA_LIT>EncodingFamily Gen = subtargetEncodingFamily ( ST ) ; if ( ( get ( Opcode ) . TSFlags & <STR_LIT> ) != <NUM_LIT> && ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 ) Gen = <STR_LIT> ; if ( get ( Opcode ) . TSFlags & <STR_LIT> ) Gen = ST . getGeneration ( ) == <ISA_LIT>Subtarget :: GFX9 ? <STR_LIT> : <STR_LIT> ; if ( ST . hasUnpackedD16VMem ( ) && ( get ( Opcode ) . TSFlags & <STR_LIT> ) ) Gen = <STR_LIT> ; int MCOp = <STR_LIT> ( Opcode , Gen ) ; if ( MCOp == - <NUM_LIT> ) return Opcode ; if ( MCOp == ( uint16_t ) - <NUM_LIT> ) return - <NUM_LIT> ; return MCOp ;


<s> <ISA_LIT>InstrInfo :: <ISA_LIT>InstrInfo ( const <ISA_LIT>Subtarget & ST ) : <ISA_LIT>GenInstrInfo ( <STR_LIT> , <STR_LIT> ) , RI ( ST ) , ST ( ST ) {


<s> case <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS : case <ISA_LIT>Subtarget :: GFX9 : return <STR_LIT> ; }


<s> } unsigned Width0 = MIa . memoperands ( ) . front ( ) -> getSize ( ) ; unsigned Width1 = MIb . memoperands ( ) . front ( ) -> getSize ( ) ; return offsetsDoNotOverlap ( Width0 , Offset0 , Width1 , Offset1 ) ;


<s> if ( ! BaseOp ) { return false ; } BaseOps . push_back ( BaseOp ) ; Offset = OffsetOp -> getImm ( ) ; } else { const MachineOperand * Offset0Op = getNamedOperand ( LdSt , <STR_LIT> :: offset0 ) ; const MachineOperand * Offset1Op = getNamedOperand ( LdSt , <STR_LIT> :: offset1 ) ; unsigned Offset0 = Offset0Op -> getImm ( ) ; unsigned Offset1 = Offset1Op -> getImm ( ) ; if ( Offset0 + <NUM_LIT> != Offset1 ) return false ; unsigned EltSize ; if ( LdSt . mayLoad ( ) ) EltSize = TRI -> getRegSizeInBits ( * getOpRegClass ( LdSt , <NUM_LIT> ) ) / <NUM_LIT> ; else { assert ( LdSt . mayStore ( ) ) ; int Data0Idx = <STR_LIT> ( Opc , <STR_LIT> :: data0 ) ; EltSize = TRI -> getRegSizeInBits ( * getOpRegClass ( LdSt , Data0Idx ) ) / <NUM_LIT> ; } if ( isStride64 ( Opc ) ) EltSize *= <NUM_LIT> ; BaseOps . push_back ( BaseOp ) ; Offset = EltSize * Offset0 ; } return true ; } if ( isMUBUF ( LdSt ) || isMTBUF ( LdSt ) ) { const MachineOperand * SOffset = getNamedOperand ( LdSt , <STR_LIT> :: soffset ) ; if ( SOffset && SOffset -> isReg ( ) ) { const MachineOperand * AddrReg = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ; if ( AddrReg && ! AddrReg -> isFI ( ) ) return false ; const MachineOperand * RSrc = getNamedOperand ( LdSt , <STR_LIT> :: srsrc ) ; const <ISA_LIT>MachineFunctionInfo * MFI = LdSt . getParent ( ) -> getParent ( ) -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( RSrc -> getReg ( ) != MFI -> getScratchRSrcReg ( ) ) return false ; const MachineOperand * OffsetImm = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; BaseOps . push_back ( RSrc ) ; BaseOps . push_back ( SOffset ) ; Offset = OffsetImm -> getImm ( ) ; return true ; } BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: srsrc ) ; if ( ! BaseOp ) return false ;


<s> if ( isMUBUF ( LdSt ) || isMTBUF ( LdSt ) ) { const MachineOperand * SOffset = getNamedOperand ( LdSt , <STR_LIT> :: soffset ) ; if ( SOffset && SOffset -> isReg ( ) ) { const MachineOperand * AddrReg = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ; if ( AddrReg && ! AddrReg -> isFI ( ) ) return false ; const MachineOperand * RSrc = getNamedOperand ( LdSt , <STR_LIT> :: srsrc ) ; const <ISA_LIT>MachineFunctionInfo * MFI = LdSt . getParent ( ) -> getParent ( ) -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( RSrc -> getReg ( ) != MFI -> getScratchRSrcReg ( ) ) return false ; const MachineOperand * OffsetImm = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; BaseOps . push_back ( RSrc ) ; BaseOps . push_back ( SOffset ) ; Offset = OffsetImm -> getImm ( ) ; return true ; } BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: srsrc ) ; if ( ! BaseOp ) return false ; BaseOps . push_back ( BaseOp ) ; BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ; if ( BaseOp ) BaseOps . push_back ( BaseOp ) ; const MachineOperand * OffsetImm = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; Offset = OffsetImm -> getImm ( ) ; if ( SOffset ) Offset += SOffset -> getImm ( ) ; return true ; } if ( isSMRD ( LdSt ) ) { BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: sbase ) ; if ( ! BaseOp ) return false ; BaseOps . push_back ( BaseOp ) ; OffsetOp = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; Offset = OffsetOp ? OffsetOp -> getImm ( ) : <NUM_LIT> ; return true ;


<s> const MachineOperand * Src0Mods = getNamedOperand ( MI , <STR_LIT> :: src0_modifiers ) ; const MachineOperand * Src1 = getNamedOperand ( MI , <STR_LIT> :: src1 ) ; const MachineOperand * Src1Mods = getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) ; const MachineOperand * Src2 = getNamedOperand ( MI , <STR_LIT> :: src2 ) ; const MachineOperand * Clamp = getNamedOperand ( MI , <STR_LIT> :: clamp ) ; const MachineOperand * Omod = getNamedOperand ( MI , <STR_LIT> :: omod ) ; MachineInstrBuilder MIB ; if ( ! Src0Mods && ! Src1Mods && ! Clamp && ! Omod && ( ST . getConstantBusLimit ( Opc ) > <NUM_LIT> || ! Src0 -> isReg ( ) || ! RI . isSGPRReg ( MBB -> getParent ( ) -> getRegInfo ( ) , Src0 -> getReg ( ) ) ) ) { if ( auto Imm = getFoldableImm ( Src2 ) ) { unsigned NewOpc = IsFMA ? ( IsF16 ? <STR_LIT> : <STR_LIT> ) : ( IsF16 ? <STR_LIT> : <STR_LIT> ) ; if ( pseudoToMCOpcode ( NewOpc ) != - <NUM_LIT> ) { MIB = BuildMI ( * MBB , MI , MI . getDebugLoc ( ) , get ( NewOpc ) ) . add ( * Dst ) . add ( * Src0 ) . add ( * Src1 ) . addImm ( Imm ) ; updateLiveVariables ( LV , MI , * MIB ) ; return MIB ; } } unsigned NewOpc = IsFMA ? ( IsF16 ? <STR_LIT> : <STR_LIT> ) : ( IsF16 ? <STR_LIT> : <STR_LIT> ) ; if ( auto Imm = getFoldableImm ( Src1 ) ) { if ( pseudoToMCOpcode ( NewOpc ) != - <NUM_LIT> ) { MIB = BuildMI ( * MBB , MI , MI . getDebugLoc ( ) , get ( NewOpc ) ) . add ( * Dst ) . add ( * Src0 ) . addImm ( Imm ) . add ( * Src2 ) ; updateLiveVariables ( LV , MI , * MIB ) ; return MIB ; } } if ( auto Imm = getFoldableImm ( Src0 ) ) { if ( pseudoToMCOpcode ( NewOpc ) != - <NUM_LIT> && isOperandLegal ( MI , <STR_LIT> ( NewOpc , <STR_LIT> :: src0 ) , Src1 ) ) { MIB = BuildMI ( * MBB , MI , MI . getDebugLoc ( ) , get ( NewOpc ) ) . add ( * Dst ) . add ( * Src1 ) . addImm ( Imm ) . add ( * Src2 ) ; updateLiveVariables ( LV , MI , * MIB ) ; return MIB ; } } } unsigned NewOpc = IsFMA ? ( IsF16 ? <STR_LIT> : <STR_LIT> ) : ( IsF16 ? <STR_LIT> : <STR_LIT> ) ; if ( pseudoToMCOpcode ( NewOpc ) == - <NUM_LIT> ) return nullptr ;

MIB = BuildMI ( * MBB , MI , MI . getDebugLoc ( ) , get ( NewOpc ) ) . add ( * Dst ) . addImm ( Src0Mods ? Src0Mods -> getImm ( ) : <NUM_LIT> ) . add ( * Src0 ) . addImm ( Src1Mods ? Src1Mods -> getImm ( ) : <NUM_LIT> ) . add ( * Src1 ) . addImm ( <NUM_LIT> ) . add ( * Src2 ) . addImm ( Clamp ? Clamp -> getImm ( ) : <NUM_LIT> ) . addImm ( Omod ? Omod -> getImm ( ) : <NUM_LIT> ) ;


<s> case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return ST . hasDLInsts ( ) ? <STR_LIT> : <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; } llvm_unreachable ( <STR_LIT> ) ;


<s> case <STR_LIT> : { unsigned NumOps = MI . getDesc ( ) . getNumOperands ( ) + MI . getDesc ( ) . getNumImplicitUses ( ) ; return MI . getNumOperands ( ) == NumOps ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; default : return false ; }


<s> case <STR_LIT> : case <STR_LIT> : return MI . getNumOperands ( ) == MI . getDesc ( ) . getNumOperands ( ) ; default : return false ; }


<s> bool <ISA_LIT>InstrInfo :: isReallyTriviallyReMaterializable ( const MachineInstr & MI , AAResults * AA ) const { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :

return MI . getNumOperands ( ) == MI . getDesc ( ) . getNumOperands ( ) ;


<s> BaseOps . push_back ( BaseOp ) ; Offset = OffsetOp -> getImm ( ) ; DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: vdst ) ; if ( DataOpIdx == - <NUM_LIT> ) DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: data0 ) ; Width = getOpSize ( LdSt , DataOpIdx ) ; } else { const MachineOperand * Offset0Op = getNamedOperand ( LdSt , <STR_LIT> :: offset0 ) ; const MachineOperand * Offset1Op = getNamedOperand ( LdSt , <STR_LIT> :: offset1 ) ; unsigned Offset0 = Offset0Op -> getImm ( ) ; unsigned Offset1 = Offset1Op -> getImm ( ) ; if ( Offset0 + <NUM_LIT> != Offset1 ) return false ; unsigned EltSize ; if ( LdSt . mayLoad ( ) ) EltSize = TRI -> getRegSizeInBits ( * getOpRegClass ( LdSt , <NUM_LIT> ) ) / <NUM_LIT> ; else { assert ( LdSt . mayStore ( ) ) ; int Data0Idx = <STR_LIT> ( Opc , <STR_LIT> :: data0 ) ; EltSize = TRI -> getRegSizeInBits ( * getOpRegClass ( LdSt , Data0Idx ) ) / <NUM_LIT> ; } if ( isStride64 ( Opc ) ) EltSize *= <NUM_LIT> ; BaseOps . push_back ( BaseOp ) ; Offset = EltSize * Offset0 ; DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: vdst ) ; if ( DataOpIdx == - <NUM_LIT> ) { DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: data0 ) ; Width = getOpSize ( LdSt , DataOpIdx ) ; DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: data1 ) ; Width += getOpSize ( LdSt , DataOpIdx ) ; } else { Width = getOpSize ( LdSt , DataOpIdx ) ; } } return true ; } if ( isMUBUF ( LdSt ) || isMTBUF ( LdSt ) ) { const MachineOperand * SOffset = getNamedOperand ( LdSt , <STR_LIT> :: soffset ) ; if ( SOffset && SOffset -> isReg ( ) ) { const MachineOperand * AddrReg = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ; if ( AddrReg && ! AddrReg -> isFI ( ) ) return false ; const MachineOperand * RSrc = getNamedOperand ( LdSt , <STR_LIT> :: srsrc ) ; const <ISA_LIT>MachineFunctionInfo * MFI = LdSt . getParent ( ) -> getParent ( ) -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;

if ( RSrc -> getReg ( ) != MFI -> getScratchRSrcReg ( ) ) return false ;


<s> BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ; if ( BaseOp ) BaseOps . push_back ( BaseOp ) ; const MachineOperand * OffsetImm = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; Offset = OffsetImm -> getImm ( ) ; if ( SOffset ) Offset += SOffset -> getImm ( ) ; } DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: vdst ) ; if ( DataOpIdx == - <NUM_LIT> ) DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: vdata ) ; Width = getOpSize ( LdSt , DataOpIdx ) ; return true ; } if ( isMIMG ( LdSt ) ) { int SRsrcIdx = <STR_LIT> ( Opc , <STR_LIT> :: srsrc ) ; BaseOps . push_back ( & LdSt . getOperand ( SRsrcIdx ) ) ; int VAddr0Idx = <STR_LIT> ( Opc , <STR_LIT> :: vaddr0 ) ; if ( VAddr0Idx >= <NUM_LIT> ) { for ( int I = VAddr0Idx ; I < SRsrcIdx ; ++ I ) BaseOps . push_back ( & LdSt . getOperand ( I ) ) ; } else { BaseOps . push_back ( getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ) ; } Offset = <NUM_LIT> ; return true ; } if ( isSMRD ( LdSt ) ) { BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: sbase ) ; if ( ! BaseOp ) return false ; BaseOps . push_back ( BaseOp ) ; OffsetOp = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; Offset = OffsetOp ? OffsetOp -> getImm ( ) : <NUM_LIT> ; DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: sdst ) ; Width = getOpSize ( LdSt , DataOpIdx ) ; return true ; } if ( isFLAT ( LdSt ) ) { BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ; if ( BaseOp ) BaseOps . push_back ( BaseOp ) ; BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: saddr ) ; if ( BaseOp ) BaseOps . push_back ( BaseOp ) ; Offset = getNamedOperand ( LdSt , <STR_LIT> :: offset ) -> getImm ( ) ; DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: vdst ) ; if ( DataOpIdx == - <NUM_LIT> ) DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: vdata ) ; Width = getOpSize ( LdSt , DataOpIdx ) ;


<s> SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: vdata ) ; if ( ! SecondDst ) SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: vdst ) ; } else if ( isSMRD ( FirstLdSt ) && isSMRD ( SecondLdSt ) ) { FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> :: sdst ) ; SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: sdst ) ; } else if ( isDS ( FirstLdSt ) && isDS ( SecondLdSt ) ) { FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> :: vdst ) ; SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: vdst ) ; } if ( ! FirstDst || ! SecondDst ) return false ; unsigned LoadClusterThreshold = <NUM_LIT> ; const MachineRegisterInfo & MRI = FirstLdSt . getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; const Register Reg = FirstDst -> getReg ( ) ;

const TargetRegisterClass * DstRC = Register :: isVirtualRegister ( Reg ) ? MRI . getRegClass ( Reg ) : RI . getPhysRegClass ( Reg ) ;


<s> if ( isMUBUF ( Opc1 ) || isMTBUF ( Opc1 ) ) return checkInstOffsetsDoNotOverlap ( MIa , MIb ) ; return ! isFLAT ( Opc1 ) && ! isSMRD ( Opc1 ) ; } if ( isSMRD ( Opc0 ) ) { if ( isSMRD ( Opc1 ) ) return checkInstOffsetsDoNotOverlap ( MIa , MIb ) ; return ! isFLAT ( Opc1 ) && ! isMUBUF ( Opc0 ) && ! isMTBUF ( Opc0 ) ; } if ( isFLAT ( Opc0 ) ) { if ( isFLAT ( Opc1 ) ) return checkInstOffsetsDoNotOverlap ( MIa , MIb ) ; return false ; }


<s> assert ( MIb && ( MIb -> mayLoad ( ) || MIb -> mayStore ( ) ) && <STR_LIT> ) ; if ( MIa -> hasUnmodeledSideEffects ( ) || MIb -> hasUnmodeledSideEffects ( ) ) return false ; if ( MIa -> hasOrderedMemoryRef ( ) || MIb -> hasOrderedMemoryRef ( ) ) return false ; if ( isDS ( Opc0 ) ) { if ( isDS ( Opc1 ) ) return checkInstOffsetsDoNotOverlap ( MIa , MIb ) ; return ! isFLAT ( Opc1 ) ; } if ( isMUBUF ( Opc0 ) || isMTBUF ( Opc0 ) ) { if ( isMUBUF ( Opc1 ) || isMTBUF ( Opc1 ) ) return checkInstOffsetsDoNotOverlap ( MIa , MIb ) ;

return ! isFLAT ( Opc1 ) && ! isSMRD ( Opc1 ) ;


<s> unsigned TIDIGYReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: TIDIG_Y ) ; unsigned TIDIGZReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: TIDIG_Z ) ; unsigned InputPtrReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: INPUT_PTR ) ; for ( unsigned Reg : { TIDIGXReg , TIDIGYReg , TIDIGZReg } ) { if ( ! Entry . isLiveIn ( Reg ) ) Entry . addLiveIn ( Reg ) ; } RS -> enterBasicBlock ( & Entry ) ; unsigned STmp0 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; unsigned STmp1 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp0 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Z ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Y ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( STmp1 ) . addReg ( STmp0 ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp1 ) . addReg ( TIDIGXReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp0 ) . addReg ( TIDIGYReg ) . addReg ( TIDReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( TIDReg ) . addReg ( TIDIGZReg ) ; } else {

BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( - <NUM_LIT> ) . addImm ( <NUM_LIT> ) ;


<s> if ( ! MFI -> hasCalculatedTID ( ) ) { MachineBasicBlock & Entry = MBB . getParent ( ) -> front ( ) ; MachineBasicBlock :: iterator Insert = Entry . front ( ) ; DebugLoc DL = Insert -> getDebugLoc ( ) ; TIDReg = RI . findUnusedRegister ( MF -> getRegInfo ( ) , & <STR_LIT> ) ; if ( TIDReg == <STR_LIT> ) return TIDReg ; if ( MFI -> getShaderType ( ) == ShaderType :: COMPUTE && WorkGroupSize > WavefrontSize ) { unsigned TIDIGXReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: TIDIG_X ) ; unsigned TIDIGYReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: TIDIG_Y ) ; unsigned TIDIGZReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: TIDIG_Z ) ; unsigned InputPtrReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: INPUT_PTR ) ; for ( unsigned Reg : { TIDIGXReg , TIDIGYReg , TIDIGZReg } ) { if ( ! Entry . isLiveIn ( Reg ) ) Entry . addLiveIn ( Reg ) ; } RS -> enterBasicBlock ( & Entry ) ; unsigned STmp0 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; unsigned STmp1 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp0 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Z ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Y ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( STmp1 ) . addReg ( STmp0 ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp1 ) . addReg ( TIDIGXReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp0 ) . addReg ( TIDIGYReg ) . addReg ( TIDReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( TIDReg ) . addReg ( TIDIGZReg ) ; } else { BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( - <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( - <NUM_LIT> ) . addReg ( TIDReg ) ; } BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( <NUM_LIT> ) . addReg ( TIDReg ) ; MFI -> setTIDReg ( TIDReg ) ; } unsigned LDSOffset = MFI -> LDSSize + ( FrameOffset * WorkGroupSize ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , TmpReg ) . addImm ( LDSOffset ) . addReg ( TIDReg ) ;


<s> if ( CommutedOpcode == - <NUM_LIT> ) return nullptr ; int Src0Idx = <STR_LIT> ( MI -> getOpcode ( ) , <STR_LIT> :: src0 ) ; MachineOperand & Src0 = MI -> getOperand ( Src0Idx ) ; if ( ! Src0 . isReg ( ) ) return nullptr ; int Src1Idx = <STR_LIT> ( MI -> getOpcode ( ) , <STR_LIT> :: src1 ) ; if ( ( OpIdx0 != static_cast < unsigned > ( Src0Idx ) || OpIdx1 != static_cast < unsigned > ( Src1Idx ) ) && ( OpIdx0 != static_cast < unsigned > ( Src1Idx ) || OpIdx1 != static_cast < unsigned > ( Src0Idx ) ) ) return nullptr ; MachineOperand & Src1 = MI -> getOperand ( Src1Idx ) ; if ( isVOP2 ( MI -> getOpcode ( ) ) && ( ! isOperandLegal ( MI , Src0Idx , & Src1 ) || ! isOperandLegal ( MI , Src1Idx , & Src0 ) ) ) { return nullptr ; } if ( ! Src1 . isReg ( ) ) {

if ( NewMI || ! Src1 . isImm ( ) || ( ! isVOP2 ( MI -> getOpcode ( ) ) && ! isVOP3 ( MI -> getOpcode ( ) ) ) ) {


<s> assert ( <STR_LIT> . contains ( SrcReg ) ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) ; } return ; } assert ( <STR_LIT> . contains ( SrcReg ) ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) ; return ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_3 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_7 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_15 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) || <STR_LIT> . contains ( SrcReg ) ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) ; return ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) || <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_1 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_2 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) || <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_3 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) || <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_7 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) || <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_15 ; } else { llvm_unreachable ( <STR_LIT> ) ; } while ( unsigned SubIdx = * SubIndices ++ ) {

MachineInstrBuilder Builder = BuildMI ( MBB , MI , DL , get ( Opcode ) , RI . getSubReg ( DestReg , SubIdx ) ) ;


<s> const MachineOperand & MO = MI -> getOperand ( Idx ) ; if ( RI . isSGPRClassID ( Desc . OpInfo [ Idx ] . RegClass ) ) SGPRReg = MO . getReg ( ) ; if ( MO . isReg ( ) && RI . isSGPRClass ( MRI . getRegClass ( MO . getReg ( ) ) ) ) UsedSGPRs [ i ] = MO . getReg ( ) ; } if ( SGPRReg != <STR_LIT> ) return SGPRReg ; if ( UsedSGPRs [ <NUM_LIT> ] != <STR_LIT> ) {

if ( UsedSGPRs [ <NUM_LIT> ] == UsedSGPRs [ <NUM_LIT> ] || UsedSGPRs [ <NUM_LIT> ] == UsedSGPRs [ <NUM_LIT> ] ) SGPRReg = UsedSGPRs [ <NUM_LIT> ] ;


<s> uint64_t getClampMask ( const MachineInstr & MI ) const { const uint64_t ClampFlags = <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> ;

return MI . getDesc ( ) . TSFlags & ClampFlags ;


<s> else { assert ( LdSt -> mayStore ( ) ) ; int Data0Idx = <STR_LIT> ( Opc , <STR_LIT> :: data0 ) ; EltSize = getOpRegClass ( * LdSt , Data0Idx ) -> getSize ( ) ; } if ( isStride64 ( Opc ) ) EltSize *= <NUM_LIT> ; const MachineOperand * AddrReg = getNamedOperand ( * LdSt , <STR_LIT> :: addr ) ; BaseReg = AddrReg -> getReg ( ) ; Offset = EltSize * Offset0 ; return true ; } return false ; } if ( isMUBUF ( Opc ) || isMTBUF ( Opc ) ) { if ( <STR_LIT> ( Opc , <STR_LIT> :: soffset ) != - <NUM_LIT> ) return false ; const MachineOperand * AddrReg = getNamedOperand ( * LdSt , <STR_LIT> :: vaddr ) ; if ( ! AddrReg ) return false ; const MachineOperand * OffsetImm = getNamedOperand ( * LdSt , <STR_LIT> :: offset ) ; BaseReg = AddrReg -> getReg ( ) ; Offset = OffsetImm -> getImm ( ) ; return true ; } if ( isSMRD ( Opc ) ) {


<s> return RI . getRegSizeInBits ( * getOpRegClass ( MI , OpNo ) ) / <NUM_LIT> ;


<s> const <ISA_LIT>RegisterInfo & getRegisterInfo ( ) const {


<s> return get ( Opcode ) . TSFlags & <STR_LIT> ;


<s> static bool hasIntClamp ( const MachineInstr & MI ) {

return MI . getDesc ( ) . TSFlags & <STR_LIT> ;


<s> return MI . getDesc ( ) . TSFlags & <STR_LIT> ;


BuildMI ( * MI -> getParent ( ) , MI , MI -> getDebugLoc ( ) , get ( <STR_LIT> ) ) . addImm ( Arg ) ;


<s> BuildMI ( * MI -> getParent ( ) , MI , MI -> getDebugLoc ( ) , get ( <STR_LIT> ) ) . addImm ( Arg ) ;


<s> bool isDPP ( uint16_t Opcode ) const {

return get ( Opcode ) . TSFlags & <STR_LIT> ;


<s> bool isEXP ( uint16_t Opcode ) const { return get ( Opcode ) . TSFlags & <STR_LIT> ;


<s> bool isFixedSize ( uint16_t Opcode ) const { return get ( Opcode ) . TSFlags & <STR_LIT> ;


<s> bool isGather4 ( uint16_t Opcode ) const {


<s> bool isInlineConstant ( const MachineOperand & MO ) const { const MachineInstr * Parent = MO . getParent ( ) ; return isInlineConstant ( * Parent , Parent -> getOperandNo ( & MO ) ) ;


<s> return isInlineConstant ( * Parent , Parent -> getOperandNo ( & MO ) ) ;


<s> return MO . isImm ( ) && ! isInlineConstant ( MI , OpIdx ) ;


<s> const MachineOperand & MO = MI . getOperand ( OpIdx ) ;

return MO . isImm ( ) && ! isInlineConstant ( MI , OpIdx ) ;


<s> if ( isVALU ( InstDesc . Opcode ) && usesConstantBus ( MRI , * MO , DefinedRC -> getSize ( ) ) ) { unsigned SGPRUsed = MO -> isReg ( ) ? MO -> getReg ( ) : ( unsigned ) <STR_LIT> ; for ( unsigned i = <NUM_LIT> , e = MI -> getNumOperands ( ) ; i != e ; ++ i ) { if ( i == OpIdx ) continue ; const MachineOperand & Op = MI -> getOperand ( i ) ; if ( Op . isReg ( ) && Op . getReg ( ) != SGPRUsed && usesConstantBus ( MRI , Op , getOpSize ( * MI , i ) ) ) { return false ; } } } if ( MO -> isReg ( ) ) { assert ( DefinedRC ) ; const TargetRegisterClass * RC = TargetRegisterInfo :: isVirtualRegister ( MO -> getReg ( ) ) ? MRI . getRegClass ( MO -> getReg ( ) ) : RI . getPhysRegClass ( MO -> getReg ( ) ) ; return RI . getCommonSubClass ( RC , RI . getRegClass ( OpInfo . RegClass ) ) == RC ; } assert ( MO -> isImm ( ) || MO -> isTargetIndex ( ) || MO -> isFI ( ) ) ; if ( ! DefinedRC ) { return true ; } return isImmOperandLegal ( MI , OpIdx , * MO ) ;


<s> const MachineRegisterInfo & MRI = MI -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; const MCInstrDesc & InstDesc = get ( MI -> getOpcode ( ) ) ; const MCOperandInfo & OpInfo = InstDesc . OpInfo [ OpIdx ] ; const TargetRegisterClass * DefinedRC = OpInfo . RegClass != - <NUM_LIT> ? RI . getRegClass ( OpInfo . RegClass ) : nullptr ; if ( ! MO ) MO = & MI -> getOperand ( OpIdx ) ; if ( isVALU ( InstDesc . Opcode ) && usesConstantBus ( MRI , * MO , DefinedRC -> getSize ( ) ) ) { unsigned SGPRUsed = MO -> isReg ( ) ? MO -> getReg ( ) : ( unsigned ) <STR_LIT> ; for ( unsigned i = <NUM_LIT> , e = MI -> getNumOperands ( ) ; i != e ; ++ i ) { if ( i == OpIdx ) continue ; const MachineOperand & Op = MI -> getOperand ( i ) ; if ( Op . isReg ( ) && Op . getReg ( ) != SGPRUsed && usesConstantBus ( MRI , Op , getOpSize ( * MI , i ) ) ) { return false ; } } } if ( MO -> isReg ( ) ) { assert ( DefinedRC ) ; const TargetRegisterClass * RC = TargetRegisterInfo :: isVirtualRegister ( MO -> getReg ( ) ) ? MRI . getRegClass ( MO -> getReg ( ) ) : RI . getPhysRegClass ( MO -> getReg ( ) ) ; return RI . getCommonSubClass ( RC , RI . getRegClass ( OpInfo . RegClass ) ) == RC ; } assert ( MO -> isImm ( ) || MO -> isTargetIndex ( ) || MO -> isFI ( ) ) ; if ( ! DefinedRC ) { return true ; }

return isImmOperandLegal ( MI , OpIdx , * MO ) ;


<s> bool isScalarStore ( uint16_t Opcode ) const { return get ( Opcode ) . TSFlags & <STR_LIT> ;


<s> static bool isScalarUnit ( const MachineInstr & MI ) { return MI . getDesc ( ) . TSFlags & ( <STR_LIT> | <STR_LIT> ) ;


<s> return MI . getDesc ( ) . TSFlags & ( <STR_LIT> | <STR_LIT> ) ;


<s> bool isSDWA ( uint16_t Opcode ) const { return get ( Opcode ) . TSFlags & <STR_LIT> ;


<s> auto Flags = MI . getDesc ( ) . TSFlags ;

return ( Flags & <STR_LIT> ) && ! ( Flags & <STR_LIT> ) ;


<s> auto Flags = MI . getDesc ( ) . TSFlags ; return ( Flags & <STR_LIT> ) && ! ( Flags & <STR_LIT> ) ;


<s> return get ( Opcode ) . TSFlags & <STR_LIT> ;


<s> bool isVGPRCopy ( const MachineInstr & MI ) const { assert ( MI . isCopy ( ) ) ;

unsigned Dest = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> bool isVGPRCopy ( const MachineInstr & MI ) const { assert ( MI . isCopy ( ) ) ; unsigned Dest = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;

const MachineFunction & MF = * MI . getParent ( ) -> getParent ( ) ;


<s> bool isVINTRP ( uint16_t Opcode ) const { return get ( Opcode ) . TSFlags & <STR_LIT> ;


<s> bool isVMEM ( uint16_t Opcode ) const {

return isMUBUF ( Opcode ) || isMTBUF ( Opcode ) || isMIMG ( Opcode ) ;


<s> bool isVOP3P ( uint16_t Opcode ) const { return get ( Opcode ) . TSFlags & <STR_LIT> ;


<s> } else if ( RI . hasVGPRs ( RC ) && ST . isVGPRSpillingEnabled ( MFI ) ) { switch ( RC -> getSize ( ) * <NUM_LIT> ) { case <NUM_LIT> : Opcode = <STR_LIT> ; break ; case <NUM_LIT> : Opcode = <STR_LIT> ; break ; case <NUM_LIT> : Opcode = <STR_LIT> ; break ; case <NUM_LIT> : Opcode = <STR_LIT> ; break ; case <NUM_LIT> : Opcode = <STR_LIT> ; break ; case <NUM_LIT> : Opcode = <STR_LIT> ; break ; } } if ( Opcode != - <NUM_LIT> ) { unsigned Align = <NUM_LIT> ; FrameInfo -> setObjectAlignment ( FrameIndex , Align ) ; unsigned Size = FrameInfo -> getObjectSize ( FrameIndex ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , Size , Align ) ; BuildMI ( MBB , MI , DL , get ( Opcode ) , DestReg ) . addFrameIndex ( FrameIndex ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addMemOperand ( MMO ) ; } else { LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ;

BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) ;


<s> case <STR_LIT> : splitScalar64BitBinaryOp ( Worklist , Inst , <STR_LIT> ) ; Inst -> eraseFromParent ( ) ; continue ; case <STR_LIT> : splitScalar64BitBinaryOp ( Worklist , Inst , <STR_LIT> ) ; Inst -> eraseFromParent ( ) ; continue ; case <STR_LIT> : splitScalar64BitBinaryOp ( Worklist , Inst , <STR_LIT> ) ; Inst -> eraseFromParent ( ) ; continue ; case <STR_LIT> : splitScalar64BitUnaryOp ( Worklist , Inst , <STR_LIT> ) ; Inst -> eraseFromParent ( ) ; continue ; case <STR_LIT> : splitScalar64BitBCNT ( Worklist , Inst ) ; Inst -> eraseFromParent ( ) ; continue ; case <STR_LIT> : { splitScalar64BitBFE ( Worklist , Inst ) ; Inst -> eraseFromParent ( ) ; continue ; } case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : case <STR_LIT> : llvm_unreachable ( <STR_LIT> ) ; } if ( NewOpcode == <STR_LIT> ) { legalizeOperands ( Inst ) ; continue ; } const MCInstrDesc & NewDesc = get ( NewOpcode ) ; Inst -> setDesc ( NewDesc ) ; for ( unsigned i = Inst -> getNumOperands ( ) - <NUM_LIT> ; i > <NUM_LIT> ; -- i ) { MachineOperand & Op = Inst -> getOperand ( i ) ; if ( Op . isReg ( ) && Op . getReg ( ) == <STR_LIT> ) Inst -> RemoveOperand ( i ) ; } if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { unsigned Size = ( Opcode == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; Inst -> addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ; Inst -> addOperand ( MachineOperand :: CreateImm ( Size ) ) ; } else if ( Opcode == <STR_LIT> ) { Inst -> addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ;


<s> if ( NewOpcode == <STR_LIT> ) { legalizeOperands ( Inst ) ; continue ; } const MCInstrDesc & NewDesc = get ( NewOpcode ) ; Inst -> setDesc ( NewDesc ) ; for ( unsigned i = Inst -> getNumOperands ( ) - <NUM_LIT> ; i > <NUM_LIT> ; -- i ) { MachineOperand & Op = Inst -> getOperand ( i ) ; if ( Op . isReg ( ) && Op . getReg ( ) == <STR_LIT> ) Inst -> RemoveOperand ( i ) ; } if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { unsigned Size = ( Opcode == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; Inst -> addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ; Inst -> addOperand ( MachineOperand :: CreateImm ( Size ) ) ; } else if ( Opcode == <STR_LIT> ) { Inst -> addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ; } Inst -> addImplicitDefUseOperands ( * Inst -> getParent ( ) -> getParent ( ) ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { const MachineOperand & OffsetWidthOp = Inst -> getOperand ( <NUM_LIT> ) ; assert ( OffsetWidthOp . isImm ( ) && <STR_LIT> ) ; uint32_t Imm = OffsetWidthOp . getImm ( ) ; uint32_t Offset = Imm & <NUM_LIT> ; uint32_t BitWidth = ( Imm & <NUM_LIT> ) >> <NUM_LIT> ; Inst -> RemoveOperand ( <NUM_LIT> ) ;

Inst -> addOperand ( MachineOperand :: CreateImm ( Offset ) ) ;


<s> if ( isDS ( Opc0 ) && isDS ( Opc1 ) ) return true ; if ( isSMRD ( Opc0 ) && isSMRD ( Opc1 ) ) return true ;

if ( ( isMUBUF ( Opc0 ) || isMTBUF ( Opc0 ) ) && ( isMUBUF ( Opc1 ) || isMTBUF ( Opc1 ) ) ) return true ;


<s> unsigned Opc0 = FirstLdSt -> getOpcode ( ) ; unsigned Opc1 = SecondLdSt -> getOpcode ( ) ; if ( NumLoads > <NUM_LIT> ) return false ; if ( isDS ( Opc0 ) && isDS ( Opc1 ) ) return true ; if ( isSMRD ( Opc0 ) && isSMRD ( Opc1 ) ) return true ; if ( ( isMUBUF ( Opc0 ) || isMTBUF ( Opc0 ) ) && ( isMUBUF ( Opc1 ) || isMTBUF ( Opc1 ) ) ) return true ; return false ;


<s> bool sopkIsZext ( uint16_t Opcode ) const {


<s> case <NUM_LIT> : Opcode = <STR_LIT> ; break ; case <NUM_LIT> : Opcode = <STR_LIT> ; break ; case <NUM_LIT> : Opcode = <STR_LIT> ; break ; case <NUM_LIT> : Opcode = <STR_LIT> ; break ; case <NUM_LIT> : Opcode = <STR_LIT> ; break ; case <NUM_LIT> : Opcode = <STR_LIT> ; break ; } } if ( Opcode != - <NUM_LIT> ) { MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; unsigned Size = FrameInfo -> getObjectSize ( FrameIndex ) ; unsigned Align = FrameInfo -> getObjectAlignment ( FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOStore , Size , Align ) ; FrameInfo -> setObjectAlignment ( FrameIndex , <NUM_LIT> ) ; BuildMI ( MBB , MI , DL , get ( Opcode ) ) . addReg ( SrcReg ) . addFrameIndex ( FrameIndex ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addMemOperand ( MMO ) ; } else { LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ;

BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ) . addReg ( SrcReg ) ;


<s> static bool usesLGKM_CNT ( const MachineInstr & MI ) {


<s> static bool usesVM_CNT ( const MachineInstr & MI ) { return MI . getDesc ( ) . TSFlags & <STR_LIT> ;


<s> if ( isVOP1 ( Opcode ) || isVOP2 ( Opcode ) || isVOP3 ( Opcode ) || isVOPC ( Opcode ) ) { const int OpIndices [ ] = { Src0Idx , Src1Idx , Src2Idx } ; unsigned ConstantBusCount = <NUM_LIT> ; unsigned SGPRUsed = <STR_LIT> ; for ( int OpIdx : OpIndices ) { if ( OpIdx == - <NUM_LIT> ) break ; const MachineOperand & MO = MI -> getOperand ( OpIdx ) ; if ( usesConstantBus ( MRI , MO , getOpSize ( Opcode , OpIdx ) ) ) { if ( MO . isReg ( ) ) { if ( MO . getReg ( ) != SGPRUsed ) ++ ConstantBusCount ; SGPRUsed = MO . getReg ( ) ; } else { ++ ConstantBusCount ; } } } if ( ConstantBusCount > <NUM_LIT> ) { ErrInfo = <STR_LIT> ; return false ; } } if ( Desc . getOpcode ( ) == <STR_LIT> || Desc . getOpcode ( ) == <STR_LIT> ) { const MachineOperand & Src0 = MI -> getOperand ( Src0Idx ) ; const MachineOperand & Src1 = MI -> getOperand ( Src1Idx ) ; const MachineOperand & Src2 = MI -> getOperand ( Src2Idx ) ; if ( Src0 . isReg ( ) && Src1 . isReg ( ) && Src2 . isReg ( ) ) { if ( ! compareMachineOp ( Src0 , Src1 ) && ! compareMachineOp ( Src0 , Src2 ) ) { ErrInfo = <STR_LIT> ; return false ; }


return allowsMisalignedMemoryAccessesImpl ( Ty . getSizeInBits ( ) , AddrSpace , Alignment , Flags , IsFast ) ;


<s> if ( ! Node -> hasAnyUseOfValue ( <NUM_LIT> ) ) { MI . setDesc ( TII -> get ( NoRetAtomicOp ) ) ; MI . RemoveOperand ( <NUM_LIT> ) ; return ; }

if ( ( Node -> hasNUsesOfValue ( <NUM_LIT> , <NUM_LIT> ) && Node -> use_begin ( ) -> isMachineOpcode ( ) && Node -> use_begin ( ) -> getMachineOpcode ( ) == <STR_LIT> && ! Node -> use_begin ( ) -> hasAnyUseOfValue ( <NUM_LIT> ) ) ) {


<s> if ( ( Node -> hasNUsesOfValue ( <NUM_LIT> , <NUM_LIT> ) && Node -> use_begin ( ) -> isMachineOpcode ( ) && Node -> use_begin ( ) -> getMachineOpcode ( ) == <STR_LIT> && ! Node -> use_begin ( ) -> hasAnyUseOfValue ( <NUM_LIT> ) ) ) { unsigned Def = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MI . setDesc ( TII -> get ( NoRetAtomicOp ) ) ; MI . RemoveOperand ( <NUM_LIT> ) ; BuildMI ( * MI . getParent ( ) , MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Def ) ;


<s> bool Unpacked = Subtarget -> hasUnpackedD16VMem ( ) ; EVT LoadVT = M -> getValueType ( <NUM_LIT> ) ; EVT EquivLoadVT = LoadVT ; if ( Unpacked && LoadVT . isVector ( ) ) { EquivLoadVT = LoadVT . isVector ( ) ? EVT :: getVectorVT ( * DAG . getContext ( ) , <STR_LIT> , LoadVT . getVectorNumElements ( ) ) : LoadVT ;


<s> EVT IntLoadVT = LoadVT . changeTypeToInteger ( ) ; SmallVector < SDValue , <NUM_LIT> > Elts ; DAG . ExtractVectorElements ( Result , Elts ) ; for ( SDValue & Elt : Elts ) Elt = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Elt ) ; Result = DAG . getBuildVector ( IntLoadVT , DL , Elts ) ;

return DAG . getNode ( <STR_LIT> , DL , LoadVT , Result ) ;


<s> static SDValue adjustLoadValueTypeImpl ( SDValue Result , EVT LoadVT , const SDLoc & DL , SelectionDAG & DAG , bool Unpacked ) { if ( ! LoadVT . isVector ( ) ) return Result ; if ( Unpacked ) {

EVT IntLoadVT = LoadVT . changeTypeToInteger ( ) ;


<s> return allocateSGPR32InputImpl ( CCInfo , & <STR_LIT> , <NUM_LIT> ) ;


<s> static ArgDescriptor allocateSGPR32InputImpl ( CCState & CCInfo , const TargetRegisterClass * RC , unsigned NumArgRegs ) { ArrayRef < MCPhysReg > ArgSGPRs = makeArrayRef ( RC -> begin ( ) , <NUM_LIT> ) ; unsigned RegIdx = CCInfo . getFirstUnallocated ( ArgSGPRs ) ; if ( RegIdx == ArgSGPRs . size ( ) ) report_fatal_error ( <STR_LIT> ) ; unsigned Reg = ArgSGPRs [ RegIdx ] ; Reg = CCInfo . AllocateReg ( Reg ) ; assert ( Reg != <STR_LIT> ) ; MachineFunction & MF = CCInfo . getMachineFunction ( ) ; MF . addLiveIn ( Reg , RC ) ;

return ArgDescriptor :: createRegister ( Reg ) ;


<s> } if ( Info . hasWorkItemIDY ( ) ) { unsigned Reg = <STR_LIT> ; MF . addLiveIn ( Reg , & <STR_LIT> ) ; CCInfo . AllocateReg ( Reg ) ; Info . setWorkItemIDY ( ArgDescriptor :: createRegister ( Reg ) ) ; } if ( Info . hasWorkItemIDZ ( ) ) { unsigned Reg = <STR_LIT> ; MF . addLiveIn ( Reg , & <STR_LIT> ) ; CCInfo . AllocateReg ( Reg ) ; Info . setWorkItemIDZ ( ArgDescriptor :: createRegister ( Reg ) ) ;


<s> if ( Info . hasDispatchID ( ) ) ArgInfo . DispatchID = allocateSGPR64Input ( CCInfo ) ; if ( Info . hasWorkGroupIDX ( ) ) ArgInfo . WorkGroupIDX = allocateSGPR32Input ( CCInfo ) ; if ( Info . hasWorkGroupIDY ( ) ) ArgInfo . WorkGroupIDY = allocateSGPR32Input ( CCInfo ) ; if ( Info . hasWorkGroupIDZ ( ) ) ArgInfo . WorkGroupIDZ = allocateSGPR32Input ( CCInfo ) ;

if ( Info . hasImplicitArgPtr ( ) ) ArgInfo . ImplicitArgPtr = allocateSGPR64Input ( CCInfo ) ;


<s> if ( Info . hasWorkItemIDX ( ) ) Info . setWorkItemIDX ( allocateVGPR32Input ( CCInfo ) ) ; if ( Info . hasWorkItemIDY ( ) ) Info . setWorkItemIDY ( allocateVGPR32Input ( CCInfo ) ) ;

if ( Info . hasWorkItemIDZ ( ) ) Info . setWorkItemIDZ ( allocateVGPR32Input ( CCInfo ) ) ;


<s> static void allocateSpecialInputVGPRs ( CCState & CCInfo , MachineFunction & MF , const <ISA_LIT>RegisterInfo & TRI , <ISA_LIT>MachineFunctionInfo & Info ) { if ( Info . hasWorkItemIDX ( ) ) Info . setWorkItemIDX ( allocateVGPR32Input ( CCInfo ) ) ;

if ( Info . hasWorkItemIDY ( ) ) Info . setWorkItemIDY ( allocateVGPR32Input ( CCInfo ) ) ;


<s> static ArgDescriptor allocateVGPR32Input ( CCState & CCInfo ) { ArrayRef < MCPhysReg > ArgVGPRs = makeArrayRef ( <STR_LIT> . begin ( ) , <NUM_LIT> ) ; unsigned RegIdx = CCInfo . getFirstUnallocated ( ArgVGPRs ) ; if ( RegIdx == ArgVGPRs . size ( ) ) { int64_t Offset = CCInfo . AllocateStack ( <NUM_LIT> , <NUM_LIT> ) ; return ArgDescriptor :: createStack ( Offset ) ; } unsigned Reg = ArgVGPRs [ RegIdx ] ; Reg = CCInfo . AllocateReg ( Reg ) ; assert ( Reg != <STR_LIT> ) ; MachineFunction & MF = CCInfo . getMachineFunction ( ) ; MF . addLiveIn ( Reg , & <STR_LIT> ) ; return ArgDescriptor :: createRegister ( Reg ) ;


<s> if ( IsFast ) * IsFast = false ; if ( VT == <STR_LIT> || ( VT != <STR_LIT> && VT . getSizeInBits ( ) > <NUM_LIT> && VT . getStoreSize ( ) > <NUM_LIT> ) ) { return false ; } if ( AddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || AddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) { bool AlignedBy4 = ( Align % <NUM_LIT> == <NUM_LIT> ) ;


<s> static bool bitOpWithConstantIsReducible ( unsigned Opc , uint32_t Val ) {

return ( Opc == <STR_LIT> && ( Val == <NUM_LIT> || Val == <NUM_LIT> ) ) || ( Opc == <STR_LIT> && ( Val == <NUM_LIT> || Val == <NUM_LIT> ) ) || ( Opc == <STR_LIT> && Val == <NUM_LIT> ) ;


<s> return ( Opc == <STR_LIT> && ( Val == <NUM_LIT> || Val == <NUM_LIT> ) ) || ( Opc == <STR_LIT> && ( Val == <NUM_LIT> || Val == <NUM_LIT> ) ) || ( Opc == <STR_LIT> && Val == <NUM_LIT> ) ;


<s> static SDValue buildPCRelGlobalAddress ( SelectionDAG & DAG , const GlobalValue * GV , const SDLoc & DL , unsigned Offset , EVT PtrVT , unsigned GAFlags = <ISA_LIT>InstrInfo :: MO_NONE ) { SDValue PtrLo = DAG . getTargetGlobalAddress ( GV , DL , <STR_LIT> , Offset + <NUM_LIT> , GAFlags ) ;

SDValue PtrHi = DAG . getTargetGlobalAddress ( GV , DL , <STR_LIT> , Offset + <NUM_LIT> , GAFlags == <ISA_LIT>InstrInfo :: MO_NONE ? GAFlags : GAFlags + <NUM_LIT> ) ;


<s> MachineSDNode * <ISA_LIT>TargetLowering :: buildRSRC ( SelectionDAG & DAG , const SDLoc & DL , SDValue Ptr , uint32_t RsrcDword1 , uint64_t RsrcDword2And3 ) const { SDValue PtrLo = DAG . getTargetExtractSubreg ( <STR_LIT> , DL , <STR_LIT> , Ptr ) ; SDValue PtrHi = DAG . getTargetExtractSubreg ( <STR_LIT> , DL , <STR_LIT> , Ptr ) ; if ( RsrcDword1 ) { PtrHi = SDValue ( DAG . getMachineNode ( <STR_LIT> , DL , <STR_LIT> , PtrHi , DAG . getConstant ( RsrcDword1 , DL , <STR_LIT> ) ) , <NUM_LIT> ) ; } SDValue DataLo = buildSMovImm32 ( DAG , DL , RsrcDword2And3 & UINT64_C ( <NUM_LIT> ) ) ; SDValue DataHi = buildSMovImm32 ( DAG , DL , RsrcDword2And3 >> <NUM_LIT> ) ; const SDValue Ops [ ] = { DAG . getTargetConstant ( <STR_LIT> , DL , <STR_LIT> ) , PtrLo , DAG . getTargetConstant ( <STR_LIT> , DL , <STR_LIT> ) , PtrHi , DAG . getTargetConstant ( <STR_LIT> , DL , <STR_LIT> ) , DataLo , DAG . getTargetConstant ( <STR_LIT> , DL , <STR_LIT> ) , DataHi , DAG . getTargetConstant ( <STR_LIT> , DL , <STR_LIT> ) } ;

return DAG . getMachineNode ( <STR_LIT> , DL , <STR_LIT> , Ops ) ;


<s> return SDValue ( DAG . getMachineNode ( <STR_LIT> , DL , <STR_LIT> , K ) , <NUM_LIT> ) ;


<s> bool <ISA_LIT>TargetLowering :: CanLowerReturn ( CallingConv :: ID CallConv , MachineFunction & MF , bool IsVarArg , const SmallVectorImpl < <STR_LIT> > & Outs , LLVMContext & Context ) const { if ( <STR_LIT> ( CallConv ) ) return true ; SmallVector < CCValAssign , <NUM_LIT> > RVLocs ;

CCState CCInfo ( CallConv , IsVarArg , MF , RVLocs , Context ) ;


<s> bool <ISA_LIT>TargetLowering :: CanLowerReturn ( CallingConv :: ID CallConv , MachineFunction & MF , bool IsVarArg , const SmallVectorImpl < <STR_LIT> > & Outs , LLVMContext & Context ) const { if ( <STR_LIT> ( CallConv ) ) return true ; SmallVector < CCValAssign , <NUM_LIT> > RVLocs ; CCState CCInfo ( CallConv , IsVarArg , MF , RVLocs , Context ) ;

return CCInfo . CheckReturn ( Outs , CCAssignFnForReturn ( CallConv , IsVarArg ) ) ;


<s> bool <ISA_LIT>TargetLowering :: canMergeStoresTo ( unsigned AS , EVT MemVT , const SelectionDAG & DAG ) const { if ( AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { return ( MemVT . getSizeInBits ( ) <= <NUM_LIT> * <NUM_LIT> ) ; } else if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { unsigned MaxPrivateBits = <NUM_LIT> * getSubtarget ( ) -> getMaxPrivateElementSize ( ) ; return ( MemVT . getSizeInBits ( ) <= MaxPrivateBits ) ; } else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) { return ( MemVT . getSizeInBits ( ) <= <NUM_LIT> * <NUM_LIT> ) ; } return true ;


<s> bool <ISA_LIT>TargetLowering :: canMergeStoresTo ( unsigned AS , EVT MemVT , const SelectionDAG & DAG ) const { if ( AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { return ( MemVT . getSizeInBits ( ) <= <NUM_LIT> * <NUM_LIT> ) ; } else if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { unsigned MaxPrivateBits = <NUM_LIT> * getSubtarget ( ) -> getMaxPrivateElementSize ( ) ; return ( MemVT . getSizeInBits ( ) <= MaxPrivateBits ) ; } else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) { return ( MemVT . getSizeInBits ( ) <= <NUM_LIT> * <NUM_LIT> ) ;


<s> if ( getSubtarget ( ) -> enableHugePrivateBuffer ( ) ) return ; Known . Zero . setHighBits ( AssumeFrameIndexHighZeroBits ) ;


<s> void <ISA_LIT>TargetLowering :: computeKnownBitsForFrameIndex ( const SDValue Op , KnownBits & Known , const APInt & DemandedElts , const SelectionDAG & DAG , unsigned Depth ) const { TargetLowering :: computeKnownBitsForFrameIndex ( Op , Known , DemandedElts , DAG , Depth ) ; if ( getSubtarget ( ) -> enableHugePrivateBuffer ( ) ) return ;

Known . Zero . setHighBits ( AssumeFrameIndexHighZeroBits ) ;


<s> } if ( MemVT . isFloatingPoint ( ) ) Val = getFPExtOrFPTrunc ( DAG , Val , SL , VT ) ; else if ( Signed ) Val = DAG . getSExtOrTrunc ( Val , SL , VT ) ;

else Val = DAG . getZExtOrTrunc ( Val , SL , VT ) ;


<s> if ( Arg && ( Arg -> Flags . isSExt ( ) || Arg -> Flags . isZExt ( ) ) && VT . bitsLT ( MemVT ) ) { unsigned Opc = Arg -> Flags . isZExt ( ) ? <STR_LIT> : <STR_LIT> ; Val = DAG . getNode ( Opc , SL , MemVT , Val , DAG . getValueType ( VT ) ) ; } if ( MemVT . isFloatingPoint ( ) ) Val = getFPExtOrFPTrunc ( DAG , Val , SL , VT ) ; else if ( Signed ) Val = DAG . getSExtOrTrunc ( Val , SL , VT ) ; else Val = DAG . getZExtOrTrunc ( Val , SL , VT ) ; return Val ;


<s> ObjectIdx = MF . getFrameInfo ( ) . CreateFixedObject ( <NUM_LIT> , i * <NUM_LIT> + <NUM_LIT> , true ) ; Info -> setDebuggerWorkItemIDStackObjectIndex ( i , ObjectIdx ) ; }


<s> return Subtarget -> hasFP32Denormals ( ) ; case <STR_LIT> : return Subtarget -> hasFP64Denormals ( ) ; case <STR_LIT> : return Subtarget -> hasFP16Denormals ( ) ; default : return false ;


<s> bool <ISA_LIT>TargetLowering :: denormalsEnabledForType ( EVT VT ) const { switch ( VT . getScalarType ( ) . getSimpleVT ( ) . SimpleTy ) { case <STR_LIT> : return Subtarget -> hasFP32Denormals ( ) ;


<s> MI . eraseFromParent ( ) ; return & MBB ; } if ( setM0ToIndexFromSGPR ( TII , MRI , MI , Offset , UseGPRIdxMode , false ) ) { MachineBasicBlock :: iterator I ( & MI ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; if ( UseGPRIdxMode ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( SrcVec -> getReg ( ) , RegState :: Undef , SubReg ) . add ( * Val ) . addReg ( Dst , RegState :: ImplicitDefine ) . addReg ( SrcVec -> getReg ( ) , RegState :: Implicit ) . addReg ( <STR_LIT> , RegState :: Implicit ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) ; } else { const MCInstrDesc & MovRelDesc = TII -> get ( getMOVRELDPseudo ( TRI , VecRC ) ) ; BuildMI ( MBB , I , DL , MovRelDesc ) . addReg ( Dst , RegState :: Define ) . addReg ( SrcVec -> getReg ( ) ) . add ( * Val ) . addImm ( SubReg - <STR_LIT> ) ; } MI . eraseFromParent ( ) ; return & MBB ; } if ( Val -> isReg ( ) ) MRI . clearKillFlags ( Val -> getReg ( ) ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; unsigned PhiReg = MRI . createVirtualRegister ( VecRC ) ; auto InsPt = loadM0FromVGPR ( TII , MBB , MI , SrcVec -> getReg ( ) , PhiReg , Offset , UseGPRIdxMode , false ) ; MachineBasicBlock * LoopBB = InsPt -> getParent ( ) ; if ( UseGPRIdxMode ) { BuildMI ( * LoopBB , InsPt , DL , TII -> get ( <STR_LIT> ) ) . addReg ( PhiReg , RegState :: Undef , SubReg ) . add ( * Val ) . addReg ( Dst , RegState :: ImplicitDefine ) . addReg ( PhiReg , RegState :: Implicit ) . addReg ( <STR_LIT> , RegState :: Implicit ) ;

BuildMI ( * LoopBB , InsPt , DL , TII -> get ( <STR_LIT> ) ) ;


<s> MachineFunction * MF = MBB . getParent ( ) ; MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned SrcReg = TII -> getNamedOperand ( MI , <STR_LIT> :: src ) -> getReg ( ) ; int Offset = TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) -> getImm ( ) ; const TargetRegisterClass * VecRC = MRI . getRegClass ( SrcReg ) ; unsigned SubReg ; std :: tie ( SubReg , Offset ) = computeIndirectRegAndOffset ( TRI , VecRC , SrcReg , Offset ) ; bool UseGPRIdxMode = ST . useVGPRIndexMode ( EnableVGPRIndexMode ) ; if ( setM0ToIndexFromSGPR ( TII , MRI , MI , Offset , UseGPRIdxMode , true ) ) { MachineBasicBlock :: iterator I ( & MI ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; if ( UseGPRIdxMode ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , RegState :: Undef , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) . addReg ( <STR_LIT> , RegState :: Implicit ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) ; } else { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , RegState :: Undef , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) ; } MI . eraseFromParent ( ) ;


<s> unsigned CondReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII -> get ( TargetOpcode :: PHI ) , PhiReg ) . addReg ( InitReg ) . addMBB ( & OrigBB ) . addReg ( ResultReg ) . addMBB ( & LoopBB ) ; BuildMI ( LoopBB , I , DL , TII -> get ( TargetOpcode :: PHI ) , PhiExec ) . addReg ( InitSaveExecReg ) . addMBB ( & OrigBB ) . addReg ( NewExec ) . addMBB ( & LoopBB ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , CurrentIdxReg ) . addReg ( IdxReg . getReg ( ) , getUndefRegState ( IdxReg . isUndef ( ) ) ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , CondReg ) . addReg ( CurrentIdxReg ) . addReg ( IdxReg . getReg ( ) , <NUM_LIT> , IdxReg . getSubReg ( ) ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , NewExec ) . addReg ( CondReg , RegState :: Kill ) ; MRI . setSimpleHint ( NewExec , CondReg ) ; if ( UseGPRIdxMode ) { unsigned IdxReg ; if ( Offset == <NUM_LIT> ) { IdxReg = CurrentIdxReg ; } else { IdxReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , IdxReg ) . addReg ( CurrentIdxReg , RegState :: Kill ) . addImm ( Offset ) ; } unsigned IdxMode = IsIndirectSrc ? VGPRIndexMode :: SRC0_ENABLE : VGPRIndexMode :: DST_ENABLE ; MachineInstr * SetOn = BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( IdxReg , RegState :: Kill ) . addImm ( IdxMode ) ; SetOn -> getOperand ( <NUM_LIT> ) . setIsUndef ( ) ; } else { if ( Offset == <NUM_LIT> ) { BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( CurrentIdxReg , RegState :: Kill ) ; } else { BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( CurrentIdxReg , RegState :: Kill ) . addImm ( Offset ) ; } } MachineInstr * InsertPt = BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( NewExec ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( & LoopBB ) ;


<s> BuildMI ( LoopBB , I , DL , TII -> get ( TargetOpcode :: PHI ) , PhiReg ) . addReg ( InitReg ) . addMBB ( & OrigBB ) . addReg ( ResultReg ) . addMBB ( & LoopBB ) ; BuildMI ( LoopBB , I , DL , TII -> get ( TargetOpcode :: PHI ) , PhiExec ) . addReg ( InitSaveExecReg ) . addMBB ( & OrigBB ) . addReg ( NewExec ) . addMBB ( & LoopBB ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , CurrentIdxReg ) . addReg ( IdxReg . getReg ( ) , getUndefRegState ( IdxReg . isUndef ( ) ) ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , CondReg ) . addReg ( CurrentIdxReg ) . addReg ( IdxReg . getReg ( ) , <NUM_LIT> , IdxReg . getSubReg ( ) ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , NewExec ) . addReg ( CondReg , RegState :: Kill ) ; MRI . setSimpleHint ( NewExec , CondReg ) ; if ( UseGPRIdxMode ) { unsigned IdxReg ; if ( Offset == <NUM_LIT> ) { IdxReg = CurrentIdxReg ; } else { IdxReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , IdxReg ) . addReg ( CurrentIdxReg , RegState :: Kill ) . addImm ( Offset ) ; } unsigned IdxMode = IsIndirectSrc ? VGPRIndexMode :: SRC0_ENABLE : VGPRIndexMode :: DST_ENABLE ; MachineInstr * SetOn = BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( IdxReg , RegState :: Kill ) . addImm ( IdxMode ) ; SetOn -> getOperand ( <NUM_LIT> ) . setIsUndef ( ) ; } else {


<s> DAG . getContext ( ) -> diagnose ( BadIntrin ) ; return DAG . getUNDEF ( VT ) ;


<s> DiagnosticInfoUnsupported BadIntrin ( DAG . getMachineFunction ( ) . getFunction ( ) , <STR_LIT> , DL . getDebugLoc ( ) ) ; DAG . getContext ( ) -> diagnose ( BadIntrin ) ;


<s> if ( Info -> isEntryFunction ( ) ) { reservePrivateMemoryRegs ( getTargetMachine ( ) , MF , * TRI , * Info ) ; } bool NeedSP = ! Info -> isEntryFunction ( ) || MFI . hasVarSizedObjects ( ) || MFI . hasCalls ( ) ; if ( NeedSP ) { unsigned ReservedStackPtrOffsetReg = TRI -> reservedStackPtrOffsetReg ( MF ) ; Info -> setStackPtrOffsetReg ( ReservedStackPtrOffsetReg ) ; assert ( Info -> getStackPtrOffsetReg ( ) != Info -> getFrameOffsetReg ( ) ) ; assert ( ! TRI -> isSubRegister ( Info -> getScratchRSrcReg ( ) , Info -> getStackPtrOffsetReg ( ) ) ) ;

MRI . replaceRegWith ( <STR_LIT> , Info -> getStackPtrOffsetReg ( ) ) ;


<s> reservePrivateMemoryRegs ( getTargetMachine ( ) , MF , * TRI , * Info ) ; } bool NeedSP = ! Info -> isEntryFunction ( ) || MFI . hasVarSizedObjects ( ) || MFI . hasCalls ( ) ; if ( NeedSP ) { unsigned ReservedStackPtrOffsetReg = TRI -> reservedStackPtrOffsetReg ( MF ) ; Info -> setStackPtrOffsetReg ( ReservedStackPtrOffsetReg ) ; assert ( Info -> getStackPtrOffsetReg ( ) != Info -> getFrameOffsetReg ( ) ) ; assert ( ! TRI -> isSubRegister ( Info -> getScratchRSrcReg ( ) , Info -> getStackPtrOffsetReg ( ) ) ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> Value * Ptr = II -> getArgOperand ( <NUM_LIT> ) ; AccessTy = II -> getType ( ) ; Ops . push_back ( Ptr ) ; return true ; } default : return false ;


<s> } SmallVector < SDValue , <NUM_LIT> > VecElts ( NumElts ) ; for ( unsigned i = <NUM_LIT> ; i < Elts . size ( ) ; ++ i ) { SDValue Elt = Elts [ i ] ; if ( Elt . getValueType ( ) != <STR_LIT> ) Elt = DAG . getBitcast ( <STR_LIT> , Elt ) ; VecElts [ i ] = Elt ; } for ( unsigned i = Elts . size ( ) ; i < NumElts ; ++ i ) VecElts [ i ] = DAG . getUNDEF ( <STR_LIT> ) ; if ( NumElts == <NUM_LIT> ) return VecElts [ <NUM_LIT> ] ; return DAG . getBuildVector ( Type , DL , VecElts ) ;


<s> if ( C . isNaN ( ) ) { APFloat CanonicalQNaN = APFloat :: getQNaN ( C . getSemantics ( ) ) ; if ( C . isSignaling ( ) ) { return DAG . getConstantFP ( CanonicalQNaN , SL , VT ) ; } if ( C . bitcastToAPInt ( ) != CanonicalQNaN . bitcastToAPInt ( ) ) return DAG . getConstantFP ( CanonicalQNaN , SL , VT ) ;


<s> static uint32_t getConstantPermuteMask ( uint32_t C ) { uint32_t ZeroByteMask = <NUM_LIT> ; if ( ! ( C & <NUM_LIT> ) ) ZeroByteMask |= <NUM_LIT> ; if ( ! ( C & <NUM_LIT> ) ) ZeroByteMask |= <NUM_LIT> ; if ( ! ( C & <NUM_LIT> ) ) ZeroByteMask |= <NUM_LIT> ; if ( ! ( C & <NUM_LIT> ) ) ZeroByteMask |= <NUM_LIT> ; uint32_t NonZeroByteMask = ~ ZeroByteMask ; if ( ( NonZeroByteMask & C ) != NonZeroByteMask ) return <NUM_LIT> ; return C ;


<s> if ( ! ( C & <NUM_LIT> ) ) ZeroByteMask |= <NUM_LIT> ; uint32_t NonZeroByteMask = ~ ZeroByteMask ; if ( ( NonZeroByteMask & C ) != NonZeroByteMask ) return <NUM_LIT> ; return C ;


<s> if ( GlueChain -> getNumValues ( ) <= <NUM_LIT> ) { return DAG . getNode ( Opcode , SL , VT , A , B ) ; } assert ( GlueChain -> getNumValues ( ) == <NUM_LIT> ) ; SDVTList VTList = DAG . getVTList ( VT , <STR_LIT> , <STR_LIT> ) ; switch ( Opcode ) {

default : llvm_unreachable ( <STR_LIT> ) ;


<s> return Op . getValueType ( ) . bitsLE ( VT ) ? DAG . getNode ( <STR_LIT> , DL , VT , Op ) : DAG . getNode ( <STR_LIT> , DL , VT , Op ) ;


<s> SDValue <ISA_LIT>TargetLowering :: getFPExtOrFPTrunc ( SelectionDAG & DAG , SDValue Op , const SDLoc & DL , EVT VT ) const { return Op . getValueType ( ) . bitsLE ( VT ) ? DAG . getNode ( <STR_LIT> , DL , VT , Op ) : DAG . getNode ( <STR_LIT> , DL , VT , Op ) ;


<s> static SDValue getFPTernOp ( SelectionDAG & DAG , unsigned Opcode , const SDLoc & SL , EVT VT , SDValue A , SDValue B , SDValue C , SDValue GlueChain ) { if ( GlueChain -> getNumValues ( ) <= <NUM_LIT> ) {

return DAG . getNode ( Opcode , SL , VT , A , B , C ) ;


<s> EVT VT = N0 -> getValueType ( <NUM_LIT> ) ; if ( ( VT == <STR_LIT> && ! Subtarget -> hasFP32Denormals ( ) ) || ( VT == <STR_LIT> && ! Subtarget -> hasFP16Denormals ( ) ) ) return <STR_LIT> ; const TargetOptions & Options = DAG . getTarget ( ) . Options ; if ( ( Options . AllowFPOpFusion == FPOpFusion :: Fast || Options . UnsafeFPMath || ( N0 -> getFlags ( ) . hasAllowContract ( ) && N1 -> getFlags ( ) . hasAllowContract ( ) ) ) && isFMAFasterThanFMulAndFAdd ( VT ) ) { return <STR_LIT> ; }


<s> EVT VT = N0 -> getValueType ( <NUM_LIT> ) ; if ( ( VT == <STR_LIT> && ! Subtarget -> hasFP32Denormals ( ) ) || ( VT == <STR_LIT> && ! Subtarget -> hasFP16Denormals ( ) ) ) return <STR_LIT> ; const TargetOptions & Options = DAG . getTarget ( ) . Options ; if ( ( Options . AllowFPOpFusion == FPOpFusion :: Fast || Options . UnsafeFPMath || ( N0 -> getFlags ( ) . hasAllowContract ( ) && N1 -> getFlags ( ) . hasAllowContract ( ) ) ) && isFMAFasterThanFMulAndFAdd ( VT ) ) {


<s> uint64_t Offset = getImplicitParameterOffset ( DAG . getMachineFunction ( ) , FIRST_IMPLICIT ) ; return lowerKernArgParameterPtr ( DAG , SL , DAG . getEntryNode ( ) , Offset ) ;


<s> static SDValue getLoadExtOrTrunc ( SelectionDAG & DAG , <STR_LIT> ExtType , SDValue Op , const SDLoc & SL , EVT VT ) { if ( VT . bitsLT ( Op . getValueType ( ) ) ) return DAG . getNode ( <STR_LIT> , SL , VT , Op ) ; switch ( ExtType ) {


<s> if ( VT . bitsLT ( Op . getValueType ( ) ) ) return DAG . getNode ( <STR_LIT> , SL , VT , Op ) ; switch ( ExtType ) { case <STR_LIT> : return DAG . getNode ( <STR_LIT> , SL , VT , Op ) ; case <STR_LIT> : return DAG . getNode ( <STR_LIT> , SL , VT , Op ) ;


<s> SDVTList VTs = DAG . getVTList ( <STR_LIT> , <STR_LIT> ) ; SDValue Mad = DAG . getNode ( MadOpc , SL , VTs , N0 , N1 , N2 ) ;

return DAG . getNode ( <STR_LIT> , SL , VT , Mad ) ;


<s> static unsigned getMOVRELDPseudo ( const <ISA_LIT>RegisterInfo & TRI , const TargetRegisterClass * VecRC ) { switch ( TRI . getRegSizeInBits ( * VecRC ) ) { case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ;


<s> return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ;


<s> if ( Size == <NUM_LIT> ) return <NUM_LIT> * NumElts ; if ( Size == <NUM_LIT> && Subtarget -> has16BitInsts ( ) ) return ( VT . getVectorNumElements ( ) + <NUM_LIT> ) / <NUM_LIT> ; } return TargetLowering :: getNumRegistersForCallingConv ( Context , CC , VT ) ;


<s> if ( Size == <NUM_LIT> ) return NumElts ; if ( Size == <NUM_LIT> ) return <NUM_LIT> * NumElts ;

if ( Size == <NUM_LIT> && Subtarget -> has16BitInsts ( ) ) return ( VT . getVectorNumElements ( ) + <NUM_LIT> ) / <NUM_LIT> ;


<s> static uint32_t getPermuteMask ( SelectionDAG & DAG , SDValue V ) { assert ( V . getValueSizeInBits ( ) == <NUM_LIT> ) ; if ( V . getNumOperands ( ) != <NUM_LIT> ) return ~ <NUM_LIT> ; ConstantSDNode * N1 = dyn_cast < ConstantSDNode > ( V . getOperand ( <NUM_LIT> ) ) ; if ( ! N1 ) return ~ <NUM_LIT> ; uint32_t C = N1 -> getZExtValue ( ) ; switch ( V . getOpcode ( ) ) { default : break ; case <STR_LIT> : if ( uint32_t ConstMask = getConstantPermuteMask ( C ) ) { return ( <NUM_LIT> & ConstMask ) | ( <NUM_LIT> & ~ ConstMask ) ; } break ; case <STR_LIT> : if ( uint32_t ConstMask = getConstantPermuteMask ( C ) ) { return ( <NUM_LIT> & ~ ConstMask ) | ConstMask ; }


<s> SDValue <ISA_LIT>TargetLowering :: getPreloadedValue ( SelectionDAG & DAG , const <ISA_LIT>MachineFunctionInfo & MFI , EVT VT , <STR_LIT> PVID ) const { const ArgDescriptor * Reg ; const TargetRegisterClass * RC ; std :: tie ( Reg , RC ) = MFI . getPreloadedValue ( PVID ) ;

return CreateLiveInRegister ( DAG , RC , Reg -> getRegister ( ) , VT ) ;


<s> const TargetRegisterClass * RC = nullptr ; if ( Constraint . size ( ) == <NUM_LIT> ) { switch ( Constraint [ <NUM_LIT> ] ) { default : return TargetLowering :: getRegForInlineAsmConstraint ( TRI , Constraint , VT ) ; case 's' : case 'r' : switch ( VT . getSizeInBits ( ) ) { default : return std :: make_pair ( <NUM_LIT> , nullptr ) ; case <NUM_LIT> : case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; } break ; case 'v' : switch ( VT . getSizeInBits ( ) ) { default : return std :: make_pair ( <NUM_LIT> , nullptr ) ; case <NUM_LIT> : case <NUM_LIT> : RC = & <STR_LIT> ; break ;


<s> break ; } if ( RC && ( isTypeLegal ( VT ) || VT . SimpleTy == <STR_LIT> || VT . SimpleTy == <STR_LIT> || VT . SimpleTy == <STR_LIT> ) ) return std :: make_pair ( <NUM_LIT> , RC ) ; } if ( Constraint . size ( ) > <NUM_LIT> ) { if ( Constraint [ <NUM_LIT> ] == 'v' ) { RC = & <STR_LIT> ; } else if ( Constraint [ <NUM_LIT> ] == 's' ) { RC = & <STR_LIT> ; } if ( RC ) { uint32_t Idx ; bool Failed = Constraint . substr ( <NUM_LIT> ) . getAsInteger ( <NUM_LIT> , Idx ) ;

if ( ! Failed && Idx < RC -> getNumRegs ( ) ) return std :: make_pair ( RC -> getRegister ( Idx ) , RC ) ;


<s> if ( Reg == <STR_LIT> ) { report_fatal_error ( Twine ( <STR_LIT> + StringRef ( RegName ) + <STR_LIT> ) ) ; } if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS && Subtarget -> getRegisterInfo ( ) -> regsOverlap ( Reg , <STR_LIT> ) ) { report_fatal_error ( Twine ( <STR_LIT> + StringRef ( RegName ) + <STR_LIT> ) ) ; } switch ( Reg ) { case <STR_LIT> : case <STR_LIT> :


<s> unsigned Size = ScalarVT . getSizeInBits ( ) ; if ( Size == <NUM_LIT> ) return ScalarVT . getSimpleVT ( ) ; if ( Size == <NUM_LIT> ) return <STR_LIT> ;

if ( Size == <NUM_LIT> && Subtarget -> has16BitInsts ( ) ) return VT . isInteger ( ) ? <STR_LIT> : <STR_LIT> ;


<s> unsigned Size = ScalarVT . getSizeInBits ( ) ; if ( Size == <NUM_LIT> ) return ScalarVT . getSimpleVT ( ) ; if ( Size == <NUM_LIT> ) return <STR_LIT> ; if ( Size == <NUM_LIT> && Subtarget -> has16BitInsts ( ) ) return VT . isInteger ( ) ? <STR_LIT> : <STR_LIT> ;


<s> if ( Subtarget -> hasApertureRegs ( ) ) { unsigned Offset = AS == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> :: OFFSET_SRC_SHARED_BASE : <STR_LIT> :: OFFSET_SRC_PRIVATE_BASE ; unsigned WidthM1 = AS == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> :: WIDTH_M1_SRC_SHARED_BASE : <STR_LIT> :: WIDTH_M1_SRC_PRIVATE_BASE ; unsigned Encoding = <STR_LIT> :: ID_MEM_BASES << <STR_LIT> :: ID_SHIFT_ | Offset << <STR_LIT> :: OFFSET_SHIFT_ | WidthM1 << <STR_LIT> :: WIDTH_M1_SHIFT_ ; SDValue EncodingImm = DAG . getTargetConstant ( Encoding , DL , <STR_LIT> ) ; SDValue ApertureReg = SDValue ( DAG . getMachineNode ( <STR_LIT> , DL , <STR_LIT> , EncodingImm ) , <NUM_LIT> ) ; SDValue ShiftAmount = DAG . getTargetConstant ( WidthM1 + <NUM_LIT> , DL , <STR_LIT> ) ; return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , ApertureReg , ShiftAmount ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;

unsigned UserSGPR = Info -> getQueuePtrUserSGPR ( ) ;


<s> if ( ConstantFPSDNode * C = dyn_cast < ConstantFPSDNode > ( Op ) ) return C ; if ( BuildVectorSDNode * BV = dyn_cast < BuildVectorSDNode > ( Op ) ) { if ( ConstantFPSDNode * C = BV -> getConstantFPSplatNode ( ) ) return C ; }


<s> static ConstantFPSDNode * getSplatConstantFP ( SDValue Op ) {

if ( ConstantFPSDNode * C = dyn_cast < ConstantFPSDNode > ( Op ) ) return C ;


<s> const GCNSubtarget * <ISA_LIT>TargetLowering :: getSubtarget ( ) const { return Subtarget ;


<s> const GCNSubtarget * <ISA_LIT>TargetLowering :: getSubtarget ( ) const {


<s> if ( Size == <NUM_LIT> ) { RegisterVT = <STR_LIT> ; IntermediateVT = RegisterVT ; NumIntermediates = <NUM_LIT> * NumElts ; return NumIntermediates ; } if ( Size == <NUM_LIT> && Subtarget -> has16BitInsts ( ) ) { RegisterVT = VT . isInteger ( ) ? <STR_LIT> : <STR_LIT> ; IntermediateVT = RegisterVT ; NumIntermediates = ( NumElts + <NUM_LIT> ) / <NUM_LIT> ; return NumIntermediates ;


<s> SDValue IntVData = DAG . getNode ( <STR_LIT> , DL , IntStoreVT , VData ) ; EVT EquivStoreVT = EVT :: getVectorVT ( * DAG . getContext ( ) , <STR_LIT> , StoreVT . getVectorNumElements ( ) ) ; SDValue ZExt = DAG . getNode ( <STR_LIT> , DL , EquivStoreVT , IntVData ) ; return DAG . UnrollVectorOp ( ZExt . getNode ( ) ) ; }


<s> if ( Subtarget -> hasUnpackedD16VMem ( ) ) { EVT IntStoreVT = StoreVT . changeTypeToInteger ( ) ; SDValue IntVData = DAG . getNode ( <STR_LIT> , DL , IntStoreVT , VData ) ; EVT EquivStoreVT = EVT :: getVectorVT ( * DAG . getContext ( ) , <STR_LIT> , StoreVT . getVectorNumElements ( ) ) ; SDValue ZExt = DAG . getNode ( <STR_LIT> , DL , EquivStoreVT , IntVData ) ;

return DAG . UnrollVectorOp ( ZExt . getNode ( ) ) ;


<s> bool <ISA_LIT>TargetLowering :: hasBitPreservingFPLogic ( EVT VT ) const {

return isTypeLegal ( VT . getScalarType ( ) ) ;


<s> return isTypeLegal ( VT . getScalarType ( ) ) ;


<s> void <ISA_LIT>TargetLowering :: initializeSplitCSR ( MachineBasicBlock * Entry ) const {


<s> void <ISA_LIT>TargetLowering :: insertCopiesSplitCSR ( MachineBasicBlock * Entry , const SmallVectorImpl < MachineBasicBlock * > & Exits ) const { const <ISA_LIT>RegisterInfo * TRI = getSubtarget ( ) -> getRegisterInfo ( ) ; const MCPhysReg * IStart = TRI -> getCalleeSavedRegsViaCopy ( Entry -> getParent ( ) ) ; if ( ! IStart ) return ; const TargetInstrInfo * TII = Subtarget -> getInstrInfo ( ) ; MachineRegisterInfo * MRI = & Entry -> getParent ( ) -> getRegInfo ( ) ;

MachineBasicBlock :: iterator MBBI = Entry -> begin ( ) ;


<s> unsigned NewVR = MRI -> createVirtualRegister ( RC ) ; Entry -> addLiveIn ( * I ) ; BuildMI ( * Entry , MBBI , DebugLoc ( ) , TII -> get ( TargetOpcode :: COPY ) , NewVR ) . addReg ( * I ) ; for ( auto * Exit : Exits ) BuildMI ( * Exit , Exit -> getFirstTerminator ( ) , DebugLoc ( ) , TII -> get ( TargetOpcode :: COPY ) , * I ) . addReg ( NewVR ) ;


<s> switch ( V . getOpcode ( ) ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; }


<s> unsigned Opcode = Op . getOpcode ( ) ; if ( Opcode == <STR_LIT> ) return true ; if ( auto * CFP = dyn_cast < ConstantFPSDNode > ( Op ) ) { auto F = CFP -> getValueAPF ( ) ; if ( F . isNaN ( ) && F . isSignaling ( ) ) return false ; return ! F . isDenormal ( ) || denormalsEnabledForType ( Op . getValueType ( ) ) ; } if ( MaxDepth == <NUM_LIT> ) return false ; switch ( Opcode ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return Op . getValueType ( ) . getScalarType ( ) != <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> if ( Intr -> getOpcode ( ) == <STR_LIT> ) { switch ( cast < ConstantSDNode > ( Intr -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) { case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : llvm_unreachable ( <STR_LIT> ) ; default : return <NUM_LIT> ; } }


<s> bool <ISA_LIT>TargetLowering :: isCheapAddrSpaceCast ( unsigned SrcAS , unsigned DestAS ) const {

if ( SrcAS == <ISA_LIT>AS :: FLAT_ADDRESS ) return true ;


<s> if ( ConstantFPSDNode * CA = dyn_cast < ConstantFPSDNode > ( A ) ) { if ( ConstantFPSDNode * CB = dyn_cast < ConstantFPSDNode > ( B ) ) { return ( CA -> isExactlyValue ( <NUM_LIT> ) && CB -> isExactlyValue ( <NUM_LIT> ) ) || ( CA -> isExactlyValue ( <NUM_LIT> ) && CB -> isExactlyValue ( <NUM_LIT> ) ) ; }


<s> do { N = N -> getOperand ( <NUM_LIT> ) . getNode ( ) ; if ( N -> getOpcode ( ) == <STR_LIT> ) return true ; } while ( N -> getOpcode ( ) == <STR_LIT> ) ; return false ;


<s> LLVM_ATTRIBUTE_UNUSED static bool isCopyFromRegOfInlineAsm ( const SDNode * N ) { assert ( N -> getOpcode ( ) == <STR_LIT> ) ; do { N = N -> getOperand ( <NUM_LIT> ) . getNode ( ) ;

if ( N -> getOpcode ( ) == <STR_LIT> ) return true ;


<s> if ( ! mayTailCallThisCC ( CalleeCC ) ) return false ; MachineFunction & MF = DAG . getMachineFunction ( ) ; const Function & CallerF = MF . getFunction ( ) ; CallingConv :: ID CallerCC = CallerF . getCallingConv ( ) ; const <ISA_LIT>RegisterInfo * TRI = getSubtarget ( ) -> getRegisterInfo ( ) ; const uint32_t * CallerPreserved = TRI -> getCallPreservedMask ( MF , CallerCC ) ; if ( ! CallerPreserved ) return false ; bool CCMatch = CallerCC == CalleeCC ; if ( DAG . getTarget ( ) . Options . GuaranteedTailCallOpt ) { if ( canGuaranteeTCO ( CalleeCC ) && CCMatch ) return true ; return false ; } if ( IsVarArg ) return false ; for ( const Argument & Arg : CallerF . args ( ) ) { if ( Arg . hasByValAttr ( ) ) return false ; }

LLVMContext & Ctx = * DAG . getContext ( ) ;


<s> switch ( VT . getSimpleVT ( ) . SimpleTy ) { case <STR_LIT> : { if ( Subtarget -> hasFP32Denormals ( ) ) return Subtarget -> hasFastFMAF32 ( ) || Subtarget -> hasDLInsts ( ) ; return Subtarget -> hasFastFMAF32 ( ) && Subtarget -> hasDLInsts ( ) ; } case <STR_LIT> :


<s> if ( Op . getOpcode ( ) == <STR_LIT> ) { if ( Subtarget -> enableDX10Clamp ( ) ) return true ; return DAG . isKnownNeverNaN ( Op . getOperand ( <NUM_LIT> ) , SNaN , Depth + <NUM_LIT> ) ;


<s> } else if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SEA_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs / <NUM_LIT> ) ) return false ; } else if ( Subtarget -> getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs ) ) return false ; } else llvm_unreachable ( <STR_LIT> ) ; if ( AM . Scale == <NUM_LIT> ) return true ; if ( AM . Scale == <NUM_LIT> && AM . HasBaseReg ) return true ; return false ; } else if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { return isLegalMUBUFAddressingMode ( AM ) ; } else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: REGION_ADDRESS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs ) ) return false ; if ( AM . Scale == <NUM_LIT> ) return true ; if ( AM . Scale == <NUM_LIT> && AM . HasBaseReg ) return true ; return false ; } else if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS || AS == <ISA_LIT>AS :: UNKNOWN_ADDRESS_SPACE ) { return isLegalFlatAddressingMode ( AM ) ; } else { llvm_unreachable ( <STR_LIT> ) ; }


<s> } return isUInt < <NUM_LIT> > ( AM . BaseOffs ) && AM . Scale == <NUM_LIT> ;


<s> if ( ! Subtarget -> hasFlatInstOffsets ( ) ) { return AM . BaseOffs == <NUM_LIT> && AM . Scale == <NUM_LIT> ;


<s> bool <ISA_LIT>TargetLowering :: isLegalGlobalAddressingMode ( const AddrMode & AM ) const {

if ( Subtarget -> hasFlatGlobalInsts ( ) ) return isInt < <NUM_LIT> > ( AM . BaseOffs ) && AM . Scale == <NUM_LIT> ;


<s> const MemSDNode * MemNode = cast < MemSDNode > ( N ) ;

const Value * Ptr = MemNode -> getMemOperand ( ) -> getValue ( ) ;


<s> bool <ISA_LIT>TargetLowering :: isMemOpHasNoClobberedMemOperand ( const SDNode * N ) const { const MemSDNode * MemNode = cast < MemSDNode > ( N ) ; const Value * Ptr = MemNode -> getMemOperand ( ) -> getValue ( ) ; const Instruction * I = dyn_cast_or_null < Instruction > ( Ptr ) ;

return I && I -> getMetadata ( <STR_LIT> ) ;


<s> return isFlatGlobalAddrSpace ( SrcAS ) && isFlatGlobalAddrSpace ( DestAS ) ;


<s> return ( GA -> getAddressSpace ( ) == <ISA_LIT>AS :: GLOBAL_ADDRESS || GA -> getAddressSpace ( ) == <ISA_LIT>AS :: CONSTANT_ADDRESS || GA -> getAddressSpace ( ) == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) && ! shouldEmitGOTReloc ( GA -> getGlobal ( ) ) ;


<s> const <ISA_LIT>RegisterInfo & TRI = ST . getInstrInfo ( ) -> getRegisterInfo ( ) ; unsigned Reg = R -> getReg ( ) ; if ( TRI . isPhysicalRegister ( Reg ) ) return ! TRI . isSGPRReg ( MRI , Reg ) ; if ( MRI . isLiveIn ( Reg ) ) { if ( ! TRI . isSGPRReg ( MRI , Reg ) ) return true ; else if ( ! <STR_LIT> ( FLI -> Fn -> getCallingConv ( ) ) ) return true ; return false ; } const Value * V = FLI -> getValueFromVirtualReg ( Reg ) ; if ( V ) return KDA -> isDivergent ( V ) ; assert ( Reg == FLI -> DemoteRegister || isCopyFromRegOfInlineAsm ( N ) ) ; return ! TRI . isSGPRReg ( MRI , Reg ) ; } break ; case <STR_LIT> : { const LoadSDNode * L = cast < LoadSDNode > ( N ) ; unsigned AS = L -> getAddressSpace ( ) ; return AS == <ISA_LIT>AS :: PRIVATE_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ; } break ; case <STR_LIT> : return true ; break ; case <STR_LIT> : { } return <STR_LIT> ( cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ; case <STR_LIT> : return <STR_LIT> ( cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ; case <STR_LIT> :


<s> const Value * V = FLI -> getValueFromVirtualReg ( Reg ) ; if ( V ) return KDA -> isDivergent ( V ) ; assert ( Reg == FLI -> DemoteRegister || isCopyFromRegOfInlineAsm ( N ) ) ; return ! TRI . isSGPRReg ( MRI , Reg ) ; } break ; case <STR_LIT> : { const LoadSDNode * L = cast < LoadSDNode > ( N ) ; unsigned AS = L -> getAddressSpace ( ) ; return AS == <ISA_LIT>AS :: PRIVATE_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ; } break ; case <STR_LIT> : return true ; break ; case <STR_LIT> : { } return <STR_LIT> ( cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ; case <STR_LIT> : return <STR_LIT> ( cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ; case <STR_LIT> : case <STR_LIT> :


<s> bool <ISA_LIT>TargetLowering :: isShuffleMaskLegal ( ArrayRef < int > , EVT ) const {


<s> return true ; default : return false ; } } if ( VT == <STR_LIT> && Op == <STR_LIT> ) return false ;

return TargetLowering :: isTypeDesirableForOp ( Op , VT ) ;


<s> SDNode * Glued = Node -> getGluedNode ( ) ; SDValue ToVReg = DAG . getCopyToReg ( Node -> getOperand ( <NUM_LIT> ) , SL , VReg , SrcVal , SDValue ( Glued , Glued ? Glued -> getNumValues ( ) - <NUM_LIT> : <NUM_LIT> ) ) ; SDValue ToResultReg = DAG . getCopyToReg ( ToVReg , SL , SDValue ( DestReg , <NUM_LIT> ) , VReg , ToVReg . getValue ( <NUM_LIT> ) ) ; DAG . ReplaceAllUsesWith ( Node , ToResultReg . getNode ( ) ) ; DAG . RemoveDeadNode ( Node ) ; return ToResultReg . getNode ( ) ; } } SmallVector < SDValue , <NUM_LIT> > Ops ; for ( unsigned i = <NUM_LIT> ; i < Node -> getNumOperands ( ) ; ++ i ) { if ( ! isFrameIndexOp ( Node -> getOperand ( i ) ) ) {

Ops . push_back ( Node -> getOperand ( i ) ) ;