<s> buildDAGWithRegPressure ( ) ; DEBUG ( for ( SUnit & SU : SUnits ) SU . dumpAll ( this ) ) ; topologicalSort ( ) ; findRootsAndBiasEdges ( TopRoots , BotRoots ) ; SchedImpl -> initialize ( this ) ; initQueues ( TopRoots , BotRoots ) ; SUnitsLinksBackup = SUnits ; IsLowLatencySU . clear ( ) ; LowLatencyOffset . clear ( ) ; IsHighLatencySU . clear ( ) ; IsLowLatencySU . resize ( SUnits . size ( ) , <NUM_LIT> ) ; LowLatencyOffset . resize ( SUnits . size ( ) , <NUM_LIT> ) ; IsHighLatencySU . resize ( SUnits . size ( ) , <NUM_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = ( unsigned ) SUnits . size ( ) ; i != e ; ++ i ) { SUnit * SU = & SUnits [ i ] ; unsigned BaseLatReg ; int64_t OffLatReg ; if ( <ISA_LIT>TII -> isLowLatencyInstruction ( * SU -> getInstr ( ) ) ) { IsLowLatencySU [ i ] = <NUM_LIT> ; if ( <ISA_LIT>TII -> getMemOpBaseRegImmOfs ( * SU -> getInstr ( ) , BaseLatReg , OffLatReg , TRI ) ) LowLatencyOffset [ i ] = OffLatReg ; } else if ( <ISA_LIT>TII -> isHighLatencyInstruction ( * SU -> getInstr ( ) ) ) IsHighLatencySU [ i ] = <NUM_LIT> ; } <ISA_LIT>Scheduler Scheduler ( this ) ; Best = Scheduler . scheduleVariant ( <STR_LIT> , <STR_LIT> ) ; if ( Best . MaxVGPRUsage > <NUM_LIT> ) { static const std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > Variants [ ] = { { LatenciesAlone , BlockRegUsageLatency } , { LatenciesGrouped , BlockLatencyRegUsage } , { LatenciesAlonePlusConsecutive , BlockLatencyRegUsage } , } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } if ( Best . MaxVGPRUsage


<s> <ISA_LIT>ScheduleBlock * Block = ScheduledBlocks [ b ] ; std :: vector < SUnit * > SUs = Block -> getScheduledUnits ( ) ; for ( SUnit * SU : SUs ) Res . SUs . push_back ( SU -> NodeNum ) ; } Res . MaxSGPRUsage = Scheduler . getSGPRUsage ( ) ; Res . MaxVGPRUsage = Scheduler . getVGPRUsage ( ) ; return


<s> <ISA_LIT>ScheduleBlockScheduler Scheduler ( DAG , ScheduleVariant , Blocks ) ; std :: vector < <ISA_LIT>ScheduleBlock * > ScheduledBlocks ; struct <ISA_LIT>ScheduleBlockResult Res ; ScheduledBlocks = Scheduler . getBlocks ( ) ; for ( unsigned b = <NUM_LIT> ; b < ScheduledBlocks . size ( ) ;


<s> void setRepeat ( <ISA_LIT>ScheduleCandReason


<s> int i = WorkList . back ( ) ; <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ i ] ; WorkList . pop_back ( ) ; TopDownBlock2Index [ i ] = -- Id ; TopDownIndex2Block [ Id ] = i ; for ( <ISA_LIT>ScheduleBlock * Pred : Block -> getPreds ( ) ) { if ( ! -- TopDownBlock2Index [ Pred -> getID ( ) ] ) WorkList . push_back ( Pred -> getID ( ) ) ; } } for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ i ] ; for ( <ISA_LIT>ScheduleBlock * Pred : Block -> getPreds ( ) ) { assert ( TopDownBlock2Index [ i ] > TopDownBlock2Index [


<s> if ( TryCand . IsHighLatency && tryGreater ( TryCand . Height , Cand . Height , TryCand , Cand , Depth ) ) return true ; if ( tryGreater ( TryCand . NumHighLatencySuccessors , Cand . NumHighLatencySuccessors , TryCand , Cand , Successor ) ) return true ; return


<s> if ( TryCand . IsHighLatency && tryGreater ( TryCand . Height , Cand . Height , TryCand , Cand , Depth ) ) return true ; if ( tryGreater ( TryCand . NumHighLatencySuccessors , Cand . NumHighLatencySuccessors , TryCand , Cand , Successor )


<s> void <ISA_LIT>ScheduleBlock :: tryCandidateTopDown ( <ISA_LIT>SchedCandidate & Cand , <ISA_LIT>SchedCandidate & TryCand ) { if ( ! Cand . isValid ( ) ) { TryCand . Reason = NodeOrder ; return ; } if ( Cand . SGPRUsage > <NUM_LIT> && tryLess ( TryCand . SGPRUsage , Cand . SGPRUsage , TryCand , Cand , RegUsage ) ) return ; if ( tryLess ( TryCand . HasLowLatencyNonWaitedParent , Cand . HasLowLatencyNonWaitedParent , TryCand , Cand , <STR_LIT> ) ) return ; if ( tryGreater ( TryCand . IsLowLatency , Cand . IsLowLatency , TryCand , Cand , <STR_LIT> ) ) return ; if ( TryCand . IsLowLatency && tryLess ( TryCand . LowLatencyOffset , Cand . LowLatencyOffset , TryCand , Cand , <STR_LIT> ) ) return ; if ( tryLess ( TryCand . VGPRUsage , Cand . VGPRUsage , TryCand , Cand , RegUsage ) ) return ; if ( TryCand . SU -> NodeNum < Cand . SU -> NodeNum


<s> } if ( bytes > <NUM_LIT> ) return ; for ( unsigned i = <NUM_LIT> , e = Desc . getNumOperands ( ) ; i < e ; ++ i ) { if ( ! <STR_LIT> ( Desc , i ) ) continue ; const MCOperand & Op = MI . getOperand ( i ) ; if ( getLitEncoding ( Op , Desc . OpInfo [ i ] , STI ) != <NUM_LIT> ) continue ; int64_t Imm = <NUM_LIT> ; if ( Op . isImm ( ) ) Imm = Op . getImm ( ) ; else if ( Op . isExpr ( ) ) { if ( const auto * C = dyn_cast < MCConstantExpr > ( Op . getExpr ( ) ) ) Imm = C -> getValue ( ) ; } else if ( ! Op . isExpr ( ) ) llvm_unreachable ( <STR_LIT> ) ; for ( unsigned j = <NUM_LIT> ; j < <NUM_LIT> ; j ++


<s> if ( Val == <NUM_LIT> ) return <NUM_LIT> ; if ( Val == <NUM_LIT> ) return <NUM_LIT> ; if ( Val == <NUM_LIT> ) return <NUM_LIT> ; if ( Val == <NUM_LIT> ) return <NUM_LIT> ; if ( Val == <NUM_LIT> ) return <NUM_LIT> ; if ( Val == <NUM_LIT> ) return <NUM_LIT> ; if ( Val == <NUM_LIT> && STI . getFeatureBits ( ) [ <STR_LIT> ] ) return <NUM_LIT> ; return


<s> if ( Val == FloatToBits ( <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == FloatToBits ( - <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == FloatToBits ( <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == FloatToBits ( - <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == <NUM_LIT> && STI . getFeatureBits ( ) [


<s> static uint32_t getLit64Encoding ( uint64_t Val , const MCSubtargetInfo & STI ) { uint32_t IntImm = getIntInlineImmEncoding ( static_cast < int64_t > ( Val ) ) ; if ( IntImm != <NUM_LIT> ) return IntImm ; if ( Val == DoubleToBits ( <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( - <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( - <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( - <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( - <NUM_LIT> ) )


<s> if ( Val == DoubleToBits ( - <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( - <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( -


<s> const auto * C = dyn_cast < MCConstantExpr > ( MO . getExpr ( ) ) ; if ( ! C ) return <NUM_LIT> ; Imm = C -> getValue ( ) ; } else { assert ( ! MO . isFPImm ( ) ) ; if ( ! MO . isImm ( ) ) return ~ <NUM_LIT> ; Imm = MO . getImm ( ) ; } switch ( OpInfo . OperandType ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit32Encoding ( static_cast < uint32_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit64Encoding ( static_cast < uint64_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit16Encoding ( static_cast < uint16_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : { uint16_t Lo16 = static_cast < uint16_t >


<s> Imm = MO . getImm ( ) ; } switch ( OpInfo . OperandType ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit32Encoding ( static_cast < uint32_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit64Encoding ( static_cast < uint64_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit16Encoding ( static_cast < uint16_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : { uint16_t Lo16 = static_cast < uint16_t > ( Imm


<s> const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; if ( <STR_LIT> ( Desc , OpNo ) ) { uint32_t Enc = getLitEncoding ( MO , Desc . OpInfo [ OpNo ] , STI ) ; if ( Enc != ~ <NUM_LIT> && ( Enc != <NUM_LIT> || Desc . getSize ( ) == <NUM_LIT> ) ) return Enc ; } else if ( MO . isImm ( ) ) return MO .


<s> RegEnc |= SDWA9EncValues :: SRC_SGPR_MASK ; } return RegEnc ; } else { const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; uint32_t Enc = getLitEncoding ( MO , Desc . OpInfo [ OpNo ] , STI ) ; if ( Enc != ~ <NUM_LIT> && Enc != <NUM_LIT> ) { return Enc | SDWA9EncValues :: SRC_SGPR_MASK ; } } llvm_unreachable ( <STR_LIT>


<s> unsigned <ISA_LIT>MCCodeEmitter :: getSDWAVopcDstEncoding ( const MCInst & MI , unsigned OpNo , SmallVectorImpl < MCFixup > & Fixups , const MCSubtargetInfo & STI ) const { using namespace <STR_LIT> ; uint64_t RegEnc = <NUM_LIT> ; const MCOperand & MO = MI . getOperand ( OpNo ) ; unsigned Reg = MO . getReg ( ) ; if ( Reg != <STR_LIT> ) { RegEnc |= MRI . getEncodingValue ( Reg ) ; RegEnc &= SDWA9EncValues


<s> auto * BE = cast < MCBinaryExpr > ( Expr ) ; if ( BE -> getOpcode ( ) == MCBinaryExpr :: Sub ) return false ; return needsPCRel ( BE -> getLHS ( ) ) || needsPCRel ( BE -> getRHS ( ) ) ; } case MCExpr :: Unary : return needsPCRel ( cast < MCUnaryExpr > ( Expr ) -> getSubExpr ( ) ) ; case MCExpr :: Target


<s> return new <ISA_LIT>MCCodeEmitter ( MCII , MRI , Ctx


<s> } if ( bytes > <NUM_LIT> ) return ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i < e ; ++ i ) { if ( ! isSrcOperand ( Desc , i ) ) continue ; int RCID = Desc . OpInfo [ i ] . RegClass ; const MCRegisterClass & RC = MRI . getRegClass ( RCID ) ; const MCOperand & Op = MI . getOperand ( i ) ; if ( getLitEncoding ( Op , RC . getSize ( ) ) != <NUM_LIT> ) continue ; int64_t Imm = <NUM_LIT> ; if ( Op . isImm ( ) ) Imm = Op . getImm ( ) ; else if ( ! Op . isExpr ( ) ) llvm_unreachable (


<s> if ( Imm >= - <NUM_LIT> && Imm <= - <NUM_LIT> ) return <NUM_LIT> + std :: abs ( Imm ) ; return


<s> if ( Imm >= <NUM_LIT> && Imm <= <NUM_LIT> ) return <NUM_LIT> + Imm ; if ( Imm >= - <NUM_LIT> && Imm <= - <NUM_LIT> ) return <NUM_LIT> + std :: abs ( Imm ) ; return <NUM_LIT>


<s> if ( Val == DoubleToBits ( - <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( - <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( <NUM_LIT> )


<s> assert ( ! MO . isFPImm ( ) ) ; if ( ! MO . isImm ( ) ) return ~ <NUM_LIT> ; if ( OpSize == <NUM_LIT> ) return getLit32Encoding ( static_cast < uint32_t > ( MO . getImm ( ) ) ) ; assert ( OpSize == <NUM_LIT> ) ; return getLit64Encoding ( static_cast < uint64_t > ( MO . getImm


<s> unsigned <ISA_LIT>MCCodeEmitter :: getSOPPBrEncoding ( const MCInst & MI , unsigned OpNo , SmallVectorImpl < MCFixup > & Fixups , const MCSubtargetInfo & STI ) const { const MCOperand & MO = MI . getOperand ( OpNo ) ; if ( MO . isExpr (


<s> if ( MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) ) Enc |= <NUM_LIT> ; return Enc


<s> if ( <STR_LIT> ( Opcode , op_sel_hi ) != - <NUM_LIT> ) { if ( <STR_LIT> ( Opcode , src2 ) != - <NUM_LIT> ) return <NUM_LIT> ; if ( <STR_LIT> ( Opcode , src1 ) != - <NUM_LIT> ) return OP_SEL_HI_2 ; if ( <STR_LIT> ( Opcode , src0 ) != - <NUM_LIT> ) return OP_SEL_HI_1 | OP_SEL_HI_2 ; } return OP_SEL_HI_0 | OP_SEL_HI_1


<s> uint16_t IntImm = getIntInlineImmEncoding ( static_cast < int16_t > ( Val


<s> for ( unsigned e = MI . getNumOperands ( ) ; OpNo < e ; ++ OpNo ) { if ( & MO == & MI . getOperand ( OpNo ) ) break ; } const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; if ( <STR_LIT> ( Desc , OpNo ) ) { uint32_t Enc = getLitEncoding ( MO , Desc . OpInfo [ OpNo ] , STI ) ; if ( Enc != ~ <NUM_LIT> ) return


<s> using namespace <STR_LIT> ; uint64_t RegEnc = <NUM_LIT> ; const MCOperand & MO = MI . getOperand ( OpNo ) ; unsigned Reg = MO . getReg ( ) ; if ( Reg != <STR_LIT> && Reg != <STR_LIT> ) { RegEnc |= MRI . getEncodingValue ( Reg ) ; RegEnc &= SDWA9EncValues :: VOPC_DST_SGPR_MASK ; RegEnc |= SDWA9EncValues :: VOPC_DST_VCC_MASK ; } return RegEnc


<s> unsigned <ISA_LIT>MCCodeEmitter :: getSDWAVopcDstEncoding ( const MCInst & MI , unsigned OpNo , SmallVectorImpl < MCFixup > & Fixups , const MCSubtargetInfo & STI ) const { using namespace <STR_LIT> ; uint64_t RegEnc = <NUM_LIT> ; const MCOperand & MO = MI . getOperand ( OpNo ) ; unsigned Reg = MO . getReg ( ) ; if ( Reg != <STR_LIT> &&


<s> auto Offset = MI . getOperand ( OpNo ) . getImm ( ) ; assert ( ! <STR_LIT> ( STI ) || isUInt < <NUM_LIT> > ( Offset


<s> unsigned <ISA_LIT>MCCodeEmitter :: getSMEMOffsetEncoding ( const MCInst & MI , unsigned OpNo , SmallVectorImpl < MCFixup > & Fixups , const MCSubtargetInfo & STI ) const { auto Offset = MI . getOperand ( OpNo ) . getImm ( ) ; assert ( ! <STR_LIT> ( STI ) || isUInt < <NUM_LIT> > ( Offset ) ) ; return Offset


<s> unsigned Reg = MI . getOperand ( OpNo ) . getReg ( ) ; uint64_t Enc = MRI . getEncodingValue ( Reg ) ; if ( MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) ) Enc |= <NUM_LIT> ; return


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit64Encoding ( static_cast < uint64_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit16Encoding ( static_cast < uint16_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : if ( ! isUInt < <NUM_LIT> > ( Imm ) && STI . getFeatureBits ( ) [ <STR_LIT> ] ) return getLit32Encoding ( static_cast < uint32_t > ( Imm ) , STI ) ; LLVM_FALLTHROUGH ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT>


<s> uint64_t <ISA_LIT>MCCodeEmitter :: getMachineOpValue ( const MCInst & MI , const MCOperand & MO , SmallVectorImpl < MCFixup > & Fixups , const MCSubtargetInfo & STI ) const { if ( MO . isReg ( ) ) return MRI . getEncodingValue ( MO . getReg ( ) ) ; if ( MO . isExpr ( ) && MO . getExpr ( ) -> getKind ( ) != MCExpr :: Constant ) { MCFixupKind Kind ; if ( needsPCRel ( MO . getExpr ( ) ) ) Kind = FK_PCRel_4 ; else Kind = FK_Data_4


<s> if ( needsPCRel ( MO . getExpr ( ) ) ) Kind = FK_PCRel_4 ; else Kind = FK_Data_4 ; const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; uint32_t Offset = Desc . getSize ( ) ; assert ( Offset == <NUM_LIT> || Offset == <NUM_LIT> ) ; Fixups . push_back ( MCFixup :: create ( Offset , MO . getExpr ( ) , Kind , MI . getLoc ( ) ) ) ; } unsigned OpNo = <NUM_LIT> ; for ( unsigned e = MI . getNumOperands ( ) ; OpNo < e ; ++ OpNo ) { if ( & MO == & MI . getOperand ( OpNo ) ) break ; } const MCInstrDesc & Desc = MCII . get ( MI .


<s> verifyInstructionPredicates ( MI , computeAvailableFeatures ( STI . getFeatureBits ( ) ) ) ; uint64_t Encoding = getBinaryCodeForInstr ( MI , Fixups , STI ) ; const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; unsigned bytes = Desc . getSize ( ) ; for ( unsigned i = <NUM_LIT> ; i < bytes ; i ++ ) { OS . write ( ( uint8_t ) ( ( Encoding >> ( <NUM_LIT> * i ) ) & <NUM_LIT> ) ) ; } if ( <STR_LIT> ( STI ) && Desc . TSFlags & <STR_LIT> ) { int vaddr0 = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vaddr0 ) ; int srsrc = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: srsrc ) ; assert ( vaddr0 >= <NUM_LIT> && srsrc > vaddr0 ) ; unsigned NumExtraAddrs = srsrc - vaddr0 - <NUM_LIT> ; unsigned NumPadding = ( - NumExtraAddrs ) & <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i < NumExtraAddrs ; ++ i ) OS . write ( ( uint8_t ) getMachineOpValue ( MI , MI . getOperand ( vaddr0 + <NUM_LIT> + i ) , Fixups , STI ) ) ; for ( unsigned i = <NUM_LIT> ; i < NumPadding ; ++ i ) OS . write ( <NUM_LIT> ) ; } if ( ( bytes > <NUM_LIT> && STI . getFeatureBits ( ) [ <STR_LIT> ] ) || (

bytes > <NUM_LIT> && ! STI . getFeatureBits ( ) [ <STR_LIT> ] ) ) return ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit32Encoding ( static_cast < uint32_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit64Encoding ( static_cast < uint64_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit16IntEncoding ( static_cast < uint16_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit16Encoding ( static_cast < uint16_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : { if ( ! isUInt < <NUM_LIT> > ( Imm ) && STI . getFeatureBits ( ) [ <STR_LIT> ] ) return getLit32Encoding ( static_cast < uint32_t > ( Imm ) , STI ) ; if ( OpInfo . OperandType == <STR_LIT> ) return getLit16Encoding ( static_cast < uint16_t > ( Imm ) , STI ) ; LLVM_FALLTHROUGH ; } case <STR_LIT> : case <STR_LIT> : return getLit16IntEncoding ( static_cast < uint16_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT>


<s> return new <ISA_LIT>MCCodeEmitter ( MCII , Ctx


<s> if ( MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) ) Enc |= <NUM_LIT> ; return Enc


<s> if ( MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI

. getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) ) Enc |= <NUM_LIT> ;


<s> unsigned NumExtraAddrs = srsrc - vaddr0 - <NUM_LIT> ; unsigned NumPadding = ( - NumExtraAddrs ) & <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i < NumExtraAddrs ; ++ i ) OS . write ( ( uint8_t ) getMachineOpValue ( MI , MI . getOperand ( vaddr0 + <NUM_LIT> + i ) , Fixups , STI ) ) ; for ( unsigned i = <NUM_LIT> ; i < NumPadding ; ++ i ) OS . write ( <NUM_LIT> ) ; } if ( ( bytes > <NUM_LIT> && STI . getFeatureBits ( ) [ <STR_LIT> ] ) || ( bytes > <NUM_LIT> && ! STI . getFeatureBits ( ) [ <STR_LIT> ] ) ) return ; for ( unsigned i = <NUM_LIT> , e = Desc . getNumOperands ( ) ; i < e ; ++ i ) { if ( ! <STR_LIT> ( Desc , i ) ) continue ; const MCOperand & Op = MI . getOperand ( i ) ; if ( getLitEncoding ( Op , Desc . OpInfo [ i ] , STI ) != <NUM_LIT> ) continue ; int64_t Imm = <NUM_LIT>


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit32Encoding ( static_cast < uint32_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit64Encoding ( static_cast < uint64_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit16IntEncoding ( static_cast < uint16_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit16Encoding ( static_cast < uint16_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : { if ( ! isUInt < <NUM_LIT> > ( Imm ) && STI . getFeatureBits ( ) [ <STR_LIT> ] ) return getLit32Encoding ( static_cast < uint32_t > ( Imm ) , STI ) ; if ( OpInfo . OperandType == <STR_LIT> ) return getLit16Encoding ( static_cast < uint16_t > ( Imm ) , STI ) ; LLVM_FALLTHROUGH ; } case <STR_LIT> : case <STR_LIT> : return getLit16IntEncoding ( static_cast < uint16_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : { uint16_t Lo16 = static_cast < uint16_t > ( Imm ) ; uint32_t Encoding = getLit16Encoding ( Lo16 , STI ) ; return Encoding


<s> unsigned OpNo = <NUM_LIT> ; for ( unsigned e = MI . getNumOperands ( ) ; OpNo < e ; ++ OpNo ) { if ( & MO == & MI . getOperand ( OpNo ) ) break ; } const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; if ( isSrcOperand ( Desc , OpNo ) ) { int RCID = Desc . OpInfo [ OpNo ] . RegClass ; const MCRegisterClass & RC = MRI . getRegClass ( RCID ) ; uint32_t Enc = getLitEncoding ( MO , RC . getSize ( ) ) ; if ( Enc != ~ <NUM_LIT> && ( Enc != <NUM_LIT> || Desc . getSize ( ) == <NUM_LIT> ) ) return Enc ; } else if ( MO . isImm ( )


<s> if ( & MO == & MI . getOperand ( OpNo ) ) break ; } const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; if ( isSrcOperand ( Desc , OpNo ) ) { int RCID = Desc . OpInfo [ OpNo ] . RegClass ; const MCRegisterClass & RC = MRI . getRegClass ( RCID ) ; uint32_t Enc = getLitEncoding ( MO , RC . getSize ( ) ) ; if ( Enc != ~ <NUM_LIT> && ( Enc != <NUM_LIT> || Desc . getSize ( ) == <NUM_LIT> ) ) return Enc ; } else if ( MO . isImm ( ) ) return MO . getImm ( ) ; llvm_unreachable ( <STR_LIT>


<s> unsigned <ISA_LIT>MCCodeEmitter :: getAVOperandEncoding ( const MCInst & MI , unsigned OpNo , SmallVectorImpl < MCFixup > & Fixups , const MCSubtargetInfo & STI ) const { unsigned Reg = MI . getOperand ( OpNo ) . getReg ( ) ; uint64_t Enc = MRI . getEncodingValue ( Reg ) ; if ( MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) )


<s> unsigned bytes = Desc . getSize ( ) ; for ( unsigned i = <NUM_LIT> ; i < bytes ; i ++ ) { OS . write ( ( uint8_t ) ( ( Encoding >> ( <NUM_LIT> * i ) ) & <NUM_LIT> ) ) ; } if ( bytes > <NUM_LIT> ) return ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i < e ; ++ i ) { if ( ! <STR_LIT> ( Desc , i ) ) continue ; const MCOperand & Op = MI . getOperand


<s> } unsigned OpNo = <NUM_LIT> ; for ( unsigned e = MI . getNumOperands ( ) ; OpNo < e ; ++ OpNo ) { if ( & MO == & MI . getOperand ( OpNo ) ) break ; } const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; if ( <STR_LIT> ( Desc , OpNo ) ) { uint32_t Enc = getLitEncoding ( MO , Desc . OpInfo [ OpNo ] , STI ) ; if ( Enc != ~ <NUM_LIT> && ( Enc != <NUM_LIT> || Desc . getSize ( ) == <NUM_LIT> ) ) return Enc ; } else if ( MO . isImm ( ) ) return MO . getImm ( ) ; llvm_unreachable ( <STR_LIT> ) ; return <NUM_LIT>


<s> const MCOperand & MO = MI . getOperand ( OpNo ) ; unsigned Reg = MO . getReg ( ) ; RegEnc |= MRI . getEncodingValue ( Reg ) ; RegEnc &= SDWA9EncValues :: SRC_VGPR_MASK ; if ( <STR_LIT> ( <STR_LIT> ( Reg ) , & MRI ) ) { RegEnc |= SDWA9EncValues :: SRC_SGPR_MASK ; } return


<s> return getLit16Encoding ( static_cast < uint16_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : if ( ! isUInt < <NUM_LIT> > ( Imm ) && STI . getFeatureBits ( ) [ <STR_LIT> ] ) return getLit32Encoding ( static_cast < uint32_t > ( Imm ) , STI ) ; LLVM_FALLTHROUGH ; case <STR_LIT> : case <STR_LIT> : { uint16_t Lo16 = static_cast < uint16_t > ( Imm ) ; uint32_t Encoding = getLit16Encoding ( Lo16 , STI ) ; return Encoding ; } default : llvm_unreachable ( <STR_LIT> )


<s> const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; unsigned bytes = Desc . getSize ( ) ; for ( unsigned i = <NUM_LIT> ; i < bytes ; i ++ ) { OS . write ( ( uint8_t ) ( ( Encoding >> ( <NUM_LIT> * i ) ) & <NUM_LIT> ) ) ; } if ( bytes > <NUM_LIT> ) return ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i < e ; ++ i ) { if ( ! isSrcOperand ( Desc , i ) ) continue ; int RCID = Desc . OpInfo [ i ] . RegClass ; const MCRegisterClass & RC = MRI . getRegClass ( RCID ) ; const MCOperand & Op = MI . getOperand ( i ) ; if ( getLitEncoding ( Op , RC . getSize ( ) ) != <NUM_LIT> ) continue ; int64_t Imm = <NUM_LIT> ; if ( Op . isImm ( ) ) Imm = Op . getImm ( ) ; else if ( Op . isExpr ( ) ) { if ( const MCConstantExpr * C = dyn_cast < MCConstantExpr > ( Op . getExpr ( ) ) ) Imm = C -> getValue ( ) ; } else if ( ! Op . isExpr ( ) ) llvm_unreachable (


<s> uint32_t <ISA_LIT>MCCodeEmitter :: getLitEncoding ( const MCOperand & MO , unsigned OpSize ) const { int64_t Imm ; if ( MO . isExpr ( ) ) { const MCConstantExpr * C = dyn_cast < MCConstantExpr > ( MO . getExpr ( ) ) ; if ( ! C ) return <NUM_LIT> ; Imm = C -> getValue ( ) ; } else { assert ( ! MO . isFPImm ( ) ) ; if ( ! MO . isImm ( ) ) return ~ <NUM_LIT> ; Imm = MO . getImm ( ) ; } if ( OpSize == <NUM_LIT> ) return getLit32Encoding ( static_cast < uint32_t > ( Imm ) ) ; assert ( OpSize == <NUM_LIT> ) ; return getLit64Encoding ( static_cast < uint64_t > (


<s> } const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; if ( isSrcOperand ( Desc , OpNo ) ) { int RCID = Desc . OpInfo [ OpNo ] . RegClass ; const MCRegisterClass & RC = MRI . getRegClass ( RCID ) ; uint32_t Enc = getLitEncoding ( MO , RC . getSize ( ) ) ; if ( Enc != ~ <NUM_LIT> && ( Enc != <NUM_LIT> || Desc . getSize ( ) == <NUM_LIT> ) ) return Enc ; } else if ( MO . isImm ( ) ) return MO . getImm ( ) ; llvm_unreachable (


<s> case <NUM_LIT> : return getLit32Encoding ( static_cast < uint32_t > ( Imm ) , STI ) ; case <NUM_LIT> : return getLit64Encoding ( static_cast < uint64_t > ( Imm ) , STI ) ; case <NUM_LIT> : return getLit16Encoding ( static_cast < uint16_t > ( Imm ) , STI ) ; default : llvm_unreachable ( <STR_LIT>


<s> const MCOperand & MO = MI . getOperand ( OpNo ) ; unsigned Reg = MO . getReg ( ) ; RegEnc |= MRI . getEncodingValue ( Reg ) ; RegEnc &= SDWA9EncValues :: SRC_VGPR_MASK ; if ( <STR_LIT> ( <STR_LIT> ( Reg ) , & MRI ) ) { RegEnc |= SDWA9EncValues :: SRC_SGPR_MASK ; } return RegEnc


<s> uint64_t RegEnc = <NUM_LIT> ; const MCOperand & MO = MI . getOperand ( OpNo ) ; unsigned Reg = MO . getReg ( ) ; RegEnc |= MRI . getEncodingValue ( Reg ) ; RegEnc &= SDWA9EncValues :: SRC_VGPR_MASK ; if ( <STR_LIT> ( <STR_LIT> ( Reg


<s> const auto & IsSyncScopeInclusion = MMI -> isSyncScopeInclusion ( S<ISA_LIT>D , MMO -> getSyncScopeID ( ) ) ; if ( ! IsSyncScopeInclusion ) { reportUnsupported ( MI , <STR_LIT> ) ; return None ; } S<ISA_LIT>D = IsSyncScopeInclusion . getValue ( ) ? S<ISA_LIT>D : MMO -> getSyncScopeID ( ) ; Ordering = isStrongerThan ( Ordering , OpOrdering ) ? Ordering : MMO -> getOrdering ( ) ; assert ( MMO -> getFailureOrdering ( ) != AtomicOrdering :: Release && MMO -> getFailureOrdering ( ) != AtomicOrdering :: AcquireRelease ) ; FailureOrdering = isStrongerThan ( FailureOrdering , MMO -> getFailureOrdering ( ) ) ? FailureOrdering : MMO -> getFailureOrdering ( ) ; } } <ISA_LIT>AtomicScope Scope = <STR_LIT> ; <ISA_LIT>AtomicAddrSpace OrderingAddrSpace = <STR_LIT> ; bool IsCrossAddressSpaceOrdering = false ; if ( Ordering != AtomicOrdering ::


<s> if ( Generation < <ISA_LIT>Subtarget :: GFX10 ) return std :: make_unique < <ISA_LIT>Gfx7CacheControl > ( ST ) ; return std :: make_unique <


<s> if ( MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent || MOI . getFailureOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertRelease ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ; if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent || MOI . getFailureOrdering ( ) == AtomicOrdering :: Acquire || MOI . getFailureOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , isAtomicRet ( * MI ) ? <STR_LIT> : <STR_LIT> , MOI . getIsCrossAddressSpaceOrdering ( ) ,


<s> if ( MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertRelease ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ; return Changed ; } Changed |= CC -> enableVolatileAndOrNonTemporal ( MI , MOI . getInstrAddrSpace ( ) , <STR_LIT> , MOI . isVolatile ( ) , MOI . isNonTemporal


<s> <ISA_LIT>MemOpInfo ( AtomicOrdering Ordering = AtomicOrdering :: SequentiallyConsistent , <ISA_LIT>AtomicScope Scope = <STR_LIT> , <ISA_LIT>AtomicAddrSpace OrderingAddrSpace = <STR_LIT> , <ISA_LIT>AtomicAddrSpace InstrAddrSpace = <STR_LIT> , bool IsCrossAddressSpaceOrdering = true , AtomicOrdering FailureOrdering = AtomicOrdering :: SequentiallyConsistent , bool IsVolatile = false , bool IsNonTemporal = false ) : Ordering ( Ordering ) , FailureOrdering ( FailureOrdering ) , Scope ( Scope ) , OrderingAddrSpace ( OrderingAddrSpace ) , InstrAddrSpace ( InstrAddrSpace ) , IsCrossAddressSpaceOrdering ( IsCrossAddressSpaceOrdering ) , IsVolatile ( IsVolatile ) , IsNonTemporal ( IsNonTemporal ) { if ( ( OrderingAddrSpace == InstrAddrSpace ) && isPowerOf2_32 ( uint32_t ( InstrAddrSpace

) ) ) this -> IsCrossAddressSpaceOrdering = false ;


<STR_LIT> , <STR_LIT> & InstrScope , false ) ;


<s> if ( ST . hasGFX940Insts ( ) ) return std :: make_unique < <ISA_LIT>Gfx940CacheControl > ( ST ) ; if ( ST . hasGFX90AInsts ( ) ) return std :: make_unique < <ISA_LIT>Gfx90ACacheControl > ( ST ) ; if ( Generation <= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return std :: make_unique < <ISA_LIT>Gfx6CacheControl > ( ST ) ; if ( Generation < <ISA_LIT>Subtarget :: GFX10 ) return std :: make_unique < <ISA_LIT>Gfx7CacheControl > ( ST ) ; return std :: make_unique < <ISA_LIT>Gfx10CacheControl >


<s> GCNSubtarget :: Generation Generation = ST . getGeneration ( ) ; if ( ST . hasGFX940Insts ( ) ) return std :: make_unique < <ISA_LIT>Gfx940CacheControl > ( ST ) ; if ( ST . hasGFX90AInsts ( ) ) return std :: make_unique < <ISA_LIT>Gfx90ACacheControl > ( ST ) ; if ( Generation <= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return std :: make_unique < <ISA_LIT>Gfx6CacheControl >


<s> Changed |= enableSC1Bit ( MI ) ; break ; case <STR_LIT> : Changed |= enableSC0Bit ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } return


<s> assert ( MI -> mayLoad ( ) && MI -> mayStore ( ) ) ; bool Changed = false ; if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : Changed |= enableSC1Bit ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT>


<s> return enableNamedBit ( MI , <STR_LIT> :: SC1


<s> if ( Op == <STR_LIT> ) { Changed |= enableGLCBit ( MI ) ; Changed |= enableDLCBit ( MI ) ; } Changed |= insertWait ( MI , <STR_LIT> , AddrSpace , Op , false , Position :: AFTER ) ; return Changed ; } if ( IsNonTemporal ) { if ( Op == <STR_LIT> ) Changed |= enableGLCBit


<s> switch ( Scope ) { case <STR_LIT> : BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <STR_LIT> :: SC0 | <STR_LIT> :: SC1 ) ; Changed = true ; break ; case <STR_LIT> : BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <STR_LIT> :: SC1 ) ; Changed = true ; break ; case <STR_LIT> : if ( ST . isTgSplitEnabled ( ) ) { BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <STR_LIT> :: SC0 ) ; Changed = true ; } break ; case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT>


<s> if ( MI -> isBundle ( ) && MI -> mayLoadOrStore ( ) ) { MachineBasicBlock :: instr_iterator II ( MI -> getIterator ( ) ) ; for ( MachineBasicBlock :: instr_iterator I = ++ II , E = MBB . instr_end ( ) ; I != E && I -> isBundledWithPred ( ) ; ++ I ) { I -> unbundleFromPred ( ) ; for ( MachineOperand & MO : I -> operands ( ) ) if ( MO . isReg ( ) ) MO . setIsInternalRead ( false ) ; } MI -> eraseFromParent ( ) ; MI = II -> getIterator ( ) ; } if ( ! ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) )


<s> <ISA_LIT>Gfx940CacheControl ( const GCNSubtarget & ST )

: <ISA_LIT>Gfx90ACacheControl ( ST ) { }


<s> std :: unique_ptr < <ISA_LIT>CacheControl > <ISA_LIT>CacheControl :: create ( const GCNSubtarget & ST ) { GCNSubtarget :: Generation Generation = ST . getGeneration ( ) ; if ( Generation <= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return std :: make_unique < <ISA_LIT>Gfx6CacheControl > ( ST ) ; if ( Generation < <ISA_LIT>Subtarget :: GFX10 ) return std :: make_unique < <ISA_LIT>Gfx7CacheControl > (


<s> case <STR_LIT> : case <STR_LIT> : Changed |= enableGLCBit ( MI ) ; Changed |= enableDLCBit ( MI ) ; break ; case <STR_LIT> : if ( ! CuMode ) Changed |= enableGLCBit ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } return Changed


<s> bool <ISA_LIT>Gfx10CacheControl :: enableNonTemporal ( const MachineBasicBlock ::


<s> break ; case <STR_LIT> : if ( ! CuMode ) { if ( ( Op & <STR_LIT> ) != <STR_LIT> ) VMCnt |= true ; if ( ( Op & <STR_LIT> ) != <STR_LIT> ) VSCnt |= true ; } break ; case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : LGKMCnt |= IsCrossAddrSpaceOrdering ; break ; case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : case <STR_LIT> : LGKMCnt |= IsCrossAddrSpaceOrdering ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( VMCnt || LGKMCnt ) { unsigned WaitCntImmediate = <STR_LIT> ( IV , VMCnt ? <NUM_LIT> : getVmcntBitMask ( IV ) , getExpcntBitMask ( IV ) , LGKMCnt ? <NUM_LIT> : getLgkmcntBitMask ( IV ) ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( WaitCntImmediate ) ; Changed = true ; } if ( VSCnt ) { BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) ; Changed = true ; } if ( Pos == Position :: AFTER ) -- MI ; return


<s> <ISA_LIT>CacheControl :: <ISA_LIT>CacheControl ( const GCNSubtarget & ST ) { TII = ST . getInstrInfo ( ) ; IV = getIsaVersion ( ST . getCPU (


<s> <ISA_LIT>Gfx10CacheControl ( const GCNSubtarget & ST , bool CuMode ) : <ISA_LIT>Gfx7CacheControl ( ST ) , CuMode (


<s> <ISA_LIT>Gfx10CacheControl ( const GCNSubtarget & ST , bool CuMode ) : <ISA_LIT>Gfx7CacheControl ( ST ) , CuMode ( CuMode )


<s> <ISA_LIT>MemOpInfo ( AtomicOrdering Ordering = AtomicOrdering :: SequentiallyConsistent , <ISA_LIT>AtomicScope Scope = <STR_LIT> , <ISA_LIT>AtomicAddrSpace OrderingAddrSpace = <STR_LIT> , <ISA_LIT>AtomicAddrSpace InstrAddrSpace = <STR_LIT> , bool IsCrossAddressSpaceOrdering = true , AtomicOrdering FailureOrdering = AtomicOrdering :: SequentiallyConsistent , bool IsNonTemporal = false ) : Ordering ( Ordering ) , FailureOrdering ( FailureOrdering ) , Scope ( Scope ) , OrderingAddrSpace ( OrderingAddrSpace ) , InstrAddrSpace ( InstrAddrSpace ) ,

IsCrossAddressSpaceOrdering ( IsCrossAddressSpaceOrdering ) , IsNonTemporal ( IsNonTemporal ) {


<s> AtomicOrdering FailureOrdering = AtomicOrdering :: NotAtomic ; bool IsNonTemporal = true ; for ( const auto & MMO : MI -> memoperands ( ) ) { const auto & IsSyncScopeInclusion = MMI -> isSyncScopeInclusion ( S<ISA_LIT>D , MMO -> getSyncScopeID ( ) ) ; if ( ! IsSyncScopeInclusion ) { reportUnknownSyncScope ( MI ) ; return None ; } S<ISA_LIT>D = IsSyncScopeInclusion . getValue ( ) ? S<ISA_LIT>D : MMO -> getSyncScopeID ( ) ; Ordering = isStrongerThan ( Ordering , MMO -> getOrdering ( ) ) ? Ordering : MMO -> getOrdering ( ) ; FailureOrdering = isStrongerThan ( FailureOrdering , MMO -> getFailureOrdering ( ) ) ? FailureOrdering : MMO -> getFailureOrdering ( ) ; if ( ! ( MMO -> getFlags ( ) & MachineMemOperand :: MONonTemporal ) ) IsNonTemporal = false ; } return <ISA_LIT>MemOpInfo ( S<ISA_LIT>D , Ordering , FailureOrdering , IsNonTemporal


<s> for ( const auto & MMO : MI -> memoperands ( ) ) { const auto & IsSyncScopeInclusion = MMI -> isSyncScopeInclusion ( S<ISA_LIT>D , MMO -> getSyncScopeID ( ) ) ; if ( ! IsSyncScopeInclusion ) { reportUnknownSyncScope ( MI ) ; return None ; } S<ISA_LIT>D = IsSyncScopeInclusion . getValue ( ) ? S<ISA_LIT>D : MMO -> getSyncScopeID ( ) ; Ordering = isStrongerThan ( Ordering , MMO -> getOrdering ( ) ) ? Ordering : MMO -> getOrdering ( ) ; FailureOrdering = isStrongerThan ( FailureOrdering , MMO -> getFailureOrdering ( ) ) ? FailureOrdering : MMO -> getFailureOrdering ( ) ; if ( ! ( MMO -> getFlags ( ) & MachineMemOperand :: MONonTemporal ) ) IsNonTemporal =


<s> int BitIdx = <STR_LIT> ( MI -> getOpcode ( ) , BitName ) ; if ( BitIdx == - <NUM_LIT>


<s> int BitIdx = <STR_LIT> ( MI -> getOpcode ( ) , BitName ) ; if ( BitIdx == - <NUM_LIT> ) return false ; MachineOperand & Bit = MI -> getOperand ( BitIdx


<s> if ( MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent || MOI . getFailureOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= insertWaitcntVmcnt0 ( MI ) ; if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent || MOI . getFailureOrdering ( ) == AtomicOrdering :: Acquire || MOI . getFailureOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= insertWaitcntVmcnt0 ( MI , false ) ; Changed |= insertBufferWbinvl1Vol ( MI , false


<s> if ( MOI . isAtomic ( ) ) { if ( MOI . getS<ISA_LIT>D ( ) == SyncScope :: System || MOI . getS<ISA_LIT>D ( ) == MMI -> getAgentS<ISA_LIT>D ( ) ) { if ( MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= insertWaitcntVmcnt0 ( MI ) ; return Changed ; } if ( MOI . getS<ISA_LIT>D ( ) == SyncScope :: SingleThread || MOI . getS<ISA_LIT>D ( ) == MMI -> getWorkgroupS<ISA_LIT>D ( ) || MOI . getS<ISA_LIT>D ( ) == MMI -> getWavefrontS<ISA_LIT>D ( ) ) { return Changed ; } llvm_unreachable ( <STR_LIT> ) ; } if ( MOI . isNonTemporal ( ) ) { Changed |= enableGLCBit ( MI ) ; Changed |= enableSLCBit


<s> Optional < <ISA_LIT>MemOpInfo > <ISA_LIT>MemOpInfo :: getAtomicCmpxchgInfo ( const MachineBasicBlock :: iterator & MI ) { assert ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) ; if ( ! ( MI -> mayLoad ( ) && MI -> mayStore ( ) ) ) return None ; if ( MI -> getNumMemOperands ( ) == <NUM_LIT> ) return <ISA_LIT>MemOpInfo ( SyncScope :: System , AtomicOrdering :: SequentiallyConsistent , AtomicOrdering :: SequentiallyConsistent ) ; return <ISA_LIT>MemOpInfo :: constructFromMIWithMMO


<s> if ( MI -> getOpcode ( ) != <STR_LIT> ) return None ; SyncScope :: ID S<ISA_LIT>D = static_cast < SyncScope :: ID > ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ; AtomicOrdering Ordering = static_cast < AtomicOrdering > ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ; return <ISA_LIT>MemOpInfo ( S<ISA_LIT>D


<s> if ( MI -> getNumMemOperands ( ) == <NUM_LIT> ) return <ISA_LIT>MemOpInfo ( SyncScope :: System , AtomicOrdering :: SequentiallyConsistent ) ; return <ISA_LIT>MemOpInfo :: constructFromMIWithMMO


<s> if ( ! ( MI -> mayLoad ( ) && ! MI -> mayStore ( ) ) ) return None ; if ( MI -> getNumMemOperands ( ) == <NUM_LIT> ) return <ISA_LIT>MemOpInfo ( SyncScope :: System , AtomicOrdering :: SequentiallyConsistent ) ; return <ISA_LIT>MemOpInfo :: constructFromMIWithMMO


<s> assert ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) ; if ( ! ( MI -> mayLoad ( ) && ! MI -> mayStore ( )


<s> bool <ISA_LIT>MemoryLegalizer :: insertBufferWbinvl1Vol ( MachineBasicBlock :: iterator & MI , bool Before ) const { MachineBasicBlock & MBB = * MI -> getParent ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; if ( ! Before ) ++ MI ; BuildMI ( MBB , MI , DL , TII -> get ( Wbinvl1Opcode ) ) ; if ( ! Before ) -- MI ; return true


<s> bool <ISA_LIT>MemoryLegalizer :: insertBufferWbinvl1Vol ( MachineBasicBlock :: iterator & MI , bool Before ) const { MachineBasicBlock & MBB = * MI -> getParent ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; if ( ! Before ) ++


<s> if ( ! Before ) ++ MI ; BuildMI ( MBB , MI , DL , TII -> get

( <STR_LIT> ) ) . addImm ( Vmcnt0Immediate ) ;


<s> DiagnosticInfoUnsupported Diag ( * MI -> getParent ( ) -> getParent (


<s> LLVMContext * CTX = & MI -> getParent ( ) -> getParent ( )


<s> if ( ! ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) ) continue ; if ( const auto & MOI = <ISA_LIT>MemOpInfo :: getLoadInfo ( MI ) ) Changed |= expandLoad ( MOI . getValue ( ) , MI ) ; else if ( const auto & MOI = <ISA_LIT>MemOpInfo :: getStoreInfo ( MI ) ) Changed |= expandStore ( MOI . getValue ( ) , MI ) ; else if ( const auto & MOI = <ISA_LIT>MemOpInfo :: getAtomicFenceInfo ( MI ) ) Changed |= expandAtomicFence ( MOI


<s> void <ISA_LIT>MemOpInfo :: reportUnknownSyncScope ( const MachineBasicBlock :: iterator & MI ) { DiagnosticInfoUnsupported Diag ( MI -> getParent ( ) -> getParent (


<s> if ( ! InsertCacheInv ) return false ; bool Changed = false ; if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : case <STR_LIT> : break ; case


<s> std :: unique_ptr < <ISA_LIT>CacheControl > <ISA_LIT>CacheControl :: create ( const GCNSubtarget & ST ) { GCNSubtarget :: Generation Generation = ST . getGeneration ( ) ; if ( Generation <= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return make_unique < <ISA_LIT>Gfx6CacheControl > ( ST ) ; return make_unique < <ISA_LIT>Gfx7CacheControl > ( ST


<s> if ( Generation <= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return make_unique < <ISA_LIT>Gfx6CacheControl


<s> bool enableGLCBit ( const MachineBasicBlock :: iterator & MI ) const { return enableNamedBit < <STR_LIT> :: glc > (


<s> bool enableNamedBit ( const MachineBasicBlock :: iterator & MI ) { int BitIdx = <STR_LIT> ( MI -> getOpcode ( ) ,


<s> bool enableNamedBit ( const MachineBasicBlock :: iterator & MI ) { int BitIdx = <STR_LIT> ( MI -> getOpcode ( ) , BitName ) ; if ( BitIdx == - <NUM_LIT> ) return false ; MachineOperand & Bit = MI


<s> bool <ISA_LIT>Gfx6CacheControl :: enableNonTemporal ( const MachineBasicBlock :: iterator & MI ) const { assert ( MI -> mayLoad ( ) ^ MI -> mayStore ( ) ) ; bool Changed =


<s> bool enableSLCBit ( const MachineBasicBlock :: iterator & MI


<s> bool enableSLCBit ( const MachineBasicBlock :: iterator & MI )


<s> assert ( MI -> mayLoad ( ) && MI -> mayStore ( ) ) ; bool Changed = false ; if ( MOI . isAtomic ( ) ) { if ( MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent || MOI . getFailureOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , <STR_LIT> | <STR_LIT> , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ; if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent || MOI . getFailureOrdering ( ) == AtomicOrdering :: Acquire || MOI . getFailureOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( )


<s> if ( MOI . isAtomic ( ) ) { if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , <STR_LIT> | <STR_LIT> , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ; if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertCacheInvalidate ( MI

, MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , Position :: BEFORE ) ;


<s> if ( MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , <STR_LIT> | <STR_LIT> , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ; if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getInstrAddrSpace ( ) , <STR_LIT> , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: AFTER ) ; Changed |= CC -> insertCacheInvalidate ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace (


<s> if ( MOI . isAtomic ( ) ) { if ( MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , <STR_LIT> | <STR_LIT> , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ; return Changed ; } if ( MOI . isNonTemporal ( ) ) { Changed |= CC ->


<s> Optional < <ISA_LIT>MemOpInfo > <ISA_LIT>MemOpAccess :: getAtomicFenceInfo ( const MachineBasicBlock :: iterator & MI ) const { assert ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) ; if ( MI -> getOpcode ( ) != <STR_LIT> ) return None ; AtomicOrdering Ordering = static_cast < AtomicOrdering > ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ; SyncScope :: ID S<ISA_LIT>D = static_cast < SyncScope :: ID > ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ; auto ScopeOrNone = to<ISA_LIT>AtomicScope ( S<ISA_LIT>D , <STR_LIT> ) ; if ( ! ScopeOrNone ) { reportUnsupported ( MI , <STR_LIT> ) ; return None ; } <ISA_LIT>AtomicScope Scope = <STR_LIT> ; <ISA_LIT>AtomicAddrSpace OrderingAddrSpace = <STR_LIT> ; bool IsCrossAddressSpaceOrdering = false ; std :: tie ( Scope , OrderingAddrSpace , IsCrossAddressSpaceOrdering ) = ScopeOrNone . getValue ( ) ; if ( ( OrderingAddrSpace == <STR_LIT> ) || ( ( OrderingAddrSpace & <STR_LIT> ) != OrderingAddrSpace ) ) { reportUnsupported ( MI


<s> bool Changed = false ; MachineBasicBlock & MBB = * MI -> getParent ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; if ( Pos == Position :: AFTER ) ++ MI ; if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : case <STR_LIT> : BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) ; Changed = true


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : LGKMCnt = IsCrossAddrSpaceOrdering ; break ; case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : case <STR_LIT> : EXPCnt = IsCrossAddrSpaceOrdering ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( VMCnt || LGKMCnt || EXPCnt ) { unsigned WaitCntImmediate = <STR_LIT> ( IV , VMCnt ? <NUM_LIT> : getVmcntBitMask ( IV ) , EXPCnt ? <NUM_LIT> :

getExpcntBitMask ( IV ) , LGKMCnt ? <NUM_LIT> : getLgkmcntBitMask ( IV ) ) ;


<s> bool isAtomicRet ( const MachineInstr & MI ) const { return <STR_LIT> ( MI . getOpcode ( ) ) != -


<s> <ISA_LIT>MemOpAccess :: <ISA_LIT>MemOpAccess ( MachineFunction & MF ) { <ISA_LIT>AddrSpaceInfo = get<ISA_LIT>AS ( MF . getTarget ( ) ) ; MMI = & MF . getMMI ( ) . getObjFileInfo


<s> Optional < std :: tuple < <ISA_LIT>AtomicScope , <ISA_LIT>AtomicAddrSpace , bool >> <ISA_LIT>MemOpAccess :: to<ISA_LIT>AtomicScope ( SyncScope :: ID S<ISA_LIT>D , <ISA_LIT>AtomicAddrSpace InstrScope ) const { if ( S<ISA_LIT>D == SyncScope :: System ) return std :: make_tuple ( <STR_LIT> , <STR_LIT> & InstrScope , false ) ; if ( S<ISA_LIT>D == MMI -> getAgentS<ISA_LIT>D ( ) ) return std :: make_tuple ( <STR_LIT> , <STR_LIT> & InstrScope , false ) ; if ( S<ISA_LIT>D == MMI -> getWorkgroupS<ISA_LIT>D ( ) ) return std :: make_tuple ( <STR_LIT> , <STR_LIT> & InstrScope , false ) ; if ( S<ISA_LIT>D == MMI -> getWavefrontS<ISA_LIT>D ( ) ) return std :: make_tuple ( <STR_LIT> , <STR_LIT> & InstrScope , false ) ; if ( S<ISA_LIT>D == SyncScope :: SingleThread ) return std ::


<s> <ISA_LIT>Subtarget :: Generation Generation = ST . getGeneration ( ) ; if ( Generation <= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return make_unique < <ISA_LIT>Gfx6CacheControl >


<s> <ISA_LIT>CacheControl :: <ISA_LIT>CacheControl ( const <ISA_LIT>Subtarget & ST ) { TII = ST . getInstrInfo ( )


<s> <ISA_LIT>Gfx6CacheControl ( const <ISA_LIT>Subtarget & ST ) : <ISA_LIT>CacheControl ( ST )


<s> <ISA_LIT>Gfx6CacheControl ( const <ISA_LIT>Subtarget & ST )


<s> <ISA_LIT>Gfx7CacheControl ( const <ISA_LIT>Subtarget & ST ) : <ISA_LIT>Gfx6CacheControl ( ST )


<s> <ISA_LIT>Gfx7CacheControl ( const <ISA_LIT>Subtarget & ST ) :


<s> bool Changed = false ; if ( MOI . isAtomic ( ) ) { if ( MOI . getS<ISA_LIT>D ( ) == SyncScope :: System || MOI . getS<ISA_LIT>D ( ) == MMI -> getAgentS<ISA_LIT>D ( ) ) { if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= insertWaitcntVmcnt0 ( MI ) ; if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= insertVmem<ISA_LIT>MDCacheInvalidate ( MI ) ; AtomicPseudoMIs . push_back ( MI ) ; return Changed


<s> Changed |= insertWaitcntVmcnt0 ( MI , false ) ; Changed |= insertVmem<ISA_LIT>MDCacheInvalidate ( MI , false ) ; } return Changed ; } if ( MOI . getS<ISA_LIT>D ( ) == SyncScope :: SingleThread || MOI . getS<ISA_LIT>D ( ) == MMI -> getWorkgroupS<ISA_LIT>D ( ) || MOI . getS<ISA_LIT>D ( ) == MMI -> getWavefrontS<ISA_LIT>D ( ) ) { return Changed ; } llvm_unreachable ( <STR_LIT> ) ; } if ( MOI . isNonTemporal ( ) ) { Changed |= enableGLCBit ( MI ) ; Changed |= enableSLCBit ( MI ) ; return Changed ; } return Changed


<s> if ( MOI . getS<ISA_LIT>D ( ) == SyncScope :: System || MOI . getS<ISA_LIT>D ( ) == MMI -> getAgentS<ISA_LIT>D ( ) ) { if ( MOI . getOrdering ( ) == AtomicOrdering :: Monotonic || MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= enableGLCBit ( MI ) ; if ( MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= insertWaitcntVmcnt0 ( MI ) ; if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= insertWaitcntVmcnt0 ( MI , false ) ; Changed |= insertVmem<ISA_LIT>MDCacheInvalidate ( MI , false ) ; } return Changed ; } if ( MOI . getS<ISA_LIT>D ( ) == SyncScope :: SingleThread || MOI . getS<ISA_LIT>D ( ) == MMI -> getWorkgroupS<ISA_LIT>D ( ) || MOI . getS<ISA_LIT>D ( ) == MMI -> getWavefrontS<ISA_LIT>D ( ) ) { return Changed ; } llvm_unreachable ( <STR_LIT> ) ; } if ( MOI . isNonTemporal ( ) ) { Changed |= enableGLCBit ( MI ) ; Changed |= enableSLCBit ( MI ) ; return Changed ; } return Changed


<s> Optional < <ISA_LIT>MemOpInfo > <ISA_LIT>MemOpInfo :: getAtomicCmpxchgOrRmwInfo ( const MachineBasicBlock :: iterator & MI ) { assert ( MI -> getDesc ( ) .


<s> bool <ISA_LIT>MemoryLegalizer :: insertVmem<ISA_LIT>MDCacheInvalidate ( MachineBasicBlock :: iterator & MI , bool Before ) const { MachineBasicBlock & MBB = * MI -> getParent (


<s> for ( auto & MBB : MF ) { for ( auto MI = MBB . begin ( ) ; MI != MBB . end ( ) ; ++ MI ) { if ( ! ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) ) continue ; if ( const auto & MOI = <ISA_LIT>MemOpInfo :: getLoadInfo ( MI ) ) Changed |= expandLoad ( MOI . getValue ( ) , MI ) ; else if ( const auto & MOI = <ISA_LIT>MemOpInfo :: getStoreInfo ( MI ) ) Changed |= expandStore ( MOI . getValue ( ) , MI ) ; else if ( const auto & MOI = <ISA_LIT>MemOpInfo :: getAtomicFenceInfo ( MI ) ) Changed |= expandAtomicFence ( MOI . getValue ( ) , MI ) ; else if ( const auto & MOI = <ISA_LIT>MemOpInfo :: getAtomicCmpxchgOrRmwInfo ( MI ) ) Changed |= expandAtomicCmpxchgOrRmw ( MOI . getValue ( ) , MI ) ; } } Changed |= removeAtomicPseudoMIs ( ) ; return Changed


<s> IV = getIsaVersion ( ST . getCPU ( )


<s> GCNSubtarget :: Generation Generation = ST . getGeneration ( ) ; if ( Generation <= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return make_unique < <ISA_LIT>Gfx6CacheControl > ( ST ) ; if ( Generation < <ISA_LIT>Subtarget :: GFX10 ) return

make_unique < <ISA_LIT>Gfx7CacheControl > ( ST ) ;


<s> bool Changed = false ; if ( MOI . isAtomic ( ) ) { if ( MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertRelease ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ; return Changed ; } if ( MOI . isNonTemporal


<s> case <STR_LIT> : case <STR_LIT> : LGKMCnt |= IsCrossAddrSpaceOrdering ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( VMCnt || LGKMCnt ) { unsigned WaitCntImmediate = <STR_LIT> ( IV , VMCnt ? <NUM_LIT> : getVmcntBitMask ( IV ) , getExpcntBitMask ( IV ) , LGKMCnt ? <NUM_LIT> : getLgkmcntBitMask ( IV ) ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( WaitCntImmediate ) ; Changed = true ; } if ( VSCnt ) { BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) ; Changed = true ; } if ( Pos == Position :: AFTER )


<s> <ISA_LIT>CacheControl :: <ISA_LIT>CacheControl ( const GCNSubtarget & ST ) { TII = ST . getInstrInfo ( ) ; IV = getIsaVersion ( ST . getCPU


<s> assert ( MI -> mayLoad ( ) ^ MI -> mayStore ( ) ) ; assert ( Op == <STR_LIT> || Op == <STR_LIT> ) ; bool Changed = false ; if ( IsVolatile ) { if ( Op == <STR_LIT> ) { Changed |= enableGLCBit (


<s> reportUnsupported ( MI , <STR_LIT> ) ; return None ; } S<ISA_LIT>D = IsSyncScopeInclusion . getValue ( ) ? S<ISA_LIT>D : MMO -> getSyncScopeID ( ) ; Ordering = getMergedAtomicOrdering ( Ordering , OpOrdering ) ; assert ( MMO -> getFailureOrdering ( ) != AtomicOrdering :: Release && MMO -> getFailureOrdering ( ) != AtomicOrdering :: AcquireRelease ) ; FailureOrdering = getMergedAtomicOrdering ( FailureOrdering , MMO -> getFailureOrdering ( ) ) ; } } <ISA_LIT>AtomicScope Scope = <STR_LIT> ; <ISA_LIT>AtomicAddrSpace OrderingAddrSpace = <STR_LIT> ; bool IsCrossAddressSpaceOrdering = false ; if ( Ordering != AtomicOrdering :: NotAtomic ) { auto ScopeOrNone = to<ISA_LIT>AtomicScope ( S<ISA_LIT>D , InstrAddrSpace ) ; if ( ! ScopeOrNone ) { reportUnsupported ( MI , <STR_LIT> ) ; return None ; } std :: tie ( Scope , OrderingAddrSpace , IsCrossAddressSpaceOrdering


<s> bool IsCrossAddressSpaceOrdering = false ; if ( Ordering != AtomicOrdering :: NotAtomic ) { auto ScopeOrNone = to<ISA_LIT>AtomicScope ( S<ISA_LIT>D , InstrAddrSpace ) ; if ( ! ScopeOrNone ) { reportUnsupported ( MI , <STR_LIT> ) ; return None ; } std :: tie ( Scope , OrderingAddrSpace , IsCrossAddressSpaceOrdering ) = ScopeOrNone . getValue ( ) ; if ( ( OrderingAddrSpace == <STR_LIT> ) || ( ( OrderingAddrSpace & <STR_LIT> ) != OrderingAddrSpace ) || ( ( InstrAddrSpace & <STR_LIT> ) == <STR_LIT> ) ) { reportUnsupported ( MI , <STR_LIT> ) ; return None ; } } return <ISA_LIT>MemOpInfo ( Ordering , Scope , OrderingAddrSpace , InstrAddrSpace , IsCrossAddressSpaceOrdering , FailureOrdering ,


<s> if ( Generation < <ISA_LIT>Subtarget :: GFX10 ) return std :: make_unique < <ISA_LIT>Gfx7CacheControl > ( ST ) ; return std :: make_unique < <ISA_LIT>Gfx10CacheControl >


<s> FunctionPass * llvm :: create<ISA_LIT>MemoryLegalizerPass


<s> return new <ISA_LIT>MemoryLegalizer ( )


<s> bool enableDLCBit ( const MachineBasicBlock ::


<s> if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : case <STR_LIT> : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> )


<s> return enableNamedBit ( MI , <STR_LIT> ::


<s> } if ( MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent || MOI . getFailureOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertRelease ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ; if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent || MOI . getFailureOrdering ( ) == AtomicOrdering :: Acquire || MOI . getFailureOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getInstrAddrSpace ( ) , isAtomicRet ( * MI ) ? <STR_LIT> : <STR_LIT> , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: AFTER ) ; Changed |= CC -> insertAcquire ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace (


<s> bool Changed = false ; if ( MOI . isAtomic ( ) ) { if ( MOI . getOrdering ( ) == AtomicOrdering :: Monotonic || MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= CC -> enableRMWCacheBypass ( MI , MOI . getScope ( ) , MOI . getInstrAddrSpace ( ) ) ; } if ( MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent || MOI . getFailureOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertRelease ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ; if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent || MOI . getFailureOrdering ( ) == AtomicOrdering :: Acquire || MOI . getFailureOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getInstrAddrSpace

( ) , isAtomicRet ( * MI ) ? <STR_LIT> :


<s> bool <ISA_LIT>MemoryLegalizer :: expandAtomicFence ( const <ISA_LIT>MemOpInfo & MOI , MachineBasicBlock :: iterator & MI ) { assert ( MI -> getOpcode ( ) == <STR_LIT> ) ; AtomicPseudoMIs . push_back ( MI ) ; bool Changed = false ; if ( MOI . isAtomic ( ) ) { if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertRelease ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , MOI .

getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ;


<s> if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getInstrAddrSpace ( ) , <STR_LIT> , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: AFTER ) ; Changed |= CC -> insertAcquire ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , Position :: AFTER ) ; } return Changed ; } Changed |= CC -> enableVolatileAndOrNonTemporal ( MI , MOI . getInstrAddrSpace ( ) , <STR_LIT> , MOI

. isVolatile ( ) , MOI . isNonTemporal ( ) ) ;


<s> if ( MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertRelease ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ; return Changed ; } Changed |= CC -> enableVolatileAndOrNonTemporal ( MI , MOI . getInstrAddrSpace ( ) , <STR_LIT> , MOI .

isVolatile ( ) , MOI . isNonTemporal ( ) ) ;


<s> bool Changed = false ; if ( MOI . isAtomic ( ) ) { if ( MOI . getOrdering ( ) == AtomicOrdering :: Monotonic || MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= CC -> enableStoreCacheBypass ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) ) ; } if ( MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertRelease ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ; return Changed


<s> Optional < <ISA_LIT>MemOpInfo > <ISA_LIT>MemOpAccess :: getAtomicCmpxchgOrRmwInfo ( const MachineBasicBlock :: iterator & MI ) const { assert ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) ; if ( ! ( MI -> mayLoad ( ) && MI -> mayStore ( ) ) ) return None ; if ( MI -> getNumMemOperands ( ) == <NUM_LIT> ) return <ISA_LIT>MemOpInfo ( ) ; return constructFromMIWithMMO (


<s> if ( ! ( MI -> mayLoad ( ) && ! MI -> mayStore ( ) ) ) return None ; if ( MI -> getNumMemOperands ( ) == <NUM_LIT>


<s> AtomicOrdering getOrdering ( ) const { return


<s> <ISA_LIT>AtomicAddrSpace getOrderingAddrSpace (


<s> LGKMCnt |= IsCrossAddrSpaceOrdering ; break ; case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : case <STR_LIT> : LGKMCnt |= IsCrossAddrSpaceOrdering ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( VMCnt || LGKMCnt ) { unsigned WaitCntImmediate = <STR_LIT> ( IV , VMCnt ? <NUM_LIT> : getVmcntBitMask ( IV ) , getExpcntBitMask ( IV ) , LGKMCnt ? <NUM_LIT> : getLgkmcntBitMask ( IV ) ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( WaitCntImmediate ) ; Changed = true ; } if ( VSCnt ) { BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addReg ( <STR_LIT>

, RegState :: Undef ) . addImm ( <NUM_LIT> ) ;


<s> case <STR_LIT> : if ( ! ST . isCuModeEnabled ( ) ) { if ( ( Op & <STR_LIT> ) != <STR_LIT> ) VMCnt |= true ; if ( ( Op & <STR_LIT> ) != <STR_LIT> ) VSCnt |= true ; } break ; case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : LGKMCnt |= IsCrossAddrSpaceOrdering ; break ; case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : case <STR_LIT> : LGKMCnt |= IsCrossAddrSpaceOrdering ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( VMCnt || LGKMCnt ) { unsigned WaitCntImmediate = <STR_LIT> ( IV , VMCnt ? <NUM_LIT> : getVmcntBitMask ( IV ) , getExpcntBitMask ( IV ) , LGKMCnt ? <NUM_LIT> : getLgkmcntBitMask ( IV ) ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( WaitCntImmediate ) ; Changed = true ; } if ( VSCnt ) { BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) .

addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) ;


<s> bool isAtomicRet ( const MachineInstr & MI ) const { return <ISA_LIT>InstrInfo :: isAtomicRet (


<s> return <ISA_LIT>InstrInfo :: isAtomicRet ( MI


<s> bool isNonTemporal ( ) const { return IsNonTemporal


<s> const Function & Func = MI -> getParent ( ) -> getParent ( ) -> getFunction ( ) ; DiagnosticInfoUnsupported Diag ( Func , Msg , MI -> getDebugLoc


<s> void <ISA_LIT>MemOpAccess :: reportUnsupported ( const MachineBasicBlock :: iterator & MI , const char * Msg ) const { const Function & Func = MI -> getParent ( ) -> getParent ( ) -> getFunction ( ) ; DiagnosticInfoUnsupported Diag ( Func , Msg , MI -> getDebugLoc ( ) ) ; Func . getContext ( ) . diagnose


<s> <ISA_LIT>MemOpAccess MOA ( MF ) ; CC = <ISA_LIT>CacheControl :: create ( MF . getSubtarget < GCNSubtarget > ( ) ) ; for ( auto & MBB : MF ) { for ( auto MI = MBB . begin ( ) ; MI != MBB . end ( ) ; ++ MI ) { if ( MI -> isBundle ( ) ) { MachineBasicBlock :: instr_iterator II ( MI -> getIterator ( ) ) ; for ( MachineBasicBlock :: instr_iterator I = ++ II , E = MBB . instr_end ( ) ; I != E && I -> isBundledWithPred ( ) ; ++ I ) { I -> unbundleFromPred ( ) ; for ( MachineOperand & MO : I -> operands ( ) ) if ( MO . isReg ( ) ) MO . setIsInternalRead ( false ) ; } MI -> eraseFromParent ( ) ; MI = II -> getIterator ( ) ; } if ( ! ( MI -> getDesc ( ) . TSFlags & <STR_LIT> )


<s> <ISA_LIT>CacheControl :: <ISA_LIT>CacheControl ( const GCNSubtarget & ST ) : ST ( ST ) { TII = ST . getInstrInfo ( ) ; IV = getIsaVersion ( ST . getCPU ( ) ) ; InsertCacheInv =


<s> <ISA_LIT>Gfx10CacheControl ( const GCNSubtarget & ST ) : <ISA_LIT>Gfx7CacheControl ( ST )


<s> <ISA_LIT>Gfx10CacheControl ( const GCNSubtarget & ST ) : <ISA_LIT>Gfx7CacheControl


<s> <ISA_LIT>Gfx6CacheControl ( const GCNSubtarget & ST )


<s> <ISA_LIT>Gfx6CacheControl ( const GCNSubtarget & ST ) : <ISA_LIT>CacheControl


<s> <ISA_LIT>Gfx7CacheControl ( const GCNSubtarget & ST ) :


<s> <ISA_LIT>Gfx90ACacheControl ( const GCNSubtarget & ST ) : <ISA_LIT>Gfx7CacheControl ( ST )


<s> <ISA_LIT>Gfx90ACacheControl ( const GCNSubtarget & ST ) :


<s> <ISA_LIT>MemOpAccess :: <ISA_LIT>MemOpAccess ( MachineFunction &


<s> assert ( Scope != <STR_LIT> && ( OrderingAddrSpace & <STR_LIT> ) != <STR_LIT> && ( InstrAddrSpace & <STR_LIT> ) != <STR_LIT> ) ; if ( ( OrderingAddrSpace == InstrAddrSpace ) && isPowerOf2_32 ( uint32_t ( InstrAddrSpace ) ) ) this -> IsCrossAddressSpaceOrdering = false ; if ( ( InstrAddrSpace & ~ <STR_LIT> ) == <STR_LIT> ) { this -> Scope = std :: min ( Scope , <STR_LIT> ) ; } else if ( ( InstrAddrSpace & ~ ( <STR_LIT> | <STR_LIT> ) ) == <STR_LIT> ) { this -> Scope = std :: min ( Scope ,


<s> unsigned Width = countTrailingOnes < unsigned > ( InstrMode . Mask >> Offset ) ; unsigned Value = ( InstrMode . Mode >> Offset ) & ( ( <NUM_LIT> << Width ) - <NUM_LIT> ) ; BuildMI ( MBB , MI , nullptr , TII -> get ( <STR_LIT> ) ) . addImm ( Value ) . addImm ( ( ( Width - <NUM_LIT> ) << <STR_LIT> :: WIDTH_M1_SHIFT_ ) | ( Offset << <STR_LIT> :: OFFSET_SHIFT_ ) | ( <STR_LIT> :: ID_MODE << <STR_LIT> :: ID_SHIFT_ ) ) ; ++ NumSetregInserted


<s> BlockData ( ) : FirstInsertionPoint ( nullptr )


<s> unsigned Width = countTrailingOnes < unsigned > ( InstrMode . Mask >> Offset ) ; unsigned Value = ( InstrMode . Mode >> Offset ) & ( ( <NUM_LIT> << Width ) - <NUM_LIT> ) ; BuildMI ( MBB , MI , <NUM_LIT> , TII -> get ( <STR_LIT> ) ) . addImm ( Value ) . addImm ( ( ( Width - <NUM_LIT> ) << <STR_LIT> :: WIDTH_M1_SHIFT_ ) | ( Offset << <STR_LIT> :: OFFSET_SHIFT_ ) | ( <STR_LIT> :: ID_MODE << <STR_LIT> :: ID_SHIFT_ ) ) ; ++ NumSetregInserted


<s> void <ISA_LIT>ModeRegister :: insertSetreg ( MachineBasicBlock & MBB , MachineInstr * MI , const <ISA_LIT>InstrInfo * TII , Status InstrMode ) { while ( InstrMode . Mask ) { unsigned Offset = countTrailingZeros < unsigned > ( InstrMode . Mask ) ; unsigned Width = countTrailingOnes < unsigned > (


<s> } else if ( BlockInfo [ PredBlock ] -> ExitSet ) { BlockInfo [ ThisBlock ] -> Pred = BlockInfo [ PredBlock ] -> Exit ; ExitSet = true ; } else if ( PredBlock != ThisBlock ) RevisitRequired = true ; for ( P = std :: next ( P ) ; P != E ; P = std :: next ( P ) ) { MachineBasicBlock * Pred = * P ; unsigned PredBlock = Pred -> getNumber ( ) ; if ( BlockInfo [ PredBlock ] -> ExitSet ) { if ( BlockInfo [ ThisBlock ] -> ExitSet ) { BlockInfo [ ThisBlock ] -> Pred = BlockInfo [ ThisBlock ] -> Pred . intersect ( BlockInfo [ PredBlock ] -> Exit ) ; } else { BlockInfo [ ThisBlock ] -> Pred = BlockInfo [ PredBlock ] -> Exit ; } ExitSet = true ; } else if ( PredBlock != ThisBlock ) RevisitRequired = true ; } } Status TmpStatus = BlockInfo [ ThisBlock ] -> Pred . merge ( BlockInfo [ ThisBlock ] -> Change ) ; if ( BlockInfo [ ThisBlock ] -> Exit != TmpStatus


<s> bool <ISA_LIT>ModeRegister :: runOnMachineFunction ( MachineFunction & MF ) { BlockInfo . resize ( MF . getNumBlockIDs ( ) ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; for ( MachineBasicBlock & BB : MF ) processBlockPhase1 ( BB , TII ) ; for ( MachineBasicBlock & BB : MF ) Phase2List . push


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; for ( MachineBasicBlock & BB : MF ) processBlockPhase1 ( BB , TII ) ; for ( MachineBasicBlock & BB : MF ) Phase2List . push ( & BB ) ; while ( ! Phase2List . empty ( ) ) { processBlockPhase2 ( * Phase2List . front ( ) , TII ) ; Phase2List . pop ( ) ; } for ( MachineBasicBlock & BB : MF ) processBlockPhase3 ( BB


<s> Status ( unsigned NewMask , unsigned NewMode ) : Mask


<s> return Status ( FP_ROUND_MODE_DP ( <NUM_LIT> ) , FP_ROUND_MODE_DP ( FP_ROUND_ROUND_TO_INF ) ) ; } case <STR_LIT> : { MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; return Status ( FP_ROUND_MODE_DP ( <NUM_LIT> ) , FP_ROUND_MODE_DP ( FP_ROUND_ROUND_TO_NEGINF ) ) ; } default : return DefaultStatus ; } } return Status (


<s> BlockData ( ) : FirstInsertionPoint ( nullptr


<s> Status delta ( const Status & S ) const { return Status ( ( S . Mask & ( Mode ^ S . Mode )

) | ( ~ Mask & S . Mask ) , S . Mode ) ;


<s> return Status ( ( S . Mask & ( Mode ^ S . Mode ) ) | ( ~ Mask &


<s> if ( TII -> usesFPDPRounding ( MI ) ) { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case


<s> case <STR_LIT> : case <STR_LIT> : return Status ( FP_ROUND_MODE_DP ( <NUM_LIT> ) , FP_ROUND_MODE_DP ( FP_ROUND_ROUND_TO_ZERO ) ) ; default : return DefaultStatus ; } } return Status ( )


<s> Status intersect ( const Status & S ) const { unsigned NewMask = ( Mask & S . Mask ) & (


<s> return ( ( Mask & S . Mask ) == S . Mask ) && ( ( Mode & S


<s> while ( ! Phase2List . empty ( ) ) { processBlockPhase2 ( * Phase2List . front ( ) , TII ) ; Phase2List . pop ( ) ; } for ( MachineBasicBlock & BB : MF ) processBlockPhase3 ( BB , TII ) ; BlockInfo . clear ( ) ; return NumSetregInserted >


<s> Status ( unsigned Mask , unsigned Mode ) : Mask ( Mask ) , Mode ( Mode ) { Mode &=


<s> for ( ++ A ; CurrentIteration < MaxInstructions && A != E ; ++ A ) { if ( A -> isDebugInstr ( ) ) continue ; if ( Pred ( & * A ) ) return & * A ; for ( MCRegister Reg : NonModifiableRegs ) { if ( A -> modifiesRegister ( Reg , TRI ) ) return nullptr ; } ++ CurrentIteration ; } return


<s> Register SaveExecDest = SaveExec . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRReg ( MRI , SaveExecDest ) ) return nullptr ; MachineOperand * SaveExecSrc0 = TII -> getNamedOperand ( SaveExec , <STR_LIT> :: src0 ) ; if ( ! SaveExecSrc0 -> isReg ( ) ) return nullptr ; VCmp = findInstrBackwards ( SaveExec , [ & ] ( MachineInstr * Check ) { return <STR_LIT> ( Check -> getOpcode ( ) ) != - <NUM_LIT> && Check -> modifiesRegister ( SaveExecSrc0 -> getReg ( ) , TRI ) ; } , { Exec , SaveExecSrc0 -> getReg ( ) } , TRI ) ; if ( ! VCmp ) return nullptr ; MachineOperand * VCmpDest = TII -> getNamedOperand ( * VCmp , <STR_LIT> :: sdst ) ; assert ( VCmpDest && <STR_LIT> ) ; MachineOperand * Src0 = TII -> getNamedOperand ( * VCmp , <STR_LIT> :: src0 ) ; if ( Src0 -> isReg ( ) && TRI -> isSGPRReg ( MRI , Src0 -> getReg ( ) ) && SaveExec . modifiesRegister

( Src0 -> getReg ( ) , TRI ) ) return nullptr ;


<s> MachineOperand * VCmpDest = TII -> getNamedOperand ( * VCmp , <STR_LIT> :: sdst ) ; assert ( VCmpDest && <STR_LIT> ) ; MachineOperand * Src0 = TII -> getNamedOperand ( * VCmp , <STR_LIT> :: src0 ) ; if ( Src0 -> isReg ( ) && TRI -> isSGPRReg ( MRI , Src0 -> getReg ( ) ) && SaveExec . modifiesRegister ( Src0 -> getReg ( ) , TRI ) ) return nullptr ; MachineOperand * Src1 = TII -> getNamedOperand ( * VCmp , <STR_LIT> :: src1 ) ; if ( Src1 -> isReg ( ) && TRI -> isSGPRReg ( MRI , Src1 -> getReg ( ) ) && SaveExec . modifiesRegister ( Src1 -> getReg ( ) , TRI ) ) return nullptr ; if ( isLiveOut ( * VCmp -> getParent ( ) , VCmpDest ->


<s> static bool isRegisterInUseAfter ( MachineInstr & Stop , MCRegister Reg , const <ISA_LIT>RegisterInfo


<s> static bool isRegisterInUseAfter ( MachineInstr & Stop , MCRegister Reg , const <ISA_LIT>RegisterInfo * TRI , MachineRegisterInfo


<s> static MachineBasicBlock :: reverse_iterator findExecCopy ( const <ISA_LIT>InstrInfo & TII , MachineBasicBlock & MBB , MachineBasicBlock :: reverse_iterator I , unsigned CopyToExec ) { const unsigned InstLimit = <NUM_LIT>


<s> for ( ; I != E ; ++ I ) { if ( ! I -> isTerminator ( ) ) return I ; if ( removeTerminatorBit ( TII , * I ) ) return I ; } return E


<s> MachineBasicBlock :: reverse_iterator I = MBB . rbegin ( ) , E = MBB . rend ( ) ; for ( ; I != E ; ++ I ) { if ( ! I -> isTerminator ( ) ) return I ; if ( removeTerminatorBit ( TII , *


<s> static unsigned getSaveExecOp ( unsigned Opc ) { switch ( Opc ) { case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT>


<s> static unsigned isCopyFromExec ( const MachineInstr & MI ) { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const MachineOperand & Src = MI .


<s> const MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; if ( Dst . isReg ( ) && Dst . getReg ( ) == <STR_LIT> && MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; break ; } case <STR_LIT>


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const MachineOperand & Src1 = MI . getOperand ( <NUM_LIT> ) ; if ( Src1 . isReg ( ) && Src1 . getReg ( ) == <STR_LIT> ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const MachineOperand & Src2 = MI . getOperand ( <NUM_LIT> ) ; if ( Src2 . isReg ( ) && Src2 . getReg ( ) == <STR_LIT>


<s> case <STR_LIT> : { MI . setDesc ( TII . get ( <STR_LIT> ) ) ; return true ; } case <STR_LIT> : { MI . setDesc ( TII . get ( <STR_LIT> ) ) ; return true


<s> case <STR_LIT> : case <STR_LIT> : { const MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; if ( Dst . isReg ( ) && Dst . getReg ( ) == <STR_LIT> ) return

MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: reverse_iterator I = fixTerminators ( * TII , MBB ) ; MachineBasicBlock :: reverse_iterator E = MBB . rend ( ) ; if ( I == E ) continue ; unsigned CopyToExec = isCopyToExec ( * I ) ; if ( CopyToExec == <STR_LIT> ) continue ; auto CopyToExecInst = & * I ; auto CopyFromExecInst = findExecCopy ( * TII , MBB , I , CopyToExec ) ; if ( CopyFromExecInst == E ) { auto PrepareExecInst = std :: next ( I ) ; if ( PrepareExecInst == E ) continue ; if ( CopyToExecInst -> getOperand ( <NUM_LIT> ) . isKill ( ) && isLogicalOpOnExec ( * PrepareExecInst ) == CopyToExec ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * PrepareExecInst ) ; PrepareExecInst -> getOperand ( <NUM_LIT> ) . setReg ( <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * PrepareExecInst << '\n' ) ; CopyToExecInst -> eraseFromParent ( ) ; } continue ; } if ( isLiveOut ( MBB , CopyToExec ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ; } unsigned CopyFromExec = CopyFromExecInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * SaveExecInst = nullptr ; SmallVector < MachineInstr * , <NUM_LIT> > OtherUseInsts ; for ( MachineBasicBlock :: iterator J = std :: next ( CopyFromExecInst -> getIterator ( ) ) , JE = I -> getIterator ( ) ; J != JE ; ++ J ) { if ( SaveExecInst && J -> readsRegister ( <STR_LIT> , TRI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; SaveExecInst = nullptr ; break ; } bool ReadsCopyFromExec = J -> readsRegister ( CopyFromExec , TRI ) ; if ( J -> modifiesRegister ( CopyToExec , TRI ) ) { if ( SaveExecInst ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( CopyToExec , TRI ) << '\n' ) ; SaveExecInst = nullptr ; break ; } unsigned SaveExecOp = getSaveExecOp ( J -> getOpcode ( ) ) ; if ( SaveExecOp == <STR_LIT> ) break ; if ( ReadsCopyFromExec


<s> static bool removeTerminatorBit ( const <ISA_LIT>InstrInfo & TII , MachineInstr & MI ) { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : { bool RegSrc = MI . getOperand ( <NUM_LIT> ) . isReg ( ) ; MI . setDesc ( TII . get ( RegSrc ? <STR_LIT> : <STR_LIT> ) ) ; return true ; } case <STR_LIT> : { bool RegSrc = MI . getOperand ( <NUM_LIT> ) . isReg ( ) ; MI . setDesc ( TII . get ( RegSrc ? <STR_LIT> : <STR_LIT> ) ) ; return true ; } case <STR_LIT> : { MI . setDesc ( TII . get ( <STR_LIT> ) ) ; return true ; } case <STR_LIT> : { MI . setDesc ( TII . get ( <STR_LIT> ) ) ; return true ; } case <STR_LIT> : { MI . setDesc ( TII .


<s> Register CopyFromExec = CopyFromExecInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * SaveExecInst = nullptr ; SmallVector < MachineInstr * , <NUM_LIT> > OtherUseInsts ; for ( MachineBasicBlock :: iterator J = std :: next ( CopyFromExecInst -> getIterator ( ) ) , JE = I -> getIterator ( ) ; J != JE ; ++ J ) { if ( SaveExecInst && J -> readsRegister ( Exec , TRI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; SaveExecInst = nullptr ; break ; } bool ReadsCopyFromExec = J -> readsRegister ( CopyFromExec , TRI ) ; if ( J -> modifiesRegister ( CopyToExec , TRI ) ) { if ( SaveExecInst ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( CopyToExec , TRI ) << '\n' ) ; SaveExecInst = nullptr ; break ; } unsigned SaveExecOp = getSaveExecOp ( J -> getOpcode ( ) ) ; if ( SaveExecOp == <STR_LIT> ) break ; if ( ReadsCopyFromExec ) { SaveExecInst = & * J ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; continue ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } } else if ( ReadsCopyFromExec && ! SaveExecInst ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } if ( SaveExecInst && J -> readsRegister ( CopyToExec , TRI ) ) { assert ( SaveExecInst != & * J ) ; OtherUseInsts . push_back ( & * J ) ; } } if ( ! SaveExecInst ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; MachineOperand & Src0 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand * OtherOp = nullptr ; if ( Src0 . isReg ( ) && Src0 . getReg ( ) == CopyFromExec ) { OtherOp = & Src1 ; } else if ( Src1 . isReg ( ) && Src1 . getReg ( ) == CopyFromExec ) { if ( ! SaveExecInst -> isCommutable ( ) ) break ; OtherOp = & Src0 ; } else llvm_unreachable ( <STR_LIT> ) ; CopyFromExecInst -> eraseFromParent ( ) ; auto InsPt = SaveExecInst -> getIterator ( ) ; const DebugLoc & DL = SaveExecInst -> getDebugLoc ( ) ; BuildMI ( MBB , InsPt , DL , TII -> get ( getSaveExecOp ( SaveExecInst -> getOpcode ( ) ) ) , CopyFromExec ) . addReg ( OtherOp -> getReg ( ) ) ; SaveExecInst -> eraseFromParent ( ) ; CopyToExecInst -> eraseFromParent ( ) ; for ( MachineInstr * OtherInst : OtherUseInsts ) { OtherInst -> substituteRegister ( CopyToExec , Exec


<s> if ( J -> readsRegister ( CopyFromExec , TRI ) ) { SaveExecInst = & * J ; DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; continue ; } else { DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } } if ( SaveExecInst && J -> readsRegister ( CopyToExec , TRI ) ) { assert ( SaveExecInst != & * J ) ; OtherUseInsts . push_back ( & * J ) ; } } if ( ! SaveExecInst ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; MachineOperand & Src0 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand * OtherOp = nullptr ; if ( Src0 . isReg ( ) && Src0 . getReg ( ) == CopyFromExec ) { OtherOp = & Src1 ; } else if ( Src1 . isReg ( ) && Src1 . getReg ( ) == CopyFromExec ) { if ( ! SaveExecInst -> isCommutable ( ) ) break ; OtherOp = & Src0 ; } else llvm_unreachable ( <STR_LIT> ) ; CopyFromExecInst -> eraseFromParent ( ) ; auto InsPt = SaveExecInst -> getIterator ( ) ; const DebugLoc & DL = SaveExecInst -> getDebugLoc ( ) ; BuildMI ( MBB , InsPt , DL , TII -> get ( getSaveExecOp ( SaveExecInst -> getOpcode ( ) ) ) , CopyFromExec ) . addReg ( OtherOp -> getReg ( ) ) ; SaveExecInst -> eraseFromParent (


<s> break ; } unsigned SaveExecOp = getSaveExecOp ( J -> getOpcode ( ) ) ; if ( SaveExecOp == <STR_LIT> ) break ; if ( ReadsCopyFromExec ) { SaveExecInst = & * J ; DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; continue ; } else { DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } } else if ( ReadsCopyFromExec && ! SaveExecInst ) { DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } if ( SaveExecInst && J -> readsRegister ( CopyToExec , TRI ) ) { assert ( SaveExecInst != & * J ) ; OtherUseInsts . push_back ( & * J ) ; } } if ( ! SaveExecInst ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; MachineOperand & Src0 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand * OtherOp = nullptr ; if ( Src0 . isReg ( ) && Src0 . getReg ( ) == CopyFromExec ) { OtherOp = & Src1 ; } else if ( Src1 . isReg ( ) && Src1 . getReg ( ) == CopyFromExec ) { if ( ! SaveExecInst -> isCommutable ( ) ) break ; OtherOp = & Src0 ; } else llvm_unreachable ( <STR_LIT> ) ; CopyFromExecInst -> eraseFromParent ( ) ; auto InsPt = SaveExecInst -> getIterator ( ) ; const DebugLoc & DL = SaveExecInst -> getDebugLoc ( ) ; BuildMI ( MBB , InsPt , DL , TII -> get ( getSaveExecOp ( SaveExecInst -> getOpcode ( ) ) ) , CopyFromExec ) . addReg ( OtherOp -> getReg ( ) ) ; SaveExecInst -> eraseFromParent


<s> DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; continue ; } else { DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } } else if ( ReadsCopyFromExec && ! SaveExecInst ) { DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } if ( SaveExecInst && J -> readsRegister ( CopyToExec , TRI ) ) { assert ( SaveExecInst != & * J ) ; OtherUseInsts . push_back ( & * J ) ; } } if ( ! SaveExecInst ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; MachineOperand & Src0 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand * OtherOp = nullptr ; if ( Src0 . isReg ( ) && Src0 . getReg ( ) == CopyFromExec ) { OtherOp = & Src1 ; } else if ( Src1 . isReg ( ) && Src1 . getReg ( ) == CopyFromExec ) { if ( ! SaveExecInst -> isCommutable ( ) ) break ; OtherOp = & Src0 ; } else llvm_unreachable ( <STR_LIT> ) ; CopyFromExecInst -> eraseFromParent ( ) ; auto InsPt = SaveExecInst -> getIterator ( ) ; const DebugLoc & DL = SaveExecInst -> getDebugLoc ( ) ; BuildMI ( MBB , InsPt , DL , TII -> get ( getSaveExecOp ( SaveExecInst -> getOpcode ( ) ) ) , CopyFromExec ) . addReg ( OtherOp -> getReg ( ) ) ; SaveExecInst -> eraseFromParent ( ) ; CopyToExecInst -> eraseFromParent ( ) ; for ( MachineInstr * OtherInst : OtherUseInsts ) { OtherInst -> substituteRegister ( CopyToExec , <STR_LIT> , <STR_LIT>


<s> static MachineBasicBlock :: reverse_iterator findExecCopy ( const <ISA_LIT>InstrInfo & TII , const GCNSubtarget & ST , MachineBasicBlock & MBB , MachineBasicBlock :: reverse_iterator I , unsigned CopyToExec ) { const unsigned InstLimit = <NUM_LIT> ; auto E = MBB . rend ( ) ; for ( unsigned N = <NUM_LIT> ; N <= InstLimit && I != E ; ++ I , ++ N ) { unsigned CopyFromExec = isCopyFromExec ( * I , ST


<s> const MachineOperand & Src1 = MI . getOperand ( <NUM_LIT> ) ; if ( Src1 . isReg ( ) && Src1 . getReg ( ) == <STR_LIT> ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const MachineOperand & Src2 = MI . getOperand ( <NUM_LIT> ) ; if ( Src2 . isReg ( ) && Src2 . getReg ( ) == <STR_LIT> ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; break ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const MachineOperand & Src1 = MI . getOperand ( <NUM_LIT> ) ; if ( Src1 . isReg ( ) && Src1 . getReg ( ) == <STR_LIT> ) return MI . getOperand (


<s> SmallVector < MachineInstr * , <NUM_LIT> > OtherUseInsts ; for ( MachineBasicBlock :: iterator J = std :: next ( CopyFromExecInst -> getIterator ( ) ) , JE = I -> getIterator ( ) ; J != JE ; ++ J ) { if ( SaveExecInst && J -> readsRegister ( Exec , TRI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; SaveExecInst = nullptr ; break ; } bool ReadsCopyFromExec = J -> readsRegister ( CopyFromExec , TRI ) ; if ( J -> modifiesRegister ( CopyToExec , TRI ) ) { if ( SaveExecInst ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( CopyToExec , TRI ) << '\n' ) ; SaveExecInst = nullptr ; break ; } unsigned SaveExecOp = getSaveExecOp ( J -> getOpcode ( ) ) ; if ( SaveExecOp == <STR_LIT> ) break ; if ( ReadsCopyFromExec ) { SaveExecInst = & * J ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; continue ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } } else if ( ReadsCopyFromExec && ! SaveExecInst ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } if ( SaveExecInst && J -> readsRegister ( CopyToExec , TRI ) ) { assert ( SaveExecInst != & * J ) ; OtherUseInsts . push_back ( & * J ) ; } } if ( ! SaveExecInst ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; MachineOperand & Src0 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand * OtherOp = nullptr ; if ( Src0 . isReg ( ) && Src0 . getReg ( ) == CopyFromExec ) { OtherOp = & Src1 ; } else if ( Src1 . isReg ( ) && Src1 . getReg ( ) == CopyFromExec ) { if ( ! SaveExecInst -> isCommutable ( ) ) break ; OtherOp = & Src0


<s> if ( ReadsCopyFromExec ) { SaveExecInst = & * J ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; continue ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } } else if ( ReadsCopyFromExec && ! SaveExecInst ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } if ( SaveExecInst && J -> readsRegister ( CopyToExec , TRI ) ) { assert ( SaveExecInst != & * J ) ; OtherUseInsts . push_back ( & * J ) ; } } if ( ! SaveExecInst ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; MachineOperand & Src0 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand * OtherOp = nullptr ; if ( Src0 . isReg ( ) && Src0 . getReg ( ) == CopyFromExec ) { OtherOp = & Src1 ; } else if ( Src1 . isReg ( ) && Src1 . getReg ( ) == CopyFromExec ) { if ( ! SaveExecInst -> isCommutable ( ) ) break ; OtherOp = & Src0 ; } else llvm_unreachable ( <STR_LIT> ) ; CopyFromExecInst -> eraseFromParent ( ) ; auto InsPt = SaveExecInst -> getIterator ( ) ; const DebugLoc & DL = SaveExecInst -> getDebugLoc ( ) ; BuildMI ( MBB , InsPt , DL , TII -> get ( getSaveExecOp ( SaveExecInst -> getOpcode ( ) ) ) , CopyFromExec ) . addReg ( OtherOp -> getReg ( ) ) ; SaveExecInst -> eraseFromParent ( ) ; CopyToExecInst -> eraseFromParent ( ) ; for ( MachineInstr * OtherInst : OtherUseInsts ) { OtherInst -> substituteRegister ( CopyToExec , Exec , <STR_LIT> , * TRI ) ; } } return


<s> } unsigned CopyFromExec = CopyFromExecInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * SaveExecInst = nullptr ; SmallVector < MachineInstr * , <NUM_LIT> > OtherUseInsts ; for ( MachineBasicBlock :: iterator J = std :: next ( CopyFromExecInst -> getIterator ( ) ) , JE = I -> getIterator ( ) ; J != JE ; ++ J ) { if ( SaveExecInst && J -> readsRegister ( <STR_LIT> , TRI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; SaveExecInst = nullptr ; break ; } bool ReadsCopyFromExec = J -> readsRegister ( CopyFromExec , TRI ) ; if ( J -> modifiesRegister ( CopyToExec , TRI ) ) { if ( SaveExecInst ) { DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( CopyToExec , TRI ) << '\n' ) ; SaveExecInst = nullptr ; break ; } unsigned SaveExecOp = getSaveExecOp ( J -> getOpcode ( ) ) ; if ( SaveExecOp == <STR_LIT> ) break ; if ( ReadsCopyFromExec ) { SaveExecInst = & * J ; DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; continue ; } else { DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } } else if ( ReadsCopyFromExec && ! SaveExecInst ) { DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } if ( SaveExecInst && J -> readsRegister ( CopyToExec , TRI ) ) { assert ( SaveExecInst != & * J ) ; OtherUseInsts . push_back ( & * J ) ; } } if ( ! SaveExecInst ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; MachineOperand & Src0 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand * OtherOp = nullptr ; if ( Src0 . isReg ( ) && Src0 . getReg ( ) == CopyFromExec ) { OtherOp = & Src1 ; } else if ( Src1 . isReg ( ) && Src1 . getReg ( ) == CopyFromExec ) { if ( ! SaveExecInst -> isCommutable ( ) ) break ; OtherOp = & Src0 ; } else llvm_unreachable ( <STR_LIT> ) ; CopyFromExecInst -> eraseFromParent ( ) ; auto InsPt = SaveExecInst -> getIterator ( ) ; const DebugLoc & DL = SaveExecInst -> getDebugLoc


<s> J != JE ; ++ J ) { if ( SaveExecInst && J -> readsRegister ( <STR_LIT> , TRI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; SaveExecInst = nullptr ; break ; } bool ReadsCopyFromExec = J -> readsRegister ( CopyFromExec , TRI ) ; if ( J -> modifiesRegister ( CopyToExec , TRI ) ) { if ( SaveExecInst ) { DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( CopyToExec , TRI ) << '\n' ) ; SaveExecInst = nullptr ; break ; } unsigned SaveExecOp = getSaveExecOp ( J -> getOpcode ( ) ) ; if ( SaveExecOp == <STR_LIT> ) break ; if ( ReadsCopyFromExec ) { SaveExecInst = & * J ; DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; continue ; } else { DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } } else if ( ReadsCopyFromExec && ! SaveExecInst ) { DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } if ( SaveExecInst && J -> readsRegister ( CopyToExec , TRI ) ) { assert ( SaveExecInst != & * J ) ; OtherUseInsts . push_back ( & * J ) ; } } if ( ! SaveExecInst ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; MachineOperand & Src0 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand * OtherOp = nullptr ; if ( Src0 . isReg ( ) && Src0 . getReg ( ) == CopyFromExec ) { OtherOp = & Src1 ; } else if ( Src1 . isReg ( ) && Src1 . getReg ( ) == CopyFromExec ) { if ( ! SaveExecInst -> isCommutable ( ) ) break ; OtherOp = & Src0 ; } else llvm_unreachable ( <STR_LIT> ) ; CopyFromExecInst -> eraseFromParent ( ) ; auto InsPt = SaveExecInst -> getIterator ( ) ; const DebugLoc & DL = SaveExecInst -> getDebugLoc ( ) ; BuildMI ( MBB , InsPt , DL , TII -> get ( getSaveExecOp ( SaveExecInst -> getOpcode ( ) ) ) , CopyFromExec ) . addReg ( OtherOp -> getReg ( ) ) ; SaveExecInst -> eraseFromParent ( ) ; CopyToExecInst -> eraseFromParent ( ) ; for ( MachineInstr * OtherInst : OtherUseInsts ) { OtherInst -> substituteRegister ( CopyToExec , <STR_LIT> , <STR_LIT> , * TRI ,


<s> static bool isFullExecCopy ( const MachineInstr & MI , const GCNSubtarget & ST ) { unsigned Exec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; if ( MI . isFullCopy ( ) && MI . getOperand ( <NUM_LIT> )


<s> auto * Cmp = TRI -> findReachingDef ( CmpReg , CmpSubReg , * And , MRI , LIS ) ; if ( ! Cmp || ! ( Cmp -> getOpcode ( ) == <STR_LIT> || Cmp -> getOpcode ( ) == <STR_LIT> ) || Cmp -> getParent ( ) != And -> getParent ( ) ) return <STR_LIT> ; MachineOperand * Op1 = TII -> getNamedOperand ( * Cmp , <STR_LIT> :: src0 ) ; MachineOperand * Op2 = TII -> getNamedOperand ( * Cmp , <STR_LIT> :: src1 ) ; if ( Op1 -> isImm ( ) && Op2 -> isReg ( ) ) std :: swap ( Op1 , Op2 ) ; if ( ! Op1 -> isReg ( ) || ! Op2 -> isImm ( ) || Op2 -> getImm ( ) != <NUM_LIT> ) return <STR_LIT> ; Register SelReg = Op1 -> getReg ( ) ; auto * Sel = TRI -> findReachingDef ( SelReg , Op1 -> getSubReg ( ) , * Cmp , MRI , LIS ) ; if ( ! Sel || Sel -> getOpcode ( ) != <STR_LIT> ) return <STR_LIT> ; if ( TII -> hasModifiersSet ( * Sel , <STR_LIT> :: src0_modifiers ) || TII -> hasModifiersSet ( * Sel , <STR_LIT> :: src1_modifiers ) ) return <STR_LIT> ; Op1 = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src0 ) ; Op2 = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src1 ) ; MachineOperand * CC = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src2 ) ; if ( ! Op1 -> isImm ( ) || ! Op2 -> isImm ( ) || ! CC -> isReg ( ) || Op1 -> getImm ( ) != <NUM_LIT> || Op2 -> getImm ( ) != <NUM_LIT> ) return <STR_LIT> ; Register CCReg = CC -> getReg ( ) ; if ( isDefBetween ( * TRI , LIS , CCReg , * Sel , * And ) ) return <STR_LIT> ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Sel << '\t' << * Cmp << '\t' << * And ) ; LIS -> RemoveMachineInstrFromMaps ( * And


<s> MachineOperand * Op2 = TII -> getNamedOperand ( * Cmp , <STR_LIT> :: src1 ) ; if ( Op1 -> isImm ( ) && Op2 -> isReg ( ) ) std :: swap ( Op1 , Op2 ) ; if ( ! Op1 -> isReg ( ) || ! Op2 -> isImm ( ) || Op2 -> getImm ( ) != <NUM_LIT> ) return <STR_LIT> ; Register SelReg = Op1 -> getReg ( ) ; auto * Sel = TRI -> findReachingDef ( SelReg , Op1 -> getSubReg ( ) , * Cmp , MRI , LIS ) ; if ( ! Sel || Sel -> getOpcode ( ) != <STR_LIT> ) return <STR_LIT> ; if ( TII -> hasModifiersSet ( * Sel , <STR_LIT> :: src0_modifiers ) || TII -> hasModifiersSet ( * Sel , <STR_LIT> :: src1_modifiers ) ) return <STR_LIT> ; Op1 = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src0 ) ; Op2 = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src1 ) ; MachineOperand * CC = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src2 ) ; if ( ! Op1 -> isImm ( ) || ! Op2 -> isImm ( ) || ! CC -> isReg ( ) || Op1 -> getImm ( ) != <NUM_LIT> || Op2 -> getImm ( ) != <NUM_LIT> ) return <STR_LIT> ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Sel << '\t' << * Cmp << '\t' << * And ) ; Register CCReg = CC -> getReg ( ) ; LIS -> RemoveMachineInstrFromMaps ( * And ) ; MachineInstr * Andn2 = BuildMI ( MBB , * And , And -> getDebugLoc ( ) , TII -> get ( Andn2Opc ) , And -> getOperand ( <NUM_LIT> ) . getReg ( ) ) . addReg ( ExecReg ) . addReg ( CCReg , getUndefRegState ( CC -> isUndef ( ) ) , CC -> getSubReg ( ) ) ; And -> eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps ( * Andn2 ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Andn2 << '\n' ) ; if ( ( Register :: isVirtualRegister ( CmpReg ) && MRI . use_nodbg_empty ( CmpReg ) ) || ( CmpReg == CondReg && std :: none_of ( std :: next ( Cmp -> getIterator ( ) ) , Andn2 -> getIterator ( ) , [ & ] ( const MachineInstr & MI ) { return MI . readsRegister ( CondReg , TRI ) ; } ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Cmp << '\n' ) ; LIS -> RemoveMachineInstrFromMaps (


<s> if ( SavedExec == <STR_LIT> ) return nullptr ; auto SaveExecInst = MRI . getUniqueVRegDef ( SavedExec ) ; if ( ! SaveExecInst || ! isFullExecCopy ( * SaveExecInst , ST ) ) return nullptr ; return SaveExecInst


<s> auto Op = TII . getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Op -> isReg ( ) && Op -> getReg ( ) != Exec ) return Op -> getReg ( ) ; Op = TII . getNamedOperand ( MI , <STR_LIT> :: src0 ) ; if ( Op -> isReg ( ) && Op -> getReg ( ) != Exec ) return Op -> getReg ( ) ; return <STR_LIT>


<s> static unsigned getOrNonExecReg ( const MachineInstr & MI , const <ISA_LIT>InstrInfo & TII , const GCNSubtarget & ST ) { unsigned Exec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; auto Op = TII . getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Op -> isReg ( ) && Op -> getReg ( ) != Exec ) return Op -> getReg ( ) ; Op = TII . getNamedOperand ( MI , <STR_LIT> :: src0 ) ; if ( Op -> isReg ( ) && Op -> getReg ( ) != Exec ) return Op -> getReg ( ) ; return <STR_LIT>


<s> static bool isEndCF ( const MachineInstr & MI , const <ISA_LIT>RegisterInfo * TRI , const GCNSubtarget & ST ) { if ( ST . isWave32 ( ) ) { return MI . getOpcode ( ) == <STR_LIT> && MI . modifiesRegister ( <STR_LIT> , TRI ) ; } return MI . getOpcode ( ) == <STR_LIT> && MI . modifiesRegister ( <STR_LIT>


<s> if ( I != E ) { if ( I -> isUnconditionalBranch ( ) || I -> getOpcode ( ) == <STR_LIT> ) ++ I ; else if ( I -> isBranch ( ) ) continue ; } while ( I != E ) { if ( I -> isDebugInstr ( ) ) { I = std :: next ( I ) ; continue ; } if ( I -> mayStore ( ) || I -> isBarrier ( ) || I -> isCall ( ) || I -> hasUnmodeledSideEffects ( ) || I -> hasOrderedMemoryRef ( ) ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * I << '\n' ) ; for ( auto & Op : I -> operands ( ) ) { if ( Op . isReg ( ) ) RecalcRegs . insert ( Op . getReg ( ) ) ; } auto Next = std :: next ( I ) ; LIS -> RemoveMachineInstrFromMaps ( * I ) ; I -> eraseFromParent ( ) ; I = Next ; Changed = true ; } if ( I != E ) continue ; for ( auto * Pred : CurBB -> predecessors ( ) ) { if ( Pred -> succ_size ( ) == <NUM_LIT> ) Blocks . push_back ( Pred ) ; } } continue ; } auto E = MBB . end ( ) ; auto Lead = skipDebugInstructionsForward ( MBB . begin ( ) , E ) ; if ( MBB . succ_size ( ) != <NUM_LIT> || Lead == E || ! isEndCF ( * Lead , TRI , ST ) ) continue ; MachineBasicBlock * TmpMBB = &


<s> for ( ; I != E ; ++ I ) { if ( TII -> mayReadEXEC ( * MRI , * I ) ) break ; } return I


<s> MachineBasicBlock :: iterator <ISA_LIT>OptimizeExecMaskingPreRA :: skipIgnoreExecInsts ( MachineBasicBlock :: iterator I , MachineBasicBlock :: iterator E ) const { for ( ; I != E ; ++ I ) { if ( TII -> mayReadEXEC ( * MRI , *


<s> if ( It != MBB -> end ( ) || MBB -> succ_size ( ) != <NUM_LIT> ) break ; MachineBasicBlock * Succ = * MBB


<s> if ( SavedExec == <STR_LIT> ) return nullptr ; auto SaveExecInst = MRI . getUniqueVRegDef ( SavedExec ) ; if ( ! SaveExecInst || ! isFullExecCopy ( * SaveExecInst ) )


<s> Op = TII . getNamedOperand ( MI , <STR_LIT> :: src0 ) ; if ( Op -> isReg ( ) && Op -> getReg ( ) != <STR_LIT> ) return Op -> getReg ( ) ; return


<s> static bool isEndCF ( const MachineInstr & MI , const <ISA_LIT>RegisterInfo


<s> return MI . getOpcode ( ) == <STR_LIT> && MI . modifiesRegister


<s> return MI . isFullCopy ( ) && MI . getOperand (


<s> return MI . isFullCopy ( ) && MI . getOperand ( <NUM_LIT> ) . getReg (