<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit32Encoding ( static_cast < uint32_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit64Encoding ( static_cast < uint64_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit16IntEncoding ( static_cast < uint16_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit16Encoding ( static_cast < uint16_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : { if ( ! isUInt < <NUM_LIT> > ( Imm ) && STI . getFeatureBits ( ) [ <STR_LIT> ] ) return getLit32Encoding ( static_cast < uint32_t > ( Imm ) , STI ) ; if ( OpInfo . OperandType == <STR_LIT> ) return getLit16Encoding ( static_cast < uint16_t > ( Imm ) , STI ) ; LLVM_FALLTHROUGH ; } case <STR_LIT> : case <STR_LIT> : return getLit16IntEncoding ( static_cast < uint16_t > ( Imm ) , STI ) ; case <STR_LIT> :


<s> return new <ISA_LIT>MCCodeEmitter ( MCII , Ctx ) ;


<s> if ( MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) ) Enc |= <NUM_LIT> ;


<s> <ISA_LIT>MCCodeEmitter ( const MCInstrInfo & mcii , MCContext & ctx ) : <ISA_LIT>MCCodeEmitter ( mcii ) , MRI ( * ctx . getRegisterInfo ( ) ) {


<s> unsigned NumExtraAddrs = srsrc - vaddr0 - <NUM_LIT> ; unsigned NumPadding = ( - NumExtraAddrs ) & <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i < NumExtraAddrs ; ++ i ) OS . write ( ( uint8_t ) getMachineOpValue ( MI , MI . getOperand ( vaddr0 + <NUM_LIT> + i ) , Fixups , STI ) ) ; for ( unsigned i = <NUM_LIT> ; i < NumPadding ; ++ i ) OS . write ( <NUM_LIT> ) ; } if ( ( bytes > <NUM_LIT> && STI . getFeatureBits ( ) [ <STR_LIT> ] ) || ( bytes > <NUM_LIT> && ! STI . getFeatureBits ( ) [ <STR_LIT> ] ) ) return ; for ( unsigned i = <NUM_LIT> , e = Desc . getNumOperands ( ) ; i < e ; ++ i ) { if ( ! <STR_LIT> ( Desc , i ) ) continue ; const MCOperand & Op = MI . getOperand ( i ) ; if ( getLitEncoding ( Op , Desc . OpInfo [ i ] , STI ) != <NUM_LIT> ) continue ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit32Encoding ( static_cast < uint32_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit64Encoding ( static_cast < uint64_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit16IntEncoding ( static_cast < uint16_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit16Encoding ( static_cast < uint16_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : { if ( ! isUInt < <NUM_LIT> > ( Imm ) && STI . getFeatureBits ( ) [ <STR_LIT> ] ) return getLit32Encoding ( static_cast < uint32_t > ( Imm ) , STI ) ; if ( OpInfo . OperandType == <STR_LIT> ) return getLit16Encoding ( static_cast < uint16_t > ( Imm ) , STI ) ; LLVM_FALLTHROUGH ; } case <STR_LIT> : case <STR_LIT> : return getLit16IntEncoding ( static_cast < uint16_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : { uint16_t Lo16 = static_cast < uint16_t > ( Imm ) ; uint32_t Encoding = getLit16Encoding ( Lo16 , STI ) ;


<s> unsigned OpNo = <NUM_LIT> ; for ( unsigned e = MI . getNumOperands ( ) ; OpNo < e ; ++ OpNo ) { if ( & MO == & MI . getOperand ( OpNo ) ) break ; } const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; if ( isSrcOperand ( Desc , OpNo ) ) { int RCID = Desc . OpInfo [ OpNo ] . RegClass ; const MCRegisterClass & RC = MRI . getRegClass ( RCID ) ; uint32_t Enc = getLitEncoding ( MO , RC . getSize ( ) ) ; if ( Enc != ~ <NUM_LIT> && ( Enc != <NUM_LIT> || Desc . getSize ( ) == <NUM_LIT> ) ) return Enc ;

} else if ( MO . isImm ( ) ) return MO . getImm ( ) ;


<s> if ( & MO == & MI . getOperand ( OpNo ) ) break ; } const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; if ( isSrcOperand ( Desc , OpNo ) ) { int RCID = Desc . OpInfo [ OpNo ] . RegClass ; const MCRegisterClass & RC = MRI . getRegClass ( RCID ) ; uint32_t Enc = getLitEncoding ( MO , RC . getSize ( ) ) ; if ( Enc != ~ <NUM_LIT> && ( Enc != <NUM_LIT> || Desc . getSize ( ) == <NUM_LIT> ) ) return Enc ; } else if ( MO . isImm ( ) ) return MO . getImm ( ) ;


<s> <ISA_LIT>MCCodeEmitter ( const MCInstrInfo & mcii , const MCRegisterInfo & mri , MCContext & ctx ) : MCII ( mcii ) , MRI ( mri ) , Ctx ( ctx ) {


<s> unsigned <ISA_LIT>MCCodeEmitter :: getAVOperandEncoding ( const MCInst & MI , unsigned OpNo , SmallVectorImpl < MCFixup > & Fixups , const MCSubtargetInfo & STI ) const { unsigned Reg = MI . getOperand ( OpNo ) . getReg ( ) ; uint64_t Enc = MRI . getEncodingValue ( Reg ) ;

if ( MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) ) Enc |= <NUM_LIT> ;


<s> unsigned <ISA_LIT>MCCodeEmitter :: getAVOperandEncoding ( const MCInst & MI , unsigned OpNo , SmallVectorImpl < MCFixup > & Fixups , const MCSubtargetInfo & STI ) const { unsigned Reg = MI . getOperand ( OpNo ) . getReg ( ) ; uint64_t Enc = MRI . getEncodingValue ( Reg ) ; if ( MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) ) Enc |= <NUM_LIT> ; return Enc ;


<s> uint64_t Encoding = getBinaryCodeForInstr ( MI , Fixups , STI ) ; const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; unsigned bytes = Desc . getSize ( ) ; for ( unsigned i = <NUM_LIT> ; i < bytes ; i ++ ) { OS . write ( ( uint8_t ) ( ( Encoding >> ( <NUM_LIT> * i ) ) & <NUM_LIT> ) ) ; } if ( bytes > <NUM_LIT> ) return ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i < e ; ++ i ) { if ( ! <STR_LIT> ( Desc , i ) ) continue ; const MCOperand & Op = MI . getOperand ( i ) ; if ( getLitEncoding ( Op , Desc . OpInfo [ i ] , STI ) != <NUM_LIT> ) continue ; int64_t Imm = <NUM_LIT> ; if ( Op . isImm ( ) ) Imm = Op . getImm ( ) ; else if ( Op . isExpr ( ) ) { if ( const auto * C = dyn_cast < MCConstantExpr > ( Op . getExpr ( ) ) ) Imm = C -> getValue ( ) ; } else if ( ! Op . isExpr ( ) ) llvm_unreachable ( <STR_LIT> ) ; for ( unsigned j = <NUM_LIT> ; j < <NUM_LIT> ; j ++ ) { OS . write ( ( uint8_t ) ( ( Imm >> ( <NUM_LIT> * j ) ) & <NUM_LIT> ) ) ; } break ; }


<s> unsigned bytes = Desc . getSize ( ) ; for ( unsigned i = <NUM_LIT> ; i < bytes ; i ++ ) { OS . write ( ( uint8_t ) ( ( Encoding >> ( <NUM_LIT> * i ) ) & <NUM_LIT> ) ) ; } if ( bytes > <NUM_LIT> ) return ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i < e ; ++ i ) { if ( ! <STR_LIT> ( Desc , i ) ) continue ;

const MCOperand & Op = MI . getOperand ( i ) ;


<s> uint64_t <ISA_LIT>MCCodeEmitter :: getMachineOpValue ( const MCInst & MI , const MCOperand & MO , SmallVectorImpl < MCFixup > & Fixups , const MCSubtargetInfo & STI ) const { if ( MO . isReg ( ) ) return MRI . getEncodingValue ( MO . getReg ( ) ) ; if ( MO . isExpr ( ) && MO . getExpr ( ) -> getKind ( ) != MCExpr :: Constant ) { const auto * Expr = dyn_cast < MCSymbolRefExpr > ( MO . getExpr ( ) ) ; MCFixupKind Kind ; if ( Expr && Expr -> getSymbol ( ) . isExternal ( ) ) Kind = FK_Data_4 ; else Kind = FK_PCRel_4 ; Fixups . push_back ( MCFixup :: create ( <NUM_LIT> , MO . getExpr ( ) , Kind , MI . getLoc ( ) ) ) ; } unsigned OpNo = <NUM_LIT> ; for ( unsigned e = MI . getNumOperands ( ) ; OpNo < e ; ++ OpNo ) { if ( & MO == & MI . getOperand ( OpNo ) ) break ;


<s> } unsigned OpNo = <NUM_LIT> ; for ( unsigned e = MI . getNumOperands ( ) ; OpNo < e ; ++ OpNo ) { if ( & MO == & MI . getOperand ( OpNo ) ) break ; } const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; if ( <STR_LIT> ( Desc , OpNo ) ) { uint32_t Enc = getLitEncoding ( MO , Desc . OpInfo [ OpNo ] , STI ) ; if ( Enc != ~ <NUM_LIT> && ( Enc != <NUM_LIT> || Desc . getSize ( ) == <NUM_LIT> ) ) return Enc ; } else if ( MO . isImm ( ) ) return MO . getImm ( ) ; llvm_unreachable ( <STR_LIT> ) ;


<s> const MCOperand & MO = MI . getOperand ( OpNo ) ; unsigned Reg = MO . getReg ( ) ; RegEnc |= MRI . getEncodingValue ( Reg ) ; RegEnc &= SDWA9EncValues :: SRC_VGPR_MASK ; if ( <STR_LIT> ( <STR_LIT> ( Reg ) , & MRI ) ) { RegEnc |= SDWA9EncValues :: SRC_SGPR_MASK ; }


<s> return getLit16Encoding ( static_cast < uint16_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : if ( ! isUInt < <NUM_LIT> > ( Imm ) && STI . getFeatureBits ( ) [ <STR_LIT> ] ) return getLit32Encoding ( static_cast < uint32_t > ( Imm ) , STI ) ; LLVM_FALLTHROUGH ; case <STR_LIT> : case <STR_LIT> : { uint16_t Lo16 = static_cast < uint16_t > ( Imm ) ; uint32_t Encoding = getLit16Encoding ( Lo16 , STI ) ; return Encoding ; } default :


<s> const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; unsigned bytes = Desc . getSize ( ) ; for ( unsigned i = <NUM_LIT> ; i < bytes ; i ++ ) { OS . write ( ( uint8_t ) ( ( Encoding >> ( <NUM_LIT> * i ) ) & <NUM_LIT> ) ) ; } if ( bytes > <NUM_LIT> ) return ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i < e ; ++ i ) { if ( ! isSrcOperand ( Desc , i ) ) continue ; int RCID = Desc . OpInfo [ i ] . RegClass ; const MCRegisterClass & RC = MRI . getRegClass ( RCID ) ; const MCOperand & Op = MI . getOperand ( i ) ; if ( getLitEncoding ( Op , RC . getSize ( ) ) != <NUM_LIT> ) continue ; int64_t Imm = <NUM_LIT> ; if ( Op . isImm ( ) ) Imm = Op . getImm ( ) ; else if ( Op . isExpr ( ) ) { if ( const MCConstantExpr * C = dyn_cast < MCConstantExpr > ( Op . getExpr ( ) ) ) Imm = C -> getValue ( ) ;

} else if ( ! Op . isExpr ( ) ) llvm_unreachable ( <STR_LIT> ) ;


<s> uint32_t <ISA_LIT>MCCodeEmitter :: getLitEncoding ( const MCOperand & MO , unsigned OpSize ) const { int64_t Imm ; if ( MO . isExpr ( ) ) { const MCConstantExpr * C = dyn_cast < MCConstantExpr > ( MO . getExpr ( ) ) ; if ( ! C ) return <NUM_LIT> ; Imm = C -> getValue ( ) ; } else { assert ( ! MO . isFPImm ( ) ) ; if ( ! MO . isImm ( ) ) return ~ <NUM_LIT> ; Imm = MO . getImm ( ) ; } if ( OpSize == <NUM_LIT> ) return getLit32Encoding ( static_cast < uint32_t > ( Imm ) ) ; assert ( OpSize == <NUM_LIT> ) ;

return getLit64Encoding ( static_cast < uint64_t > ( Imm ) ) ;


<s> } const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; if ( isSrcOperand ( Desc , OpNo ) ) { int RCID = Desc . OpInfo [ OpNo ] . RegClass ; const MCRegisterClass & RC = MRI . getRegClass ( RCID ) ; uint32_t Enc = getLitEncoding ( MO , RC . getSize ( ) ) ; if ( Enc != ~ <NUM_LIT> && ( Enc != <NUM_LIT> || Desc . getSize ( ) == <NUM_LIT> ) ) return Enc ; } else if ( MO . isImm ( ) ) return MO . getImm ( ) ;


<s> case <NUM_LIT> : return getLit32Encoding ( static_cast < uint32_t > ( Imm ) , STI ) ; case <NUM_LIT> : return getLit64Encoding ( static_cast < uint64_t > ( Imm ) , STI ) ; case <NUM_LIT> : return getLit16Encoding ( static_cast < uint16_t > ( Imm ) , STI ) ; default :


<s> uint64_t RegEnc = <NUM_LIT> ; const MCOperand & MO = MI . getOperand ( OpNo ) ; unsigned Reg = MO . getReg ( ) ; RegEnc |= MRI . getEncodingValue ( Reg ) ; RegEnc &= SDWA9EncValues :: SRC_VGPR_MASK ;

if ( <STR_LIT> ( <STR_LIT> ( Reg ) , & MRI ) ) {


<s> uint64_t RegEnc = <NUM_LIT> ; const MCOperand & MO = MI . getOperand ( OpNo ) ; unsigned Reg = MO . getReg ( ) ; if ( Reg != <STR_LIT> ) { RegEnc |= MRI . getEncodingValue ( Reg ) ; RegEnc &= SDWA9EncValues :: VOPC_DST_SGPR_MASK ; RegEnc |= SDWA9EncValues :: VOPC_DST_VCC_MASK ;


<s> assert ( MMO -> getFailureOrdering ( ) != AtomicOrdering :: Release && MMO -> getFailureOrdering ( ) != AtomicOrdering :: AcquireRelease ) ; FailureOrdering = isStrongerThan ( FailureOrdering , MMO -> getFailureOrdering ( ) ) ? FailureOrdering : MMO -> getFailureOrdering ( ) ; } } <ISA_LIT>AtomicScope Scope = <STR_LIT> ; <ISA_LIT>AtomicAddrSpace OrderingAddrSpace = <STR_LIT> ; bool IsCrossAddressSpaceOrdering = false ; if ( Ordering != AtomicOrdering :: NotAtomic ) { auto ScopeOrNone = to<ISA_LIT>AtomicScope ( S<ISA_LIT>D , InstrAddrSpace ) ; if ( ! ScopeOrNone ) { reportUnsupported ( MI , <STR_LIT> ) ; return None ; } std :: tie ( Scope , OrderingAddrSpace , IsCrossAddressSpaceOrdering ) = ScopeOrNone . getValue ( ) ; if ( ( OrderingAddrSpace == <STR_LIT> ) || ( ( OrderingAddrSpace & <STR_LIT> ) != OrderingAddrSpace ) ) { reportUnsupported ( MI , <STR_LIT> ) ; return None ;


<s> const auto & IsSyncScopeInclusion = MMI -> isSyncScopeInclusion ( S<ISA_LIT>D , MMO -> getSyncScopeID ( ) ) ; if ( ! IsSyncScopeInclusion ) { reportUnsupported ( MI , <STR_LIT> ) ; return None ; } S<ISA_LIT>D = IsSyncScopeInclusion . getValue ( ) ? S<ISA_LIT>D : MMO -> getSyncScopeID ( ) ; Ordering = isStrongerThan ( Ordering , OpOrdering ) ? Ordering : MMO -> getOrdering ( ) ; assert ( MMO -> getFailureOrdering ( ) != AtomicOrdering :: Release && MMO -> getFailureOrdering ( ) != AtomicOrdering :: AcquireRelease ) ; FailureOrdering = isStrongerThan ( FailureOrdering , MMO -> getFailureOrdering ( ) ) ? FailureOrdering : MMO -> getFailureOrdering ( ) ; } } <ISA_LIT>AtomicScope Scope = <STR_LIT> ; <ISA_LIT>AtomicAddrSpace OrderingAddrSpace = <STR_LIT> ; bool IsCrossAddressSpaceOrdering = false ;

if ( Ordering != AtomicOrdering :: NotAtomic ) {


<s> if ( Generation < <ISA_LIT>Subtarget :: GFX10 ) return std :: make_unique < <ISA_LIT>Gfx7CacheControl > ( ST ) ;

return std :: make_unique < <ISA_LIT>Gfx10CacheControl > ( ST ) ;


<s> GCNSubtarget :: Generation Generation = ST . getGeneration ( ) ; if ( Generation <= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return std :: make_unique < <ISA_LIT>Gfx6CacheControl > ( ST ) ; if ( Generation < <ISA_LIT>Subtarget :: GFX10 ) return std :: make_unique < <ISA_LIT>Gfx7CacheControl > ( ST ) ; return std :: make_unique < <ISA_LIT>Gfx10CacheControl > ( ST ) ;


<s> return enableNamedBit < <STR_LIT> :: dlc > ( MI ) ;


<s> if ( MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent || MOI . getFailureOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertRelease ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ; if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent || MOI . getFailureOrdering ( ) == AtomicOrdering :: Acquire || MOI . getFailureOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , isAtomicRet ( * MI ) ? <STR_LIT> :

<STR_LIT> , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: AFTER ) ;


<s> if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent || MOI . getFailureOrdering ( ) == AtomicOrdering :: Acquire || MOI . getFailureOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , isAtomicRet ( * MI ) ? <STR_LIT> : <STR_LIT> , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: AFTER ) ; Changed |= CC -> insertAcquire ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , Position :: AFTER ) ;


<s> if ( MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertRelease ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ; return Changed ; }

Changed |= CC -> enableVolatileAndOrNonTemporal ( MI , MOI . getInstrAddrSpace ( ) , <STR_LIT> , MOI . isVolatile ( ) , MOI . isNonTemporal ( ) ) ;


<s> <ISA_LIT>MemOpInfo ( AtomicOrdering Ordering = AtomicOrdering :: SequentiallyConsistent , <ISA_LIT>AtomicScope Scope = <STR_LIT> , <ISA_LIT>AtomicAddrSpace OrderingAddrSpace = <STR_LIT> , <ISA_LIT>AtomicAddrSpace InstrAddrSpace = <STR_LIT> , bool IsCrossAddressSpaceOrdering = true , AtomicOrdering FailureOrdering = AtomicOrdering :: SequentiallyConsistent , bool IsVolatile = false , bool IsNonTemporal = false ) : Ordering ( Ordering ) , FailureOrdering ( FailureOrdering ) , Scope ( Scope ) , OrderingAddrSpace ( OrderingAddrSpace ) , InstrAddrSpace ( InstrAddrSpace ) , IsCrossAddressSpaceOrdering ( IsCrossAddressSpaceOrdering ) , IsVolatile ( IsVolatile ) , IsNonTemporal ( IsNonTemporal ) {

if ( ( OrderingAddrSpace == InstrAddrSpace ) && isPowerOf2_32 ( uint32_t ( InstrAddrSpace ) ) ) this -> IsCrossAddressSpaceOrdering = false ;


<s> if ( S<ISA_LIT>D == MMI -> getAgentOneAddressSpaceS<ISA_LIT>D ( ) ) return std :: make_tuple ( <STR_LIT> , <STR_LIT> & InstrScope , false ) ; if ( S<ISA_LIT>D == MMI -> getWorkgroupOneAddressSpaceS<ISA_LIT>D ( ) ) return std :: make_tuple ( <STR_LIT> , <STR_LIT> & InstrScope , false ) ; if ( S<ISA_LIT>D == MMI -> getWavefrontOneAddressSpaceS<ISA_LIT>D ( ) ) return std :: make_tuple ( <STR_LIT> , <STR_LIT> & InstrScope , false ) ;


<s> if ( ST . hasGFX940Insts ( ) ) return std :: make_unique < <ISA_LIT>Gfx940CacheControl > ( ST ) ; if ( ST . hasGFX90AInsts ( ) ) return std :: make_unique < <ISA_LIT>Gfx90ACacheControl > ( ST ) ; if ( Generation <= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return std :: make_unique < <ISA_LIT>Gfx6CacheControl > ( ST ) ; if ( Generation < <ISA_LIT>Subtarget :: GFX10 ) return std :: make_unique < <ISA_LIT>Gfx7CacheControl > ( ST ) ;


<s> GCNSubtarget :: Generation Generation = ST . getGeneration ( ) ; if ( ST . hasGFX940Insts ( ) ) return std :: make_unique < <ISA_LIT>Gfx940CacheControl > ( ST ) ; if ( ST . hasGFX90AInsts ( ) ) return std :: make_unique < <ISA_LIT>Gfx90ACacheControl > ( ST ) ;

if ( Generation <= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return std :: make_unique < <ISA_LIT>Gfx6CacheControl > ( ST ) ;


<s> Changed |= enableSC1Bit ( MI ) ; break ; case <STR_LIT> : Changed |= enableSC0Bit ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } }


<s> bool enableNTBit ( const MachineBasicBlock :: iterator & MI ) const { return enableNamedBit ( MI , <STR_LIT> :: NT ) ;


<s> assert ( MI -> mayLoad ( ) && MI -> mayStore ( ) ) ; bool Changed = false ; if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : Changed |= enableSC1Bit ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> return enableNamedBit ( MI , <STR_LIT> :: SC0 ) ;


<s> bool enableSC0Bit ( const MachineBasicBlock :: iterator & MI ) const { return enableNamedBit ( MI , <STR_LIT> :: SC0 ) ;


<s> return enableNamedBit ( MI , <STR_LIT> :: SC1 ) ;


<s> case <STR_LIT> : Changed |= enableSC0Bit ( MI ) ; Changed |= enableSC1Bit ( MI ) ; break ; case <STR_LIT> : Changed |= enableSC1Bit ( MI ) ; break ; case <STR_LIT> : Changed |= enableSC0Bit ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ;


<s> if ( Op == <STR_LIT> ) { Changed |= enableGLCBit ( MI ) ; Changed |= enableDLCBit ( MI ) ; } Changed |= insertWait ( MI , <STR_LIT> , AddrSpace , Op , false , Position :: AFTER ) ; return Changed ; } if ( IsNonTemporal ) {

if ( Op == <STR_LIT> ) Changed |= enableGLCBit ( MI ) ;


<s> if ( Op == <STR_LIT> ) { Changed |= enableGLCBit ( MI ) ; Changed |= enableDLCBit ( MI ) ; } Changed |= insertWait ( MI , <STR_LIT> , AddrSpace , Op , false , Position :: AFTER ) ; return Changed ; } if ( IsNonTemporal ) { if ( Op == <STR_LIT> ) Changed |= enableGLCBit ( MI ) ; Changed |= enableSLCBit ( MI ) ; return Changed ;


<s> switch ( Scope ) { case <STR_LIT> : BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <STR_LIT> :: SC0 | <STR_LIT> :: SC1 ) ; Changed = true ; break ; case <STR_LIT> : BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <STR_LIT> :: SC1 ) ; Changed = true ; break ; case <STR_LIT> : if ( ST . isTgSplitEnabled ( ) ) { BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <STR_LIT> :: SC0 ) ; Changed = true ; } break ; case <STR_LIT> : case <STR_LIT> : break ; default :


<s> if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <STR_LIT> :: SC0 | <STR_LIT> :: SC1 ) ; Changed = true ; break ; case <STR_LIT> : BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <STR_LIT> :: SC1 ) ; Changed = true ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; }


<s> BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <STR_LIT> :: SC1 ) ; Changed = true ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( Pos == Position :: AFTER ) -- MI ; Changed |= insertWait ( MI , Scope , AddrSpace , <STR_LIT> | <STR_LIT> , IsCrossAddrSpaceOrdering , Pos ) ; return Changed ;


<s> if ( MI -> isBundle ( ) && MI -> mayLoadOrStore ( ) ) { MachineBasicBlock :: instr_iterator II ( MI -> getIterator ( ) ) ; for ( MachineBasicBlock :: instr_iterator I = ++ II , E = MBB . instr_end ( ) ; I != E && I -> isBundledWithPred ( ) ; ++ I ) { I -> unbundleFromPred ( ) ; for ( MachineOperand & MO : I -> operands ( ) ) if ( MO . isReg ( ) ) MO . setIsInternalRead ( false ) ; } MI -> eraseFromParent ( ) ; MI = II -> getIterator ( ) ; }

if ( ! ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) ) continue ;


<s> MI = II -> getIterator ( ) ; } if ( ! ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) ) continue ; if ( const auto & MOI = MOA . getLoadInfo ( MI ) ) Changed |= expandLoad ( MOI . getValue ( ) , MI ) ; else if ( const auto & MOI = MOA . getStoreInfo ( MI ) ) Changed |= expandStore ( MOI . getValue ( ) , MI ) ; else if ( const auto & MOI = MOA . getAtomicFenceInfo ( MI ) ) Changed |= expandAtomicFence ( MOI . getValue ( ) , MI ) ; else if ( const auto & MOI = MOA . getAtomicCmpxchgOrRmwInfo ( MI ) ) Changed |= expandAtomicCmpxchgOrRmw ( MOI . getValue ( ) , MI ) ; }


<s> <ISA_LIT>Gfx10CacheControl ( const GCNSubtarget & ST ) : <ISA_LIT>Gfx7CacheControl ( ST ) {


<s> <ISA_LIT>Gfx6CacheControl ( const GCNSubtarget & ST ) : <ISA_LIT>CacheControl ( ST ) {


<s> <ISA_LIT>Gfx7CacheControl ( const GCNSubtarget & ST ) : <ISA_LIT>Gfx6CacheControl ( ST ) {


<s> <ISA_LIT>Gfx90ACacheControl ( const GCNSubtarget & ST ) : <ISA_LIT>Gfx7CacheControl ( ST ) {


<s> std :: unique_ptr < <ISA_LIT>CacheControl > <ISA_LIT>CacheControl :: create ( const GCNSubtarget & ST ) { GCNSubtarget :: Generation Generation = ST . getGeneration ( ) ; if ( Generation <= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return std :: make_unique < <ISA_LIT>Gfx6CacheControl > ( ST ) ;

if ( Generation < <ISA_LIT>Subtarget :: GFX10 ) return std :: make_unique < <ISA_LIT>Gfx7CacheControl > ( ST ) ;


<s> Changed |= enableDLCBit ( MI ) ; break ; case <STR_LIT> : if ( ! CuMode ) Changed |= enableGLCBit ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } return Changed ;


<s> case <STR_LIT> : case <STR_LIT> : Changed |= enableGLCBit ( MI ) ; Changed |= enableDLCBit ( MI ) ; break ; case <STR_LIT> : if ( ! CuMode ) Changed |= enableGLCBit ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } }


<s> case <STR_LIT> : case <STR_LIT> : BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) ; Changed = true ; break ; case <STR_LIT> : if ( ! CuMode ) { BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) ; Changed = true ;


<s> break ; case <STR_LIT> : if ( ! CuMode ) { if ( ( Op & <STR_LIT> ) != <STR_LIT> ) VMCnt |= true ; if ( ( Op & <STR_LIT> ) != <STR_LIT> ) VSCnt |= true ; } break ; case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : LGKMCnt |= IsCrossAddrSpaceOrdering ; break ; case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : case <STR_LIT> : LGKMCnt |= IsCrossAddrSpaceOrdering ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( VMCnt || LGKMCnt ) { unsigned WaitCntImmediate = <STR_LIT> ( IV , VMCnt ? <NUM_LIT> : getVmcntBitMask ( IV ) , getExpcntBitMask ( IV ) , LGKMCnt ? <NUM_LIT> : getLgkmcntBitMask ( IV ) ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( WaitCntImmediate ) ; Changed = true ; } if ( VSCnt ) { BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) ; Changed = true ; } if ( Pos == Position :: AFTER ) -- MI ;


<s> MI = II -> getIterator ( ) ; } if ( ! ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) ) continue ; if ( const auto & MOI = MOA . getLoadInfo ( MI ) ) Changed |= expandLoad ( MOI . getValue ( ) , MI ) ; else if ( const auto & MOI = MOA . getStoreInfo ( MI ) ) Changed |= expandStore ( MOI . getValue ( ) , MI ) ; else if ( const auto & MOI = MOA . getAtomicFenceInfo ( MI ) ) Changed |= expandAtomicFence ( MOI . getValue ( ) , MI ) ; else if ( const auto & MOI = MOA . getAtomicCmpxchgOrRmwInfo ( MI ) ) Changed |= expandAtomicCmpxchgOrRmw ( MOI . getValue ( ) , MI ) ;


<s> <ISA_LIT>CacheControl :: <ISA_LIT>CacheControl ( const GCNSubtarget & ST ) { TII = ST . getInstrInfo ( ) ;

IV = getIsaVersion ( ST . getCPU ( ) ) ;


<s> <ISA_LIT>Gfx10CacheControl ( const GCNSubtarget & ST , bool CuMode ) :

<ISA_LIT>Gfx7CacheControl ( ST ) , CuMode ( CuMode ) { }


<s> AtomicOrdering FailureOrdering = AtomicOrdering :: NotAtomic ; bool IsNonTemporal = true ; for ( const auto & MMO : MI -> memoperands ( ) ) { const auto & IsSyncScopeInclusion = MMI -> isSyncScopeInclusion ( S<ISA_LIT>D , MMO -> getSyncScopeID ( ) ) ; if ( ! IsSyncScopeInclusion ) { reportUnknownSyncScope ( MI ) ; return None ; } S<ISA_LIT>D = IsSyncScopeInclusion . getValue ( ) ? S<ISA_LIT>D : MMO -> getSyncScopeID ( ) ; Ordering = isStrongerThan ( Ordering , MMO -> getOrdering ( ) ) ? Ordering : MMO -> getOrdering ( ) ; FailureOrdering = isStrongerThan ( FailureOrdering , MMO -> getFailureOrdering ( ) ) ? FailureOrdering : MMO -> getFailureOrdering ( ) ; if ( ! ( MMO -> getFlags ( ) & MachineMemOperand :: MONonTemporal ) ) IsNonTemporal = false ; }

return <ISA_LIT>MemOpInfo ( S<ISA_LIT>D , Ordering , FailureOrdering , IsNonTemporal ) ;


<s> for ( const auto & MMO : MI -> memoperands ( ) ) { const auto & IsSyncScopeInclusion = MMI -> isSyncScopeInclusion ( S<ISA_LIT>D , MMO -> getSyncScopeID ( ) ) ; if ( ! IsSyncScopeInclusion ) { reportUnknownSyncScope ( MI ) ; return None ; } S<ISA_LIT>D = IsSyncScopeInclusion . getValue ( ) ? S<ISA_LIT>D : MMO -> getSyncScopeID ( ) ; Ordering = isStrongerThan ( Ordering , MMO -> getOrdering ( ) ) ? Ordering : MMO -> getOrdering ( ) ; FailureOrdering = isStrongerThan ( FailureOrdering , MMO -> getFailureOrdering ( ) ) ? FailureOrdering : MMO -> getFailureOrdering ( ) ;

if ( ! ( MMO -> getFlags ( ) & MachineMemOperand :: MONonTemporal ) ) IsNonTemporal = false ;


<s> int BitIdx = <STR_LIT> ( MI -> getOpcode ( ) , BitName ) ;

if ( BitIdx == - <NUM_LIT> ) return false ;


<s> int BitIdx = <STR_LIT> ( MI -> getOpcode ( ) , BitName ) ; if ( BitIdx == - <NUM_LIT> ) return false ;

MachineOperand & Bit = MI -> getOperand ( BitIdx ) ;


<s> if ( MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent || MOI . getFailureOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= insertWaitcntVmcnt0 ( MI ) ; if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent || MOI . getFailureOrdering ( ) == AtomicOrdering :: Acquire || MOI . getFailureOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= insertWaitcntVmcnt0 ( MI , false ) ;

Changed |= insertBufferWbinvl1Vol ( MI , false ) ;


<s> if ( MOI . isAtomic ( ) ) { if ( MOI . getS<ISA_LIT>D ( ) == SyncScope :: System || MOI . getS<ISA_LIT>D ( ) == MMI -> getAgentS<ISA_LIT>D ( ) ) { if ( MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= insertWaitcntVmcnt0 ( MI ) ; return Changed ; } if ( MOI . getS<ISA_LIT>D ( ) == SyncScope :: SingleThread || MOI . getS<ISA_LIT>D ( ) == MMI -> getWorkgroupS<ISA_LIT>D ( ) || MOI . getS<ISA_LIT>D ( ) == MMI -> getWavefrontS<ISA_LIT>D ( ) ) { return Changed ; } llvm_unreachable ( <STR_LIT> ) ; } if ( MOI . isNonTemporal ( ) ) { Changed |= enableGLCBit ( MI ) ;


<s> Optional < <ISA_LIT>MemOpInfo > <ISA_LIT>MemOpInfo :: getAtomicCmpxchgInfo ( const MachineBasicBlock :: iterator & MI ) { assert ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) ; if ( ! ( MI -> mayLoad ( ) && MI -> mayStore ( ) ) ) return None ; if ( MI -> getNumMemOperands ( ) == <NUM_LIT> ) return <ISA_LIT>MemOpInfo ( SyncScope :: System , AtomicOrdering :: SequentiallyConsistent , AtomicOrdering :: SequentiallyConsistent ) ;

return <ISA_LIT>MemOpInfo :: constructFromMIWithMMO ( MI ) ;


<s> if ( MI -> getOpcode ( ) != <STR_LIT> ) return None ; SyncScope :: ID S<ISA_LIT>D = static_cast < SyncScope :: ID > ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ; AtomicOrdering Ordering = static_cast < AtomicOrdering > ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ;

return <ISA_LIT>MemOpInfo ( S<ISA_LIT>D , Ordering ) ;


<s> if ( MI -> getNumMemOperands ( ) == <NUM_LIT> ) return <ISA_LIT>MemOpInfo ( SyncScope :: System , AtomicOrdering :: SequentiallyConsistent ) ;


<s> if ( ! ( MI -> mayLoad ( ) && MI -> mayStore ( ) ) ) return None ; if ( MI -> getNumMemOperands ( ) == <NUM_LIT> ) return <ISA_LIT>MemOpInfo ( SyncScope :: System , AtomicOrdering :: SequentiallyConsistent ) ; return <ISA_LIT>MemOpInfo :: constructFromMIWithMMO ( MI ) ;


<s> if ( ! ( MI -> mayLoad ( ) && ! MI -> mayStore ( ) ) ) return None ; if ( MI -> getNumMemOperands ( ) == <NUM_LIT> ) return <ISA_LIT>MemOpInfo ( SyncScope :: System , AtomicOrdering :: SequentiallyConsistent ) ;


<s> assert ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) ;

if ( ! ( MI -> mayLoad ( ) && ! MI -> mayStore ( ) ) ) return None ;


<s> if ( MI -> getNumMemOperands ( ) == <NUM_LIT> ) return <ISA_LIT>MemOpInfo ( SyncScope :: System , AtomicOrdering :: SequentiallyConsistent ) ; return <ISA_LIT>MemOpInfo :: constructFromMIWithMMO ( MI ) ;


<s> bool <ISA_LIT>MemoryLegalizer :: insertBufferWbinvl1Vol ( MachineBasicBlock :: iterator & MI , bool Before ) const { MachineBasicBlock & MBB = * MI -> getParent ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; if ( ! Before ) ++ MI ; BuildMI ( MBB , MI , DL , TII -> get ( Wbinvl1Opcode ) ) ; if ( ! Before ) -- MI ;


<s> bool <ISA_LIT>MemoryLegalizer :: insertBufferWbinvl1Vol ( MachineBasicBlock :: iterator & MI , bool Before ) const { MachineBasicBlock & MBB = * MI -> getParent ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ;


<s> if ( ! Before ) ++ MI ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( Vmcnt0Immediate ) ; if ( ! Before ) -- MI ; return true ;


BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( Vmcnt0Immediate ) ;


<s> if ( ! ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) ) continue ; if ( const auto & MOI = <ISA_LIT>MemOpInfo :: getLoadInfo ( MI ) ) Changed |= expandLoad ( MOI . getValue ( ) , MI ) ; else if ( const auto & MOI = <ISA_LIT>MemOpInfo :: getStoreInfo ( MI ) ) Changed |= expandStore ( MOI . getValue ( ) , MI ) ;

else if ( const auto & MOI = <ISA_LIT>MemOpInfo :: getAtomicFenceInfo ( MI ) ) Changed |= expandAtomicFence ( MOI . getValue ( ) , MI ) ;


<s> <ISA_LIT>MemOpInfo ( SyncScope :: ID S<ISA_LIT>D , AtomicOrdering Ordering , AtomicOrdering FailureOrdering , bool IsNonTemporal = false ) : S<ISA_LIT>D ( S<ISA_LIT>D ) , Ordering ( Ordering ) , FailureOrdering ( FailureOrdering ) , IsNonTemporal ( IsNonTemporal ) {


<s> return None ; } S<ISA_LIT>D = IsSyncScopeInclusion . getValue ( ) ? S<ISA_LIT>D : MMO -> getSyncScopeID ( ) ; Ordering = isStrongerThan ( Ordering , OpOrdering ) ? Ordering : MMO -> getSuccessOrdering ( ) ; assert ( MMO -> getFailureOrdering ( ) != AtomicOrdering :: Release && MMO -> getFailureOrdering ( ) != AtomicOrdering :: AcquireRelease ) ; FailureOrdering = isStrongerThan ( FailureOrdering , MMO -> getFailureOrdering ( ) ) ? FailureOrdering : MMO -> getFailureOrdering ( ) ; } } <ISA_LIT>AtomicScope Scope = <STR_LIT> ; <ISA_LIT>AtomicAddrSpace OrderingAddrSpace = <STR_LIT> ; bool IsCrossAddressSpaceOrdering = false ; if ( Ordering != AtomicOrdering :: NotAtomic ) { auto ScopeOrNone = to<ISA_LIT>AtomicScope ( S<ISA_LIT>D , InstrAddrSpace ) ; if ( ! ScopeOrNone ) { reportUnsupported ( MI , <STR_LIT> ) ; return None ; } std :: tie ( Scope , OrderingAddrSpace , IsCrossAddressSpaceOrdering ) = ScopeOrNone . getValue ( ) ; if ( ( OrderingAddrSpace == <STR_LIT> ) || ( ( OrderingAddrSpace & <STR_LIT> ) != OrderingAddrSpace ) || ( ( InstrAddrSpace & <STR_LIT> ) == <STR_LIT> ) ) { reportUnsupported ( MI , <STR_LIT> ) ; return None ;


<s> return ; } assert ( Scope != <STR_LIT> && ( OrderingAddrSpace & <STR_LIT> ) != <STR_LIT> && ( InstrAddrSpace & <STR_LIT> ) != <STR_LIT> && ! isStrongerThan ( FailureOrdering , Ordering ) ) ; if ( ( OrderingAddrSpace == InstrAddrSpace ) && isPowerOf2_32 ( uint32_t ( InstrAddrSpace ) ) ) this -> IsCrossAddressSpaceOrdering = false ; if ( ( InstrAddrSpace & ~ <STR_LIT> ) == <STR_LIT> ) { this -> Scope = std :: min ( Scope , <STR_LIT> ) ; } else if ( ( InstrAddrSpace & ~ ( <STR_LIT> | <STR_LIT> ) ) == <STR_LIT> ) { this -> Scope = std :: min ( Scope , <STR_LIT> ) ; } else if ( ( InstrAddrSpace & ~ ( <STR_LIT> | <STR_LIT> | <STR_LIT> ) ) == <STR_LIT> ) { this -> Scope = std :: min ( Scope , <STR_LIT> ) ; }


<s> void <ISA_LIT>MemOpInfo :: reportUnknownSyncScope ( const MachineBasicBlock :: iterator & MI ) {

DiagnosticInfoUnsupported Diag ( MI -> getParent ( ) -> getParent ( ) -> getFunction ( ) , <STR_LIT> ) ;


<s> void <ISA_LIT>MemOpInfo :: reportUnknownSyncScope ( const MachineBasicBlock :: iterator & MI ) { DiagnosticInfoUnsupported Diag ( MI -> getParent ( ) -> getParent ( ) -> getFunction ( ) , <STR_LIT> ) ; LLVMContext * CTX = & MI -> getParent ( ) -> getParent ( ) -> getFunction ( ) . getContext ( ) ; CTX -> diagnose ( Diag ) ;


<s> if ( ! InsertCacheInv ) return false ; bool Changed = false ; if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : case <STR_LIT> : break ;


<s> AtomicOrdering FailureOrdering = AtomicOrdering :: NotAtomic ; <ISA_LIT>AtomicAddrSpace InstrAddrSpace = <STR_LIT> ; bool IsNonTemporal = true ; for ( const auto & MMO : MI -> memoperands ( ) ) { IsNonTemporal &= MMO -> isNonTemporal ( ) ; InstrAddrSpace |= to<ISA_LIT>AtomicAddrSpace ( MMO -> getPointerInfo ( ) . getAddrSpace ( ) ) ; AtomicOrdering OpOrdering = MMO -> getOrdering ( ) ; if ( OpOrdering != AtomicOrdering :: NotAtomic ) { const auto & IsSyncScopeInclusion = MMI -> isSyncScopeInclusion ( S<ISA_LIT>D , MMO -> getSyncScopeID ( ) ) ; if ( ! IsSyncScopeInclusion ) { reportUnsupported ( MI , <STR_LIT> ) ; return None ; } S<ISA_LIT>D = IsSyncScopeInclusion . getValue ( ) ? S<ISA_LIT>D : MMO -> getSyncScopeID ( ) ; Ordering = isStrongerThan ( Ordering , OpOrdering ) ? Ordering : MMO -> getOrdering ( ) ; assert ( MMO -> getFailureOrdering ( ) != AtomicOrdering :: Release && MMO -> getFailureOrdering ( ) != AtomicOrdering :: AcquireRelease ) ; FailureOrdering = isStrongerThan ( FailureOrdering , MMO -> getFailureOrdering ( ) ) ? FailureOrdering : MMO -> getFailureOrdering ( ) ; } } <ISA_LIT>AtomicScope Scope = <STR_LIT> ; <ISA_LIT>AtomicAddrSpace OrderingAddrSpace = <STR_LIT> ; bool IsCrossAddressSpaceOrdering = false ; if ( Ordering != AtomicOrdering :: NotAtomic ) { auto ScopeOrNone = to<ISA_LIT>AtomicScope ( S<ISA_LIT>D , InstrAddrSpace ) ; if ( ! ScopeOrNone ) { reportUnsupported ( MI , <STR_LIT> ) ; return None ; } std :: tie ( Scope , OrderingAddrSpace , IsCrossAddressSpaceOrdering ) = ScopeOrNone . getValue ( ) ; if ( ( OrderingAddrSpace == <STR_LIT> ) || ( ( OrderingAddrSpace & <STR_LIT> ) != OrderingAddrSpace ) ) { reportUnsupported ( MI , <STR_LIT> ) ; return None ; }


<s> std :: unique_ptr < <ISA_LIT>CacheControl > <ISA_LIT>CacheControl :: create ( const GCNSubtarget & ST ) { GCNSubtarget :: Generation Generation = ST . getGeneration ( ) ; if ( Generation <= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return make_unique < <ISA_LIT>Gfx6CacheControl > ( ST ) ;

return make_unique < <ISA_LIT>Gfx7CacheControl > ( ST ) ;


<s> bool enableGLCBit ( const MachineBasicBlock :: iterator & MI ) const {

return enableNamedBit < <STR_LIT> :: glc > ( MI ) ;


<s> bool enableGLCBit ( const MachineBasicBlock :: iterator & MI ) const { return enableNamedBit < <STR_LIT> :: glc > ( MI ) ;


<s> bool enableNamedBit ( const MachineBasicBlock :: iterator & MI ) {

int BitIdx = <STR_LIT> ( MI -> getOpcode ( ) , BitName ) ;


<s> bool enableNamedBit ( const MachineBasicBlock :: iterator & MI ) { int BitIdx = <STR_LIT> ( MI -> getOpcode ( ) , BitName ) ; if ( BitIdx == - <NUM_LIT> ) return false ;


<s> bool <ISA_LIT>Gfx6CacheControl :: enableNonTemporal ( const MachineBasicBlock :: iterator & MI ) const { assert ( MI -> mayLoad ( ) ^ MI -> mayStore ( ) ) ;


<s> assert ( MI -> mayLoad ( ) && MI -> mayStore ( ) ) ; bool Changed = false ; if ( MOI . isAtomic ( ) ) { if ( MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent || MOI . getFailureOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , <STR_LIT> | <STR_LIT> , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ; if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent || MOI . getFailureOrdering ( ) == AtomicOrdering :: Acquire || MOI . getFailureOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) {

Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , isAtomicRet ( * MI ) ? <STR_LIT> :


<s> if ( MOI . isAtomic ( ) ) { if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , <STR_LIT> | <STR_LIT> , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ;

if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertCacheInvalidate ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , Position :: BEFORE ) ;


<s> if ( MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , <STR_LIT> | <STR_LIT> , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ; if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getInstrAddrSpace ( ) , <STR_LIT> , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: AFTER ) ;

Changed |= CC -> insertCacheInvalidate ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , Position :: AFTER ) ;


<s> if ( MOI . isAtomic ( ) ) { if ( MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , <STR_LIT> | <STR_LIT> , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ; return Changed ; } if ( MOI . isNonTemporal ( ) ) {

Changed |= CC -> enableNonTemporal ( MI ) ;


<s> Optional < <ISA_LIT>MemOpInfo > <ISA_LIT>MemOpAccess :: getAtomicFenceInfo ( const MachineBasicBlock :: iterator & MI ) const { assert ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) ; if ( MI -> getOpcode ( ) != <STR_LIT> ) return None ; AtomicOrdering Ordering = static_cast < AtomicOrdering > ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ; SyncScope :: ID S<ISA_LIT>D = static_cast < SyncScope :: ID > ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ; auto ScopeOrNone = to<ISA_LIT>AtomicScope ( S<ISA_LIT>D , <STR_LIT> ) ; if ( ! ScopeOrNone ) { reportUnsupported ( MI , <STR_LIT> ) ; return None ; } <ISA_LIT>AtomicScope Scope = <STR_LIT> ; <ISA_LIT>AtomicAddrSpace OrderingAddrSpace = <STR_LIT> ; bool IsCrossAddressSpaceOrdering = false ; std :: tie ( Scope , OrderingAddrSpace , IsCrossAddressSpaceOrdering ) = ScopeOrNone . getValue ( ) ; if ( ( OrderingAddrSpace == <STR_LIT> ) || ( ( OrderingAddrSpace & <STR_LIT> ) != OrderingAddrSpace ) ) {


<s> assert ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) ; if ( MI -> getOpcode ( ) != <STR_LIT> ) return None ; AtomicOrdering Ordering = static_cast < AtomicOrdering > ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ; SyncScope :: ID S<ISA_LIT>D = static_cast < SyncScope :: ID > ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ; auto ScopeOrNone = to<ISA_LIT>AtomicScope ( S<ISA_LIT>D , <STR_LIT> ) ; if ( ! ScopeOrNone ) { reportUnsupported ( MI , <STR_LIT> ) ; return None ; } <ISA_LIT>AtomicScope Scope = <STR_LIT> ; <ISA_LIT>AtomicAddrSpace OrderingAddrSpace = <STR_LIT> ; bool IsCrossAddressSpaceOrdering = false ; std :: tie ( Scope , OrderingAddrSpace , IsCrossAddressSpaceOrdering ) = ScopeOrNone . getValue ( ) ; if ( ( OrderingAddrSpace == <STR_LIT> ) || ( ( OrderingAddrSpace & <STR_LIT> ) != OrderingAddrSpace ) ) { reportUnsupported ( MI , <STR_LIT> ) ; return None ; } return <ISA_LIT>MemOpInfo ( Ordering , Scope , OrderingAddrSpace , <STR_LIT> , IsCrossAddressSpaceOrdering ) ;


<s> bool Changed = false ; MachineBasicBlock & MBB = * MI -> getParent ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; if ( Pos == Position :: AFTER ) ++ MI ; if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : case <STR_LIT> : BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : LGKMCnt = IsCrossAddrSpaceOrdering ; break ; case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : case <STR_LIT> : EXPCnt = IsCrossAddrSpaceOrdering ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( VMCnt || LGKMCnt || EXPCnt ) {

unsigned WaitCntImmediate = <STR_LIT> ( IV , VMCnt ? <NUM_LIT> : getVmcntBitMask ( IV ) , EXPCnt ? <NUM_LIT> : getExpcntBitMask ( IV ) , LGKMCnt ? <NUM_LIT> : getLgkmcntBitMask ( IV ) ) ;


<s> return <STR_LIT> ( MI . getOpcode ( ) ) != - <NUM_LIT> ;


<s> bool isAtomicRet ( const MachineInstr & MI ) const {

return <STR_LIT> ( MI . getOpcode ( ) ) != - <NUM_LIT> ;


<s> TII = ST . getInstrInfo ( ) ; IV = IsaInfo :: getIsaVersion ( ST . getFeatureBits ( ) ) ;


<s> MMI = & MF . getMMI ( ) . getObjFileInfo < <ISA_LIT>MachineModuleInfo > ( ) ;


<s> <ISA_LIT>MemOpAccess :: <ISA_LIT>MemOpAccess ( MachineFunction & MF ) { <ISA_LIT>AddrSpaceInfo = get<ISA_LIT>AS ( MF . getTarget ( ) ) ;

MMI = & MF . getMMI ( ) . getObjFileInfo < <ISA_LIT>MachineModuleInfo > ( ) ;


<s> <ISA_LIT>MemOpInfo ( AtomicOrdering Ordering = AtomicOrdering :: SequentiallyConsistent , <ISA_LIT>AtomicScope Scope = <STR_LIT> , <ISA_LIT>AtomicAddrSpace OrderingAddrSpace = <STR_LIT> , <ISA_LIT>AtomicAddrSpace InstrAddrSpace = <STR_LIT> , bool IsCrossAddressSpaceOrdering = true , AtomicOrdering FailureOrdering = AtomicOrdering :: SequentiallyConsistent , bool IsNonTemporal = false ) : Ordering ( Ordering ) , FailureOrdering ( FailureOrdering ) , Scope ( Scope ) , OrderingAddrSpace ( OrderingAddrSpace ) , InstrAddrSpace ( InstrAddrSpace ) , IsCrossAddressSpaceOrdering ( IsCrossAddressSpaceOrdering ) , IsNonTemporal ( IsNonTemporal ) { if ( ( OrderingAddrSpace == InstrAddrSpace ) && isPowerOf2_32 ( uint32_t ( InstrAddrSpace ) ) ) IsCrossAddressSpaceOrdering = false ;


<s> <ISA_LIT>AtomicAddrSpace <ISA_LIT>MemOpAccess :: to<ISA_LIT>AtomicAddrSpace ( unsigned AS ) const { if ( AS == <ISA_LIT>AddrSpaceInfo . FLAT_ADDRESS ) return <STR_LIT> ; if ( AS == <ISA_LIT>AddrSpaceInfo . GLOBAL_ADDRESS ) return <STR_LIT> ; if ( AS == <ISA_LIT>AddrSpaceInfo . LOCAL_ADDRESS ) return <STR_LIT> ; if ( AS == <ISA_LIT>AddrSpaceInfo . PRIVATE_ADDRESS ) return <STR_LIT> ; if ( AS == <ISA_LIT>AddrSpaceInfo . REGION_ADDRESS ) return <STR_LIT> ; return <STR_LIT> ;


<s> Optional < std :: tuple < <ISA_LIT>AtomicScope , <ISA_LIT>AtomicAddrSpace , bool >> <ISA_LIT>MemOpAccess :: to<ISA_LIT>AtomicScope ( SyncScope :: ID S<ISA_LIT>D , <ISA_LIT>AtomicAddrSpace InstrScope ) const { if ( S<ISA_LIT>D == SyncScope :: System ) return std :: make_tuple ( <STR_LIT> , <STR_LIT> & InstrScope , false ) ; if ( S<ISA_LIT>D == MMI -> getAgentS<ISA_LIT>D ( ) ) return std :: make_tuple ( <STR_LIT> , <STR_LIT> & InstrScope , false ) ; if ( S<ISA_LIT>D == MMI -> getWorkgroupS<ISA_LIT>D ( ) ) return std :: make_tuple ( <STR_LIT> , <STR_LIT> & InstrScope , false ) ; if ( S<ISA_LIT>D == MMI -> getWavefrontS<ISA_LIT>D ( ) ) return std :: make_tuple ( <STR_LIT> , <STR_LIT> & InstrScope , false ) ;

if ( S<ISA_LIT>D == SyncScope :: SingleThread ) return std :: make_tuple ( <STR_LIT> , <STR_LIT> & InstrScope , false ) ;


<s> <ISA_LIT>Subtarget :: Generation Generation = ST . getGeneration ( ) ;

if ( Generation <= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return make_unique < <ISA_LIT>Gfx6CacheControl > ( ST ) ;


<s> IV = IsaInfo :: getIsaVersion ( ST . getFeatureBits ( ) ) ;


<s> <ISA_LIT>CacheControl :: <ISA_LIT>CacheControl ( const <ISA_LIT>Subtarget & ST ) {


<s> bool Changed = false ; if ( MOI . isAtomic ( ) ) { if ( MOI . getS<ISA_LIT>D ( ) == SyncScope :: System || MOI . getS<ISA_LIT>D ( ) == MMI -> getAgentS<ISA_LIT>D ( ) ) { if ( MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent || MOI . getFailureOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= insertWaitcntVmcnt0 ( MI ) ; if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent || MOI . getFailureOrdering ( ) == AtomicOrdering :: Acquire || MOI . getFailureOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= insertWaitcntVmcnt0 ( MI , false ) ; Changed |= insertVmem<ISA_LIT>MDCacheInvalidate ( MI , false ) ; } return Changed ;


<s> bool Changed = false ; if ( MOI . isAtomic ( ) ) { if ( MOI . getS<ISA_LIT>D ( ) == SyncScope :: System || MOI . getS<ISA_LIT>D ( ) == MMI -> getAgentS<ISA_LIT>D ( ) ) { if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= insertWaitcntVmcnt0 ( MI ) ; if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= insertVmem<ISA_LIT>MDCacheInvalidate ( MI ) ; AtomicPseudoMIs . push_back ( MI ) ;


<s> Changed |= insertWaitcntVmcnt0 ( MI , false ) ; Changed |= insertVmem<ISA_LIT>MDCacheInvalidate ( MI , false ) ; } return Changed ; } if ( MOI . getS<ISA_LIT>D ( ) == SyncScope :: SingleThread || MOI . getS<ISA_LIT>D ( ) == MMI -> getWorkgroupS<ISA_LIT>D ( ) || MOI . getS<ISA_LIT>D ( ) == MMI -> getWavefrontS<ISA_LIT>D ( ) ) { return Changed ; } llvm_unreachable ( <STR_LIT> ) ; } if ( MOI . isNonTemporal ( ) ) { Changed |= enableGLCBit ( MI ) ; Changed |= enableSLCBit ( MI ) ; return Changed ; }


<s> if ( MOI . getS<ISA_LIT>D ( ) == SyncScope :: System || MOI . getS<ISA_LIT>D ( ) == MMI -> getAgentS<ISA_LIT>D ( ) ) { if ( MOI . getOrdering ( ) == AtomicOrdering :: Monotonic || MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= enableGLCBit ( MI ) ; if ( MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= insertWaitcntVmcnt0 ( MI ) ; if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= insertWaitcntVmcnt0 ( MI , false ) ; Changed |= insertVmem<ISA_LIT>MDCacheInvalidate ( MI , false ) ; } return Changed ; } if ( MOI . getS<ISA_LIT>D ( ) == SyncScope :: SingleThread || MOI . getS<ISA_LIT>D ( ) == MMI -> getWorkgroupS<ISA_LIT>D ( ) || MOI . getS<ISA_LIT>D ( ) == MMI -> getWavefrontS<ISA_LIT>D ( ) ) { return Changed ; } llvm_unreachable ( <STR_LIT> ) ; } if ( MOI . isNonTemporal ( ) ) { Changed |= enableGLCBit ( MI ) ; Changed |= enableSLCBit ( MI ) ; return Changed ; }


<s> Optional < <ISA_LIT>MemOpInfo > <ISA_LIT>MemOpInfo :: getAtomicCmpxchgOrRmwInfo ( const MachineBasicBlock :: iterator & MI ) {

assert ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) ;


<s> assert ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) ; if ( ! ( MI -> mayLoad ( ) && MI -> mayStore ( ) ) ) return None ; if ( MI -> getNumMemOperands ( ) == <NUM_LIT> ) return <ISA_LIT>MemOpInfo ( SyncScope :: System , AtomicOrdering :: SequentiallyConsistent , AtomicOrdering :: SequentiallyConsistent ) ; return <ISA_LIT>MemOpInfo :: constructFromMIWithMMO ( MI ) ;


<s> MachineBasicBlock & MBB = * MI -> getParent ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; if ( ! Before ) ++ MI ; BuildMI ( MBB , MI , DL , TII -> get ( Vmem<ISA_LIT>MDCacheInvalidateOpc ) ) ; if ( ! Before ) -- MI ; return true ;


<s> bool <ISA_LIT>MemoryLegalizer :: insertVmem<ISA_LIT>MDCacheInvalidate ( MachineBasicBlock :: iterator & MI , bool Before ) const {

MachineBasicBlock & MBB = * MI -> getParent ( ) ;


<s> for ( auto & MBB : MF ) { for ( auto MI = MBB . begin ( ) ; MI != MBB . end ( ) ; ++ MI ) { if ( ! ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) ) continue ; if ( const auto & MOI = <ISA_LIT>MemOpInfo :: getLoadInfo ( MI ) ) Changed |= expandLoad ( MOI . getValue ( ) , MI ) ; else if ( const auto & MOI = <ISA_LIT>MemOpInfo :: getStoreInfo ( MI ) ) Changed |= expandStore ( MOI . getValue ( ) , MI ) ; else if ( const auto & MOI = <ISA_LIT>MemOpInfo :: getAtomicFenceInfo ( MI ) ) Changed |= expandAtomicFence ( MOI . getValue ( ) , MI ) ; else if ( const auto & MOI = <ISA_LIT>MemOpInfo :: getAtomicCmpxchgOrRmwInfo ( MI ) ) Changed |= expandAtomicCmpxchgOrRmw ( MOI . getValue ( ) , MI ) ; } } Changed |= removeAtomicPseudoMIs ( ) ;


<s> bool Changed = false ; MachineBasicBlock & MBB = * MI -> getParent ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; if ( Pos == Position :: AFTER ) ++ MI ; if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : case <STR_LIT> : BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) ; Changed = true ; break ; case <STR_LIT> : if ( ! CuMode ) { BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) ; Changed = true ; } break ; case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; }


<s> GCNSubtarget :: Generation Generation = ST . getGeneration ( ) ; if ( Generation <= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return make_unique < <ISA_LIT>Gfx6CacheControl > ( ST ) ;

if ( Generation < <ISA_LIT>Subtarget :: GFX10 ) return make_unique < <ISA_LIT>Gfx7CacheControl > ( ST ) ;


<s> bool <ISA_LIT>MemoryLegalizer :: expandLoad ( const <ISA_LIT>MemOpInfo & MOI , MachineBasicBlock :: iterator & MI ) { assert ( MI -> mayLoad ( ) && ! MI -> mayStore ( ) ) ; bool Changed = false ; if ( MOI . isAtomic ( ) ) { if ( MOI . getOrdering ( ) == AtomicOrdering :: Monotonic || MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= CC -> enableLoadCacheBypass ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) ) ; } if ( MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , <STR_LIT> | <STR_LIT> , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ; if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getInstrAddrSpace ( ) , <STR_LIT> , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: AFTER ) ; Changed |= CC -> insertAcquire ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , Position :: AFTER ) ; } return Changed ; } if ( MOI . isNonTemporal ( ) ) { Changed |= CC -> enableNonTemporal ( MI ) ; return Changed ; } return Changed ;


<s> bool Changed = false ; if ( MOI . isAtomic ( ) ) { if ( MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertRelease ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ; return Changed ; }


<s> case <STR_LIT> : case <STR_LIT> : LGKMCnt |= IsCrossAddrSpaceOrdering ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( VMCnt || LGKMCnt ) { unsigned WaitCntImmediate = <STR_LIT> ( IV , VMCnt ? <NUM_LIT> : getVmcntBitMask ( IV ) , getExpcntBitMask ( IV ) , LGKMCnt ? <NUM_LIT> : getLgkmcntBitMask ( IV ) ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( WaitCntImmediate ) ; Changed = true ; } if ( VSCnt ) { BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) ; Changed = true ; }


<s> <ISA_LIT>CacheControl :: <ISA_LIT>CacheControl ( const GCNSubtarget & ST ) { TII = ST . getInstrInfo ( ) ;

IV = getIsaVersion ( ST . getCPU ( ) ) ;


<s> assert ( MI -> mayLoad ( ) ^ MI -> mayStore ( ) ) ; assert ( Op == <STR_LIT> || Op == <STR_LIT> ) ; bool Changed = false ; if ( IsVolatile ) { if ( Op == <STR_LIT> ) {


<s> reportUnsupported ( MI , <STR_LIT> ) ; return None ; } S<ISA_LIT>D = IsSyncScopeInclusion . getValue ( ) ? S<ISA_LIT>D : MMO -> getSyncScopeID ( ) ; Ordering = getMergedAtomicOrdering ( Ordering , OpOrdering ) ; assert ( MMO -> getFailureOrdering ( ) != AtomicOrdering :: Release && MMO -> getFailureOrdering ( ) != AtomicOrdering :: AcquireRelease ) ; FailureOrdering = getMergedAtomicOrdering ( FailureOrdering , MMO -> getFailureOrdering ( ) ) ; } } <ISA_LIT>AtomicScope Scope = <STR_LIT> ; <ISA_LIT>AtomicAddrSpace OrderingAddrSpace = <STR_LIT> ; bool IsCrossAddressSpaceOrdering = false ; if ( Ordering != AtomicOrdering :: NotAtomic ) { auto ScopeOrNone = to<ISA_LIT>AtomicScope ( S<ISA_LIT>D , InstrAddrSpace ) ; if ( ! ScopeOrNone ) { reportUnsupported ( MI , <STR_LIT> ) ; return None ; }

std :: tie ( Scope , OrderingAddrSpace , IsCrossAddressSpaceOrdering ) = ScopeOrNone . getValue ( ) ;


<s> bool IsCrossAddressSpaceOrdering = false ; if ( Ordering != AtomicOrdering :: NotAtomic ) { auto ScopeOrNone = to<ISA_LIT>AtomicScope ( S<ISA_LIT>D , InstrAddrSpace ) ; if ( ! ScopeOrNone ) { reportUnsupported ( MI , <STR_LIT> ) ; return None ; } std :: tie ( Scope , OrderingAddrSpace , IsCrossAddressSpaceOrdering ) = ScopeOrNone . getValue ( ) ; if ( ( OrderingAddrSpace == <STR_LIT> ) || ( ( OrderingAddrSpace & <STR_LIT> ) != OrderingAddrSpace ) || ( ( InstrAddrSpace & <STR_LIT> ) == <STR_LIT> ) ) { reportUnsupported ( MI , <STR_LIT> ) ; return None ; } }

return <ISA_LIT>MemOpInfo ( Ordering , Scope , OrderingAddrSpace , InstrAddrSpace , IsCrossAddressSpaceOrdering , FailureOrdering , IsVolatile , IsNonTemporal ) ;


<s> if ( Generation < <ISA_LIT>Subtarget :: GFX10 ) return std :: make_unique < <ISA_LIT>Gfx7CacheControl > ( ST ) ;

return std :: make_unique < <ISA_LIT>Gfx10CacheControl > ( ST ) ;


<s> if ( ST . hasGFX90AInsts ( ) ) return std :: make_unique < <ISA_LIT>Gfx90ACacheControl > ( ST ) ; if ( Generation <= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return std :: make_unique < <ISA_LIT>Gfx6CacheControl > ( ST ) ; if ( Generation < <ISA_LIT>Subtarget :: GFX10 ) return std :: make_unique < <ISA_LIT>Gfx7CacheControl > ( ST ) ; return std :: make_unique < <ISA_LIT>Gfx10CacheControl > ( ST ) ;


<s> return enableNamedBit ( MI , <STR_LIT> :: DLC ) ;


<s> bool enableGLCBit ( const MachineBasicBlock :: iterator & MI ) const { return enableNamedBit ( MI , <STR_LIT> :: GLC ) ;


<s> return enableNamedBit ( MI , <STR_LIT> :: GLC ) ;


<s> if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : case <STR_LIT> : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : break ; default :


<s> bool enableSLCBit ( const MachineBasicBlock :: iterator & MI ) const { return enableNamedBit ( MI , <STR_LIT> :: SLC ) ;


<s> bool <ISA_LIT>Gfx10CacheControl :: enableVolatileAndOrNonTemporal ( MachineBasicBlock :: iterator & MI , <ISA_LIT>AtomicAddrSpace AddrSpace , <ISA_LIT>MemOp Op , bool IsVolatile , bool IsNonTemporal ) const { assert ( MI -> mayLoad ( ) ^ MI -> mayStore ( ) ) ; assert ( Op == <STR_LIT> || Op == <STR_LIT> ) ; bool Changed = false ; if ( IsVolatile ) { if ( Op == <STR_LIT> ) { Changed |= enableGLCBit ( MI ) ; Changed |= enableDLCBit ( MI ) ;


<s> } if ( MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent || MOI . getFailureOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertRelease ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ; if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent || MOI . getFailureOrdering ( ) == AtomicOrdering :: Acquire || MOI . getFailureOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getInstrAddrSpace ( ) , isAtomicRet ( * MI ) ? <STR_LIT> : <STR_LIT> , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: AFTER ) ;

Changed |= CC -> insertAcquire ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , Position :: AFTER ) ;


<s> bool Changed = false ; if ( MOI . isAtomic ( ) ) { if ( MOI . getOrdering ( ) == AtomicOrdering :: Monotonic || MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= CC -> enableRMWCacheBypass ( MI , MOI . getScope ( ) , MOI . getInstrAddrSpace ( ) ) ; } if ( MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent || MOI . getFailureOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertRelease ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ; if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent || MOI . getFailureOrdering ( ) == AtomicOrdering :: Acquire || MOI . getFailureOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) {

Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getInstrAddrSpace ( ) , isAtomicRet ( * MI ) ? <STR_LIT> :


<s> bool <ISA_LIT>MemoryLegalizer :: expandAtomicFence ( const <ISA_LIT>MemOpInfo & MOI , MachineBasicBlock :: iterator & MI ) { assert ( MI -> getOpcode ( ) == <STR_LIT> ) ; AtomicPseudoMIs . push_back ( MI ) ; bool Changed = false ; if ( MOI . isAtomic ( ) ) { if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertRelease ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ; if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertAcquire ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , Position :: BEFORE ) ; return Changed ;


<s> bool <ISA_LIT>MemoryLegalizer :: expandAtomicFence ( const <ISA_LIT>MemOpInfo & MOI , MachineBasicBlock :: iterator & MI ) { assert ( MI -> getOpcode ( ) == <STR_LIT> ) ; AtomicPseudoMIs . push_back ( MI ) ; bool Changed = false ; if ( MOI . isAtomic ( ) ) {

if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: AcquireRelease || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertRelease ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ;


<s> if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getInstrAddrSpace ( ) , <STR_LIT> , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: AFTER ) ; Changed |= CC -> insertAcquire ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , Position :: AFTER ) ; } return Changed ; }

Changed |= CC -> enableVolatileAndOrNonTemporal ( MI , MOI . getInstrAddrSpace ( ) , <STR_LIT> , MOI . isVolatile ( ) , MOI . isNonTemporal ( ) ) ;


<s> if ( MOI . getOrdering ( ) == AtomicOrdering :: Monotonic || MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= CC -> enableLoadCacheBypass ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) ) ; } if ( MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , <STR_LIT> | <STR_LIT> , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ; if ( MOI . getOrdering ( ) == AtomicOrdering :: Acquire || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= CC -> insertWait ( MI , MOI . getScope ( ) , MOI . getInstrAddrSpace ( ) , <STR_LIT> , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: AFTER ) ; Changed |= CC -> insertAcquire ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , Position :: AFTER ) ; } return Changed ; } Changed |= CC -> enableVolatileAndOrNonTemporal ( MI , MOI . getInstrAddrSpace ( ) , <STR_LIT> , MOI . isVolatile ( ) , MOI . isNonTemporal ( ) ) ; return Changed ;


<s> if ( MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertRelease ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ; return Changed ; }


<s> bool Changed = false ; if ( MOI . isAtomic ( ) ) { if ( MOI . getOrdering ( ) == AtomicOrdering :: Monotonic || MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) { Changed |= CC -> enableStoreCacheBypass ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) ) ; } if ( MOI . getOrdering ( ) == AtomicOrdering :: Release || MOI . getOrdering ( ) == AtomicOrdering :: SequentiallyConsistent ) Changed |= CC -> insertRelease ( MI , MOI . getScope ( ) , MOI . getOrderingAddrSpace ( ) , MOI . getIsCrossAddressSpaceOrdering ( ) , Position :: BEFORE ) ;


<s> Optional < <ISA_LIT>MemOpInfo > <ISA_LIT>MemOpAccess :: getAtomicCmpxchgOrRmwInfo ( const MachineBasicBlock :: iterator & MI ) const { assert ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) ; if ( ! ( MI -> mayLoad ( ) && MI -> mayStore ( ) ) ) return None ; if ( MI -> getNumMemOperands ( ) == <NUM_LIT> ) return <ISA_LIT>MemOpInfo ( ) ;


<s> AtomicOrdering getFailureOrdering ( ) const { return FailureOrdering ;


<s> return IsCrossAddressSpaceOrdering ;


<s> if ( ! ( MI -> mayLoad ( ) && ! MI -> mayStore ( ) ) ) return None ;

if ( MI -> getNumMemOperands ( ) == <NUM_LIT> ) return <ISA_LIT>MemOpInfo ( ) ;


<s> AtomicOrdering getOrdering ( ) const { return Ordering ;


<s> AtomicOrdering getOrdering ( ) const {


<s> assert ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) ; if ( ! ( ! MI -> mayLoad ( ) && MI -> mayStore ( ) ) ) return None ; if ( MI -> getNumMemOperands ( ) == <NUM_LIT> ) return <ISA_LIT>MemOpInfo ( ) ; return constructFromMIWithMMO ( MI ) ;


<s> if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : case <STR_LIT> : BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) ; Changed = true ; break ; case <STR_LIT> : if ( ! ST . isCuModeEnabled ( ) ) { BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) ; Changed = true ; } break ; case <STR_LIT> : case <STR_LIT> : break ;


<s> LGKMCnt |= IsCrossAddrSpaceOrdering ; break ; case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : case <STR_LIT> : LGKMCnt |= IsCrossAddrSpaceOrdering ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( VMCnt || LGKMCnt ) { unsigned WaitCntImmediate = <STR_LIT> ( IV , VMCnt ? <NUM_LIT> : getVmcntBitMask ( IV ) , getExpcntBitMask ( IV ) , LGKMCnt ? <NUM_LIT> : getLgkmcntBitMask ( IV ) ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( WaitCntImmediate ) ; Changed = true ; } if ( VSCnt ) {

BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) ;


<s> case <STR_LIT> : if ( ! ST . isCuModeEnabled ( ) ) { if ( ( Op & <STR_LIT> ) != <STR_LIT> ) VMCnt |= true ; if ( ( Op & <STR_LIT> ) != <STR_LIT> ) VSCnt |= true ; } break ; case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : LGKMCnt |= IsCrossAddrSpaceOrdering ; break ; case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( ( AddrSpace & <STR_LIT> ) != <STR_LIT> ) { switch ( Scope ) { case <STR_LIT> : case <STR_LIT> : LGKMCnt |= IsCrossAddrSpaceOrdering ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } if ( VMCnt || LGKMCnt ) { unsigned WaitCntImmediate = <STR_LIT> ( IV , VMCnt ? <NUM_LIT> : getVmcntBitMask ( IV ) , getExpcntBitMask ( IV ) , LGKMCnt ? <NUM_LIT> : getLgkmcntBitMask ( IV ) ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( WaitCntImmediate ) ; Changed = true ; } if ( VSCnt ) {


<s> bool isAtomicRet ( const MachineInstr & MI ) const {

return <ISA_LIT>InstrInfo :: isAtomicRet ( MI ) ;


<s> bool isNonTemporal ( ) const { return IsNonTemporal ;


<s> const Function & Func = MI -> getParent ( ) -> getParent ( ) -> getFunction ( ) ;

DiagnosticInfoUnsupported Diag ( Func , Msg , MI -> getDebugLoc ( ) ) ;


<s> void <ISA_LIT>MemOpAccess :: reportUnsupported ( const MachineBasicBlock :: iterator & MI , const char * Msg ) const { const Function & Func = MI -> getParent ( ) -> getParent ( ) -> getFunction ( ) ; DiagnosticInfoUnsupported Diag ( Func , Msg , MI -> getDebugLoc ( ) ) ;

Func . getContext ( ) . diagnose ( Diag ) ;


<s> MachineBasicBlock :: instr_iterator II ( MI -> getIterator ( ) ) ; for ( MachineBasicBlock :: instr_iterator I = ++ II , E = MBB . instr_end ( ) ; I != E && I -> isBundledWithPred ( ) ; ++ I ) { I -> unbundleFromPred ( ) ; for ( MachineOperand & MO : I -> operands ( ) ) if ( MO . isReg ( ) ) MO . setIsInternalRead ( false ) ; } MI -> eraseFromParent ( ) ; MI = II -> getIterator ( ) ; } if ( ! ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) ) continue ; if ( const auto & MOI = MOA . getLoadInfo ( MI ) ) Changed |= expandLoad ( MOI . getValue ( ) , MI ) ; else if ( const auto & MOI = MOA . getStoreInfo ( MI ) ) Changed |= expandStore ( MOI . getValue ( ) , MI ) ; else if ( const auto & MOI = MOA . getAtomicFenceInfo ( MI ) ) Changed |= expandAtomicFence ( MOI . getValue ( ) , MI ) ; else if ( const auto & MOI = MOA . getAtomicCmpxchgOrRmwInfo ( MI ) ) Changed |= expandAtomicCmpxchgOrRmw ( MOI . getValue ( ) , MI ) ; }


<s> <ISA_LIT>MemOpAccess MOA ( MF ) ; CC = <ISA_LIT>CacheControl :: create ( MF . getSubtarget < GCNSubtarget > ( ) ) ; for ( auto & MBB : MF ) { for ( auto MI = MBB . begin ( ) ; MI != MBB . end ( ) ; ++ MI ) { if ( MI -> isBundle ( ) ) { MachineBasicBlock :: instr_iterator II ( MI -> getIterator ( ) ) ; for ( MachineBasicBlock :: instr_iterator I = ++ II , E = MBB . instr_end ( ) ; I != E && I -> isBundledWithPred ( ) ; ++ I ) { I -> unbundleFromPred ( ) ; for ( MachineOperand & MO : I -> operands ( ) ) if ( MO . isReg ( ) ) MO . setIsInternalRead ( false ) ; } MI -> eraseFromParent ( ) ; MI = II -> getIterator ( ) ; }

if ( ! ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) ) continue ;


<s> <ISA_LIT>CacheControl :: <ISA_LIT>CacheControl ( const GCNSubtarget & ST ) : ST ( ST ) { TII = ST . getInstrInfo ( ) ; IV = getIsaVersion ( ST . getCPU ( ) ) ;

InsertCacheInv = ! AmdgcnSkipCacheInvalidations ;


<s> TII = ST . getInstrInfo ( ) ; IV = getIsaVersion ( ST . getCPU ( ) ) ; InsertCacheInv = ! AmdgcnSkipCacheInvalidations ;


<s> <ISA_LIT>MemOpAccess :: <ISA_LIT>MemOpAccess ( MachineFunction & MF ) { MMI = & MF . getMMI ( ) . getObjFileInfo < <ISA_LIT>MachineModuleInfo > ( ) ;


<s> assert ( Scope != <STR_LIT> && ( OrderingAddrSpace & <STR_LIT> ) != <STR_LIT> && ( InstrAddrSpace & <STR_LIT> ) != <STR_LIT> ) ; if ( ( OrderingAddrSpace == InstrAddrSpace ) && isPowerOf2_32 ( uint32_t ( InstrAddrSpace ) ) ) this -> IsCrossAddressSpaceOrdering = false ; if ( ( InstrAddrSpace & ~ <STR_LIT> ) == <STR_LIT> ) { this -> Scope = std :: min ( Scope , <STR_LIT> ) ; } else if ( ( InstrAddrSpace & ~ ( <STR_LIT> | <STR_LIT> ) ) == <STR_LIT> ) {


<s> <ISA_LIT>MemoryLegalizer ( ) : MachineFunctionPass ( ID ) {


<s> unsigned Width = countTrailingOnes < unsigned > ( InstrMode . Mask >> Offset ) ; unsigned Value = ( InstrMode . Mode >> Offset ) & ( ( <NUM_LIT> << Width ) - <NUM_LIT> ) ; BuildMI ( MBB , MI , nullptr , TII -> get ( <STR_LIT> ) ) . addImm ( Value ) . addImm ( ( ( Width - <NUM_LIT> ) << <STR_LIT> :: WIDTH_M1_SHIFT_ ) | ( Offset << <STR_LIT> :: OFFSET_SHIFT_ ) | ( <STR_LIT> :: ID_MODE << <STR_LIT> :: ID_SHIFT_ ) ) ;


<s> BuildMI ( MBB , MI , nullptr , TII -> get ( <STR_LIT> ) ) . addImm ( Value ) . addImm ( ( ( Width - <NUM_LIT> ) << <STR_LIT> :: WIDTH_M1_SHIFT_ ) | ( Offset << <STR_LIT> :: OFFSET_SHIFT_ ) | ( <STR_LIT> :: ID_MODE << <STR_LIT> :: ID_SHIFT_ ) ) ; ++ NumSetregInserted ; Changed = true ; InstrMode . Mask &= ~ ( ( ( <NUM_LIT> << Width ) - <NUM_LIT> ) << Offset ) ;


<s> if ( MBB . pred_empty ( ) ) { BlockInfo [ ThisBlock ] -> Pred = DefaultStatus ; ExitSet = true ; } else { MachineBasicBlock :: pred_iterator P = MBB . pred_begin ( ) , E = MBB . pred_end ( ) ; MachineBasicBlock & PB = * ( * P ) ; unsigned PredBlock = PB . getNumber ( ) ; if ( ( ThisBlock == PredBlock ) && ( std :: next ( P ) == E ) ) { BlockInfo [ ThisBlock ] -> Pred = DefaultStatus ; ExitSet = true ; } else if ( BlockInfo [ PredBlock ] -> ExitSet ) { BlockInfo [ ThisBlock ] -> Pred = BlockInfo [ PredBlock ] -> Exit ; ExitSet = true ; } else if ( PredBlock != ThisBlock ) RevisitRequired = true ; for ( P = std :: next ( P ) ; P != E ; P = std :: next ( P ) ) { MachineBasicBlock * Pred = * P ; unsigned PredBlock = Pred -> getNumber ( ) ; if ( BlockInfo [ PredBlock ] -> ExitSet ) { if ( BlockInfo [ ThisBlock ] -> ExitSet ) { BlockInfo [ ThisBlock ] -> Pred = BlockInfo [ ThisBlock ] -> Pred . intersect ( BlockInfo [ PredBlock ] -> Exit ) ; } else { BlockInfo [ ThisBlock ] -> Pred = BlockInfo [ PredBlock ] -> Exit ; } ExitSet = true ; } else if ( PredBlock != ThisBlock ) RevisitRequired = true ; }


<s> unsigned Width = countTrailingOnes < unsigned > ( InstrMode . Mask >> Offset ) ; unsigned Value = ( InstrMode . Mode >> Offset ) & ( ( <NUM_LIT> << Width ) - <NUM_LIT> ) ; BuildMI ( MBB , MI , <NUM_LIT> , TII -> get ( <STR_LIT> ) ) . addImm ( Value ) . addImm ( ( ( Width - <NUM_LIT> ) << <STR_LIT> :: WIDTH_M1_SHIFT_ ) | ( Offset << <STR_LIT> :: OFFSET_SHIFT_ ) | ( <STR_LIT> :: ID_MODE << <STR_LIT> :: ID_SHIFT_ ) ) ;


<s> void <ISA_LIT>ModeRegister :: insertSetreg ( MachineBasicBlock & MBB , MachineInstr * MI , const <ISA_LIT>InstrInfo * TII , Status InstrMode ) { while ( InstrMode . Mask ) { unsigned Offset = countTrailingZeros < unsigned > ( InstrMode . Mask ) ;

unsigned Width = countTrailingOnes < unsigned > ( InstrMode . Mask >> Offset ) ;


<s> insertSetreg ( MBB , InsertionPoint , TII , IPChange . delta ( NewInfo -> Change ) ) ; InsertionPoint = nullptr ; } if ( MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ) { unsigned Val = TII -> getNamedOperand ( MI , <STR_LIT> :: imm ) -> getImm ( ) ; unsigned Mode = ( Val << Offset ) & Mask ; Status Setreg = Status ( Mask , Mode ) ; RequirePending = false ; NewInfo -> Change = NewInfo -> Change . merge ( Setreg ) ; } else { NewInfo -> Change = NewInfo -> Change . mergeUnknown ( Mask ) ; } } else if ( ! NewInfo -> Change . isCompatible ( InstrMode ) ) { if ( InsertionPoint ) { if ( ! IPChange . delta ( NewInfo -> Change ) . isCombinable ( InstrMode ) ) { if ( RequirePending ) { NewInfo -> FirstInsertionPoint = InsertionPoint ; NewInfo -> Require = NewInfo -> Change ; RequirePending = false ; } else { insertSetreg ( MBB , InsertionPoint , TII , IPChange . delta ( NewInfo -> Change ) ) ; IPChange = NewInfo -> Change ; } InsertionPoint = & MI ; } NewInfo -> Change = NewInfo -> Change . merge ( InstrMode ) ; } else { InsertionPoint = & MI ; IPChange = NewInfo -> Change ; NewInfo -> Change = NewInfo -> Change . merge ( InstrMode ) ; }


<s> } else if ( BlockInfo [ PredBlock ] -> ExitSet ) { BlockInfo [ ThisBlock ] -> Pred = BlockInfo [ PredBlock ] -> Exit ; ExitSet = true ; } else if ( PredBlock != ThisBlock ) RevisitRequired = true ; for ( P = std :: next ( P ) ; P != E ; P = std :: next ( P ) ) { MachineBasicBlock * Pred = * P ; unsigned PredBlock = Pred -> getNumber ( ) ; if ( BlockInfo [ PredBlock ] -> ExitSet ) { if ( BlockInfo [ ThisBlock ] -> ExitSet ) { BlockInfo [ ThisBlock ] -> Pred = BlockInfo [ ThisBlock ] -> Pred . intersect ( BlockInfo [ PredBlock ] -> Exit ) ; } else { BlockInfo [ ThisBlock ] -> Pred = BlockInfo [ PredBlock ] -> Exit ; } ExitSet = true ; } else if ( PredBlock != ThisBlock ) RevisitRequired = true ; } } Status TmpStatus = BlockInfo [ ThisBlock ] -> Pred . merge ( BlockInfo [ ThisBlock ] -> Change ) ;

if ( BlockInfo [ ThisBlock ] -> Exit != TmpStatus ) {


<s> bool <ISA_LIT>ModeRegister :: runOnMachineFunction ( MachineFunction & MF ) { BlockInfo . resize ( MF . getNumBlockIDs ( ) ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; for ( MachineBasicBlock & BB : MF ) processBlockPhase1 ( BB , TII ) ;

for ( MachineBasicBlock & BB : MF ) Phase2List . push ( & BB ) ;


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; for ( MachineBasicBlock & BB : MF ) processBlockPhase1 ( BB , TII ) ; for ( MachineBasicBlock & BB : MF ) Phase2List . push ( & BB ) ; while ( ! Phase2List . empty ( ) ) { processBlockPhase2 ( * Phase2List . front ( ) , TII ) ; Phase2List . pop ( ) ; }

for ( MachineBasicBlock & BB : MF ) processBlockPhase3 ( BB , TII ) ;


<s> void <ISA_LIT>ModeRegister :: processBlockPhase1 ( MachineBasicBlock & MBB , const <ISA_LIT>InstrInfo * TII ) { auto NewInfo = std :: make_unique < BlockData > ( ) ; MachineInstr * InsertionPoint = nullptr ; bool RequirePending = true ; Status IPChange ; for ( MachineInstr & MI : MBB ) { Status InstrMode = getInstructionMode ( MI , TII ) ; if ( ( MI . getOpcode ( ) == <STR_LIT> ) || ( MI . getOpcode ( ) == <STR_LIT> ) ) { unsigned Dst = TII -> getNamedOperand ( MI , <STR_LIT> :: simm16 ) -> getImm ( ) ; if ( ( ( Dst & <STR_LIT> :: ID_MASK_ ) >> <STR_LIT> :: ID_SHIFT_ ) != <STR_LIT> :: ID_MODE ) continue ; unsigned Width = ( ( Dst & <STR_LIT> :: WIDTH_M1_MASK_ ) >> <STR_LIT> :: WIDTH_M1_SHIFT_ ) + <NUM_LIT> ; unsigned Offset = ( Dst & <STR_LIT> :: OFFSET_MASK_ ) >> <STR_LIT> :: OFFSET_SHIFT_ ; unsigned Mask = ( ( <NUM_LIT> << Width ) - <NUM_LIT> ) << Offset ; if ( InsertionPoint ) { insertSetreg ( MBB , InsertionPoint , TII , IPChange . delta ( NewInfo -> Change ) ) ; InsertionPoint = nullptr ; } if ( MI . getOpcode ( ) == <STR_LIT> ) { unsigned Val = TII -> getNamedOperand ( MI , <STR_LIT> :: imm ) -> getImm ( ) ; unsigned Mode = ( Val << Offset ) & Mask ; Status Setreg = Status ( Mask , Mode ) ; RequirePending = false ; NewInfo -> Change = NewInfo -> Change . merge ( Setreg ) ; } else { NewInfo -> Change = NewInfo -> Change . mergeUnknown ( Mask ) ; } } else if ( ! NewInfo -> Change . isCompatible ( InstrMode ) ) { if ( InsertionPoint ) { if ( ! IPChange . delta ( NewInfo -> Change ) . isCombinable ( InstrMode ) ) { if ( RequirePending ) { NewInfo -> FirstInsertionPoint = InsertionPoint ; NewInfo -> Require = NewInfo -> Change ; RequirePending = false ; } else { insertSetreg ( MBB , InsertionPoint , TII , IPChange . delta ( NewInfo -> Change ) ) ; IPChange = NewInfo -> Change ; } InsertionPoint = & MI ; } NewInfo -> Change = NewInfo -> Change . merge ( InstrMode ) ; } else { InsertionPoint = & MI ; IPChange = NewInfo -> Change ; NewInfo -> Change = NewInfo -> Change . merge ( InstrMode ) ; } } } if ( RequirePending ) { NewInfo -> FirstInsertionPoint = InsertionPoint ; NewInfo -> Require = NewInfo -> Change ; } else if ( InsertionPoint ) { insertSetreg ( MBB , InsertionPoint , TII , IPChange . delta ( NewInfo -> Change ) ) ; } NewInfo -> Exit = NewInfo -> Change ; BlockInfo [ MBB . getNumber ( ) ] = std :: move ( NewInfo ) ;


<s> return Status ( FP_ROUND_MODE_DP ( <NUM_LIT> ) , FP_ROUND_MODE_DP ( FP_ROUND_ROUND_TO_INF ) ) ; } case <STR_LIT> : { MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; return Status ( FP_ROUND_MODE_DP ( <NUM_LIT> ) , FP_ROUND_MODE_DP ( FP_ROUND_ROUND_TO_NEGINF ) ) ; } default : return DefaultStatus ; } }


<s> return new <ISA_LIT>ModeRegister ( ) ;


<s> FunctionPass * llvm :: create<ISA_LIT>ModeRegisterPass ( ) { return new <ISA_LIT>ModeRegister ( ) ;


<s> Status delta ( const Status & S ) const {

return Status ( ( S . Mask & ( Mode ^ S . Mode ) ) | ( ~ Mask & S . Mask ) , S . Mode ) ;


<s> if ( TII -> usesFPDPRounding ( MI ) ) { switch ( MI . getOpcode ( ) ) { case <STR_LIT> :


<s> case <STR_LIT> : case <STR_LIT> : return Status ( FP_ROUND_MODE_DP ( <NUM_LIT> ) , FP_ROUND_MODE_DP ( FP_ROUND_ROUND_TO_ZERO ) ) ; default : return DefaultStatus ; } }


<s> BuildMI ( MBB , MI , <NUM_LIT> , TII -> get ( <STR_LIT> ) ) . addImm ( Value ) . addImm ( ( ( Width - <NUM_LIT> ) << <STR_LIT> :: WIDTH_M1_SHIFT_ ) | ( Offset << <STR_LIT> :: OFFSET_SHIFT_ ) | ( <STR_LIT> :: ID_MODE << <STR_LIT> :: ID_SHIFT_ ) ) ; ++ NumSetregInserted ; InstrMode . Mask &= ~ ( ( ( <NUM_LIT> << Width ) - <NUM_LIT> ) << Offset ) ;


<s> unsigned Value = ( InstrMode . Mode >> Offset ) & ( ( <NUM_LIT> << Width ) - <NUM_LIT> ) ; BuildMI ( MBB , MI , <NUM_LIT> , TII -> get ( <STR_LIT> ) ) . addImm ( Value ) . addImm ( ( ( Width - <NUM_LIT> ) << <STR_LIT> :: WIDTH_M1_SHIFT_ ) | ( Offset << <STR_LIT> :: OFFSET_SHIFT_ ) | ( <STR_LIT> :: ID_MODE << <STR_LIT> :: ID_SHIFT_ ) ) ; ++ NumSetregInserted ; InstrMode . Mask &= ~ ( ( ( <NUM_LIT> << Width ) - <NUM_LIT> ) << Offset ) ;


<s> Status intersect ( const Status & S ) const {

unsigned NewMask = ( Mask & S . Mask ) & ( Mode ^ ~ S . Mode ) ;


<s> bool isCombinable ( Status & S ) { return ! ( Mask & S . Mask ) || isCompatible ( S ) ;


<s> bool isCompatible ( Status & S ) { return ( ( Mask & S . Mask ) == S . Mask ) && ( ( Mode & S . Mask ) == S . Mode ) ;


<s> Status merge ( const Status & S ) const { return Status ( ( Mask | S . Mask ) , ( ( Mode & ~ S . Mask ) | ( S . Mode & S . Mask ) ) ) ;


<s> Status mergeUnknown ( unsigned newMask ) { return Status ( Mask & ~ newMask , Mode & ~ newMask ) ;


<s> if ( BlockInfo [ ThisBlock ] -> Exit != TmpStatus ) { BlockInfo [ ThisBlock ] -> Exit = TmpStatus ; for ( MachineBasicBlock :: succ_iterator S = MBB . succ_begin ( ) , E = MBB . succ_end ( ) ; S != E ; S = std :: next ( S ) ) { MachineBasicBlock & B = * ( * S ) ; Phase2List . push ( & B ) ; } }


<s> if ( ! BlockInfo [ ThisBlock ] -> Pred . isCompatible ( BlockInfo [ ThisBlock ] -> Require ) ) { Status Delta = BlockInfo [ ThisBlock ] -> Pred . delta ( BlockInfo [ ThisBlock ] -> Require ) ; if ( BlockInfo [ ThisBlock ] -> FirstInsertionPoint ) insertSetreg ( MBB , BlockInfo [ ThisBlock ] -> FirstInsertionPoint , TII , Delta ) ; else insertSetreg ( MBB , & MBB . instr_front ( ) , TII , Delta ) ; }


<s> while ( ! Phase2List . empty ( ) ) { processBlockPhase2 ( * Phase2List . front ( ) , TII ) ; Phase2List . pop ( ) ; } for ( MachineBasicBlock & BB : MF ) processBlockPhase3 ( BB , TII ) ; BlockInfo . clear ( ) ;


<s> <ISA_LIT>ModeRegister ( ) : MachineFunctionPass ( ID ) {


<s> Status ( unsigned Mask , unsigned Mode ) : Mask ( Mask ) , Mode ( Mode ) { Mode &= Mask ;


<s> Status ( unsigned Mask , unsigned Mode ) : Mask ( Mask ) , Mode ( Mode ) {


<s> for ( ++ A ; CurrentIteration < MaxInstructions && A != E ; ++ A ) { if ( A -> isDebugInstr ( ) ) continue ; if ( Pred ( & * A ) ) return & * A ; for ( MCRegister Reg : NonModifiableRegs ) { if ( A -> modifiesRegister ( Reg , TRI ) ) return nullptr ; } ++ CurrentIteration ; }


<s> Register SaveExecDest = SaveExec . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRReg ( MRI , SaveExecDest ) ) return nullptr ; MachineOperand * SaveExecSrc0 = TII -> getNamedOperand ( SaveExec , <STR_LIT> :: src0 ) ; if ( ! SaveExecSrc0 -> isReg ( ) ) return nullptr ; VCmp = findInstrBackwards ( SaveExec , [ & ] ( MachineInstr * Check ) { return <STR_LIT> ( Check -> getOpcode ( ) ) != - <NUM_LIT> && Check -> modifiesRegister ( SaveExecSrc0 -> getReg ( ) , TRI ) ; } , { Exec , SaveExecSrc0 -> getReg ( ) } , TRI ) ; if ( ! VCmp ) return nullptr ; MachineOperand * VCmpDest = TII -> getNamedOperand ( * VCmp , <STR_LIT> :: sdst ) ; assert ( VCmpDest && <STR_LIT> ) ; MachineOperand * Src0 = TII -> getNamedOperand ( * VCmp , <STR_LIT> :: src0 ) ;

if ( Src0 -> isReg ( ) && TRI -> isSGPRReg ( MRI , Src0 -> getReg ( ) ) && SaveExec . modifiesRegister ( Src0 -> getReg ( ) , TRI ) ) return nullptr ;


<s> MachineOperand * VCmpDest = TII -> getNamedOperand ( * VCmp , <STR_LIT> :: sdst ) ; assert ( VCmpDest && <STR_LIT> ) ; MachineOperand * Src0 = TII -> getNamedOperand ( * VCmp , <STR_LIT> :: src0 ) ; if ( Src0 -> isReg ( ) && TRI -> isSGPRReg ( MRI , Src0 -> getReg ( ) ) && SaveExec . modifiesRegister ( Src0 -> getReg ( ) , TRI ) ) return nullptr ; MachineOperand * Src1 = TII -> getNamedOperand ( * VCmp , <STR_LIT> :: src1 ) ; if ( Src1 -> isReg ( ) && TRI -> isSGPRReg ( MRI , Src1 -> getReg ( ) ) && SaveExec . modifiesRegister ( Src1 -> getReg ( ) , TRI ) ) return nullptr ;

if ( isLiveOut ( * VCmp -> getParent ( ) , VCmpDest -> getReg ( ) ) ) return nullptr ;


<s> LivePhysRegs LR ( * TRI ) ; if ( useLiveOuts ) LR . addLiveOuts ( * Stop . getParent ( ) ) ; MachineBasicBlock :: reverse_iterator A ( Start ) ; MachineBasicBlock :: reverse_iterator E ( Stop ) ; if ( ignoreStart ) ++ A ; for ( ; A != Stop . getParent ( ) -> rend ( ) && A != Stop ; ++ A ) { LR . stepBackward ( * A ) ;


<s> static MachineBasicBlock :: reverse_iterator findExecCopy ( const <ISA_LIT>InstrInfo & TII , MachineBasicBlock & MBB , MachineBasicBlock :: reverse_iterator I , unsigned CopyToExec ) {


<s> for ( ; I != E ; ++ I ) { if ( ! I -> isTerminator ( ) ) return I ; if ( removeTerminatorBit ( TII , * I ) ) return I ; }


<s> MachineBasicBlock :: reverse_iterator I = MBB . rbegin ( ) , E = MBB . rend ( ) ; for ( ; I != E ; ++ I ) { if ( ! I -> isTerminator ( ) ) return I ;

if ( removeTerminatorBit ( TII , * I ) ) return I ;


<s> static unsigned getSaveExecOp ( unsigned Opc ) { switch ( Opc ) { case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ;


<s> if ( Src . isReg ( ) && Src . getReg ( ) == <STR_LIT> ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; }


<s> static unsigned isCopyFromExec ( const MachineInstr & MI ) { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : {

const MachineOperand & Src = MI . getOperand ( <NUM_LIT> ) ;


<s> const MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; if ( Dst . isReg ( ) && Dst . getReg ( ) == <STR_LIT> && MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; break ; }


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : { const MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; if ( Dst . isReg ( ) && Dst . getReg ( ) == <STR_LIT> && MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const MachineOperand & Src1 = MI . getOperand ( <NUM_LIT> ) ; if ( Src1 . isReg ( ) && Src1 . getReg ( ) == <STR_LIT> ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const MachineOperand & Src2 = MI . getOperand ( <NUM_LIT> ) ;

if ( Src2 . isReg ( ) && Src2 . getReg ( ) == <STR_LIT> ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> if ( Src1 . isReg ( ) && Src1 . getReg ( ) == <STR_LIT> ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const MachineOperand & Src2 = MI . getOperand ( <NUM_LIT> ) ; if ( Src2 . isReg ( ) && Src2 . getReg ( ) == <STR_LIT> ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; }


<s> case <STR_LIT> : { MI . setDesc ( TII . get ( <STR_LIT> ) ) ; return true ; } case <STR_LIT> : { MI . setDesc ( TII . get ( <STR_LIT> ) ) ;


<s> auto CopyFromExecInst = findExecCopy ( * TII , MBB , I , CopyToExec ) ; if ( CopyFromExecInst == E ) { auto PrepareExecInst = std :: next ( I ) ; if ( PrepareExecInst == E ) continue ; if ( CopyToExecInst -> getOperand ( <NUM_LIT> ) . isKill ( ) && isLogicalOpOnExec ( * PrepareExecInst ) == CopyToExec ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * PrepareExecInst ) ; PrepareExecInst -> getOperand ( <NUM_LIT> ) . setReg ( <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * PrepareExecInst << '\n' ) ; CopyToExecInst -> eraseFromParent ( ) ; } continue ; } if ( isLiveOut ( MBB , CopyToExec ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ; } unsigned CopyFromExec = CopyFromExecInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * SaveExecInst = nullptr ; SmallVector < MachineInstr * , <NUM_LIT> > OtherUseInsts ; for ( MachineBasicBlock :: iterator J = std :: next ( CopyFromExecInst -> getIterator ( ) ) , JE = I -> getIterator ( ) ; J != JE ; ++ J ) { if ( SaveExecInst && J -> readsRegister ( <STR_LIT> , TRI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; SaveExecInst = nullptr ; break ; } bool ReadsCopyFromExec = J -> readsRegister ( CopyFromExec , TRI ) ; if ( J -> modifiesRegister ( CopyToExec , TRI ) ) { if ( SaveExecInst ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( CopyToExec , TRI ) << '\n' ) ; SaveExecInst = nullptr ; break ; } unsigned SaveExecOp = getSaveExecOp ( J -> getOpcode ( ) ) ; if ( SaveExecOp == <STR_LIT> ) break ; if ( ReadsCopyFromExec ) { SaveExecInst = & * J ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; continue ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } } else if ( ReadsCopyFromExec && ! SaveExecInst ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ;


<s> return true ; } case <STR_LIT> : { MI . setDesc ( TII . get ( <STR_LIT> ) ) ; return true ; } case <STR_LIT> : { MI . setDesc ( TII . get ( <STR_LIT> ) ) ; return true ; } case <STR_LIT> : { MI . setDesc ( TII . get ( <STR_LIT> ) ) ; return true ; } default : return false ;


<s> case <STR_LIT> : case <STR_LIT> : { const MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ;

if ( Dst . isReg ( ) && Dst . getReg ( ) == <STR_LIT> ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: reverse_iterator I = fixTerminators ( * TII , MBB ) ; MachineBasicBlock :: reverse_iterator E = MBB . rend ( ) ; if ( I == E ) continue ; unsigned CopyToExec = isCopyToExec ( * I ) ; if ( CopyToExec == <STR_LIT> ) continue ; auto CopyToExecInst = & * I ; auto CopyFromExecInst = findExecCopy ( * TII , MBB , I , CopyToExec ) ; if ( CopyFromExecInst == E ) { auto PrepareExecInst = std :: next ( I ) ; if ( PrepareExecInst == E ) continue ; if ( CopyToExecInst -> getOperand ( <NUM_LIT> ) . isKill ( ) && isLogicalOpOnExec ( * PrepareExecInst ) == CopyToExec ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * PrepareExecInst ) ; PrepareExecInst -> getOperand ( <NUM_LIT> ) . setReg ( <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * PrepareExecInst << '\n' ) ; CopyToExecInst -> eraseFromParent ( ) ; } continue ; } if ( isLiveOut ( MBB , CopyToExec ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ; } unsigned CopyFromExec = CopyFromExecInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * SaveExecInst = nullptr ; SmallVector < MachineInstr * , <NUM_LIT> > OtherUseInsts ; for ( MachineBasicBlock :: iterator J = std :: next ( CopyFromExecInst -> getIterator ( ) ) , JE = I -> getIterator ( ) ; J != JE ; ++ J ) { if ( SaveExecInst && J -> readsRegister ( <STR_LIT> , TRI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; SaveExecInst = nullptr ; break ; } bool ReadsCopyFromExec = J -> readsRegister ( CopyFromExec , TRI ) ; if ( J -> modifiesRegister ( CopyToExec , TRI ) ) { if ( SaveExecInst ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( CopyToExec , TRI ) << '\n' ) ; SaveExecInst = nullptr ; break ; } unsigned SaveExecOp = getSaveExecOp ( J -> getOpcode ( ) ) ; if ( SaveExecOp == <STR_LIT> ) break ;


<s> static bool removeTerminatorBit ( const <ISA_LIT>InstrInfo & TII , MachineInstr & MI ) { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : { bool RegSrc = MI . getOperand ( <NUM_LIT> ) . isReg ( ) ; MI . setDesc ( TII . get ( RegSrc ? <STR_LIT> : <STR_LIT> ) ) ; return true ; } case <STR_LIT> : { bool RegSrc = MI . getOperand ( <NUM_LIT> ) . isReg ( ) ; MI . setDesc ( TII . get ( RegSrc ? <STR_LIT> : <STR_LIT> ) ) ; return true ; } case <STR_LIT> : { MI . setDesc ( TII . get ( <STR_LIT> ) ) ; return true ; } case <STR_LIT> : { MI . setDesc ( TII . get ( <STR_LIT> ) ) ; return true ; } case <STR_LIT> : {


<s> return true ; } case <STR_LIT> : { MI . setDesc ( TII . get ( <STR_LIT> ) ) ; return true ; } case <STR_LIT> : { MI . setDesc ( TII . get ( <STR_LIT> ) ) ; return true ; } case <STR_LIT> : { MI . setDesc ( TII . get ( <STR_LIT> ) ) ; return true ; } case <STR_LIT> : { MI . setDesc ( TII . get ( <STR_LIT> ) ) ; return true ; } case <STR_LIT> : { MI . setDesc ( TII . get ( <STR_LIT> ) ) ; return true ; } case <STR_LIT> : { MI . setDesc ( TII . get ( <STR_LIT> ) ) ; return true ; }


<s> Register CopyFromExec = CopyFromExecInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * SaveExecInst = nullptr ; SmallVector < MachineInstr * , <NUM_LIT> > OtherUseInsts ; for ( MachineBasicBlock :: iterator J = std :: next ( CopyFromExecInst -> getIterator ( ) ) , JE = I -> getIterator ( ) ; J != JE ; ++ J ) { if ( SaveExecInst && J -> readsRegister ( Exec , TRI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; SaveExecInst = nullptr ; break ; } bool ReadsCopyFromExec = J -> readsRegister ( CopyFromExec , TRI ) ; if ( J -> modifiesRegister ( CopyToExec , TRI ) ) { if ( SaveExecInst ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( CopyToExec , TRI ) << '\n' ) ; SaveExecInst = nullptr ; break ; } unsigned SaveExecOp = getSaveExecOp ( J -> getOpcode ( ) ) ; if ( SaveExecOp == <STR_LIT> ) break ; if ( ReadsCopyFromExec ) { SaveExecInst = & * J ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; continue ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } } else if ( ReadsCopyFromExec && ! SaveExecInst ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } if ( SaveExecInst && J -> readsRegister ( CopyToExec , TRI ) ) { assert ( SaveExecInst != & * J ) ; OtherUseInsts . push_back ( & * J ) ; } } if ( ! SaveExecInst ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; MachineOperand & Src0 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand * OtherOp = nullptr ; if ( Src0 . isReg ( ) && Src0 . getReg ( ) == CopyFromExec ) { OtherOp = & Src1 ; } else if ( Src1 . isReg ( ) && Src1 . getReg ( ) == CopyFromExec ) { if ( ! SaveExecInst -> isCommutable ( ) ) break ; OtherOp = & Src0 ; } else llvm_unreachable ( <STR_LIT> ) ; CopyFromExecInst -> eraseFromParent ( ) ; auto InsPt = SaveExecInst -> getIterator ( ) ; const DebugLoc & DL = SaveExecInst -> getDebugLoc ( ) ; BuildMI ( MBB , InsPt , DL , TII -> get ( getSaveExecOp ( SaveExecInst -> getOpcode ( ) ) ) , CopyFromExec ) . addReg ( OtherOp -> getReg ( ) ) ; SaveExecInst -> eraseFromParent ( ) ; CopyToExecInst -> eraseFromParent ( ) ; for ( MachineInstr * OtherInst : OtherUseInsts ) {

OtherInst -> substituteRegister ( CopyToExec , Exec , <STR_LIT> , * TRI ) ;


<s> if ( J -> readsRegister ( CopyFromExec , TRI ) ) { SaveExecInst = & * J ; DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; continue ; } else { DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } } if ( SaveExecInst && J -> readsRegister ( CopyToExec , TRI ) ) { assert ( SaveExecInst != & * J ) ; OtherUseInsts . push_back ( & * J ) ; } } if ( ! SaveExecInst ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; MachineOperand & Src0 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand * OtherOp = nullptr ; if ( Src0 . isReg ( ) && Src0 . getReg ( ) == CopyFromExec ) { OtherOp = & Src1 ; } else if ( Src1 . isReg ( ) && Src1 . getReg ( ) == CopyFromExec ) { if ( ! SaveExecInst -> isCommutable ( ) ) break ; OtherOp = & Src0 ; } else llvm_unreachable ( <STR_LIT> ) ; CopyFromExecInst -> eraseFromParent ( ) ; auto InsPt = SaveExecInst -> getIterator ( ) ; const DebugLoc & DL = SaveExecInst -> getDebugLoc ( ) ; BuildMI ( MBB , InsPt , DL , TII -> get ( getSaveExecOp ( SaveExecInst -> getOpcode ( ) ) ) , CopyFromExec ) . addReg ( OtherOp -> getReg ( ) ) ;


<s> break ; } unsigned SaveExecOp = getSaveExecOp ( J -> getOpcode ( ) ) ; if ( SaveExecOp == <STR_LIT> ) break ; if ( ReadsCopyFromExec ) { SaveExecInst = & * J ; DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; continue ; } else { DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } } else if ( ReadsCopyFromExec && ! SaveExecInst ) { DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } if ( SaveExecInst && J -> readsRegister ( CopyToExec , TRI ) ) { assert ( SaveExecInst != & * J ) ; OtherUseInsts . push_back ( & * J ) ; } } if ( ! SaveExecInst ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; MachineOperand & Src0 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand * OtherOp = nullptr ; if ( Src0 . isReg ( ) && Src0 . getReg ( ) == CopyFromExec ) { OtherOp = & Src1 ; } else if ( Src1 . isReg ( ) && Src1 . getReg ( ) == CopyFromExec ) { if ( ! SaveExecInst -> isCommutable ( ) ) break ; OtherOp = & Src0 ; } else llvm_unreachable ( <STR_LIT> ) ; CopyFromExecInst -> eraseFromParent ( ) ; auto InsPt = SaveExecInst -> getIterator ( ) ; const DebugLoc & DL = SaveExecInst -> getDebugLoc ( ) ; BuildMI ( MBB , InsPt , DL , TII -> get ( getSaveExecOp ( SaveExecInst -> getOpcode ( ) ) ) , CopyFromExec ) . addReg ( OtherOp -> getReg ( ) ) ;