<s> InstructionCost GCNTTIImpl :: getIntrinsicInstrCost ( const IntrinsicCostAttributes & ICA , TTI :: TargetCostKind CostKind ) { if ( ICA . getID ( ) == <STR_LIT> ) return <NUM_LIT> ; if ( ! intrinsicHasPackedVectorBenefit ( ICA . getID ( ) ) ) return BaseT :: getIntrinsicInstrCost ( ICA , CostKind ) ; Type * RetTy = ICA . getReturnType ( ) ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , RetTy ) ; unsigned NElts = LT . second . isVector ( ) ? LT . second . getVectorNumElements ( ) : <NUM_LIT> ; <STR_LIT> SLT = LT . second . getScalarType ( ) . SimpleTy ; if ( SLT == <STR_LIT> ) return LT . first * NElts * get64BitInstrCost ( CostKind ) ; if ( ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) || ( ST -> hasPackedFP32Ops ( ) && SLT == <STR_LIT> ) ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; unsigned InstRate = getQuarterRateInstrCost ( CostKind ) ; switch ( ICA . getID ( ) ) { case <STR_LIT> :

InstRate = ST -> hasFastFMAF32 ( ) ? getHalfRateInstrCost ( CostKind ) : getQuarterRateInstrCost ( CostKind ) ;


<s> if ( AddrSpace == <ISA_LIT>AS :: PRIVATE_ADDRESS ) return <NUM_LIT> * ST -> getMaxPrivateElementSize ( ) ;


<s> return <NUM_LIT> ; } if ( AddrSpace == <ISA_LIT>AS :: PRIVATE_ADDRESS ) return <NUM_LIT> * ST -> getMaxPrivateElementSize ( ) ;


<s> unsigned GCNTTIImpl :: getLoadVectorFactor ( unsigned VF , unsigned LoadSize , unsigned ChainSizeInBytes , VectorType * VecTy ) const { unsigned VecRegBitWidth = VF * LoadSize ; if ( VecRegBitWidth > <NUM_LIT> && VecTy -> getScalarSizeInBits ( ) < <NUM_LIT> ) return <NUM_LIT> / LoadSize ; return VF ;


<s> unsigned GCNTTIImpl :: getMaximumVF ( unsigned ElemWidth , unsigned Opcode ) const { if ( Opcode == Instruction :: Load || Opcode == Instruction :: Store ) return <NUM_LIT> * <NUM_LIT> / ElemWidth ;

return ( ElemWidth == <NUM_LIT> && ST -> has16BitInsts ( ) ) ? <NUM_LIT> : ( ElemWidth == <NUM_LIT> && ST -> hasPackedFP32Ops ( ) ) ? <NUM_LIT> : <NUM_LIT> ;


<s> if ( Opcode == Instruction :: Load || Opcode == Instruction :: Store ) return <NUM_LIT> * <NUM_LIT> / ElemWidth ;


<s> unsigned GCNTTIImpl :: getMaxInterleaveFactor ( unsigned VF ) {

if ( VF == <NUM_LIT> ) return <NUM_LIT> ;


<s> if ( AtomicElementSize ) return Type :: getIntNTy ( Context , * AtomicElementSize * <NUM_LIT> ) ; unsigned MinAlign = std :: min ( SrcAlign , DestAlign ) ; if ( MinAlign == <NUM_LIT> ) return Type :: getInt16Ty ( Context ) ; if ( SrcAddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || SrcAddrSpace == <ISA_LIT>AS :: REGION_ADDRESS || DestAddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || DestAddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) { return FixedVectorType :: get ( Type :: getInt32Ty ( Context ) , <NUM_LIT> ) ; }


<s> Type * I64Ty = Type :: getInt64Ty ( Context ) ; while ( RemainingBytes >= <NUM_LIT> ) { OpsOut . push_back ( I64Ty ) ; RemainingBytes -= <NUM_LIT> ; } Type * I32Ty = Type :: getInt32Ty ( Context ) ; while ( RemainingBytes >= <NUM_LIT> ) { OpsOut . push_back ( I32Ty ) ; RemainingBytes -= <NUM_LIT> ; } } Type * I16Ty = Type :: getInt16Ty ( Context ) ; while ( RemainingBytes >= <NUM_LIT> ) { OpsOut . push_back ( I16Ty ) ; RemainingBytes -= <NUM_LIT> ; } Type * I8Ty = Type :: getInt8Ty ( Context ) ; while ( RemainingBytes ) {


<s> InstructionCost GCNTTIImpl :: getMinMaxReductionCost ( VectorType * Ty , VectorType * CondTy , bool IsUnsigned , TTI :: TargetCostKind CostKind ) { EVT OrigTy = TLI -> getValueType ( DL , Ty ) ;

if ( ! ST -> hasVOP3PInsts ( ) || OrigTy . getScalarSizeInBits ( ) != <NUM_LIT> ) return BaseT :: getMinMaxReductionCost ( Ty , CondTy , IsUnsigned , CostKind ) ;


<s> EVT OrigTy = TLI -> getValueType ( DL , Ty ) ;


<s> unsigned GCNTTIImpl :: getNumberOfRegisters ( unsigned RCID ) const { return <NUM_LIT> ;


<s> void GCNTTIImpl :: getPeelingPreferences ( Loop * L , ScalarEvolution & SE , TTI :: PeelingPreferences & PP ) { CommonTTI . getPeelingPreferences ( L , SE , PP ) ;


<s> void GCNTTIImpl :: getPeelingPreferences ( Loop * L , ScalarEvolution & SE , TTI :: PeelingPreferences & PP ) {

CommonTTI . getPeelingPreferences ( L , SE , PP ) ;


<s> TTI :: PopcntSupportKind getPopcntSupport ( unsigned TyWidth ) {

assert ( isPowerOf2_32 ( TyWidth ) && <STR_LIT> ) ;


<s> return <NUM_LIT> * TargetTransformInfo :: TCC_Basic ;


<s> case TargetTransformInfo :: RGK_FixedWidthVector : return TypeSize :: getFixed ( ST -> hasPackedFP32Ops ( ) ? <NUM_LIT> : <NUM_LIT> ) ; case TargetTransformInfo :: RGK_ScalableVector : return TypeSize :: getScalable ( <NUM_LIT> ) ; } llvm_unreachable ( <STR_LIT> ) ;


<s> return TypeSize :: getFixed ( <NUM_LIT> ) ; case TargetTransformInfo :: RGK_FixedWidthVector : return TypeSize :: getFixed ( ST -> hasPackedFP32Ops ( ) ? <NUM_LIT> : <NUM_LIT> ) ; case TargetTransformInfo :: RGK_ScalableVector : return TypeSize :: getScalable ( <NUM_LIT> ) ; } llvm_unreachable ( <STR_LIT> ) ;


<s> unsigned GCNTTIImpl :: getStoreVectorFactor ( unsigned VF , unsigned StoreSize , unsigned ChainSizeInBytes , VectorType * VecTy ) const { unsigned VecRegBitWidth = VF * StoreSize ; if ( VecRegBitWidth > <NUM_LIT> ) return <NUM_LIT> / StoreSize ; return VF ;


<s> unsigned GCNTTIImpl :: getStoreVectorFactor ( unsigned VF , unsigned StoreSize , unsigned ChainSizeInBytes , VectorType * VecTy ) const {

unsigned VecRegBitWidth = VF * StoreSize ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { auto * Ordering = dyn_cast < ConstantInt > ( Inst -> getArgOperand ( <NUM_LIT> ) ) ; auto * Volatile = dyn_cast < ConstantInt > ( Inst -> getArgOperand ( <NUM_LIT> ) ) ; if ( ! Ordering || ! Volatile ) return false ; unsigned OrderingVal = Ordering -> getZExtValue ( ) ; if ( OrderingVal > static_cast < unsigned > ( AtomicOrdering :: SequentiallyConsistent ) ) return false ; Info . PtrVal = Inst -> getArgOperand ( <NUM_LIT> ) ; Info . Ordering = static_cast < AtomicOrdering > ( OrderingVal ) ; Info . ReadMem = true ; Info . WriteMem = true ; Info . IsVolatile = ! Volatile -> isZero ( ) ; return true ; } default : return false ; }


<s> auto * Ordering = dyn_cast < ConstantInt > ( Inst -> getArgOperand ( <NUM_LIT> ) ) ; auto * Volatile = dyn_cast < ConstantInt > ( Inst -> getArgOperand ( <NUM_LIT> ) ) ; if ( ! Ordering || ! Volatile ) return false ; unsigned OrderingVal = Ordering -> getZExtValue ( ) ; if ( OrderingVal > static_cast < unsigned > ( AtomicOrdering :: SequentiallyConsistent ) ) return false ; Info . PtrVal = Inst -> getArgOperand ( <NUM_LIT> ) ; Info . Ordering = static_cast < AtomicOrdering > ( OrderingVal ) ; Info . ReadMem = true ; Info . WriteMem = true ; Info . IsVolatile = ! Volatile -> isZero ( ) ; return true ; } default :


<s> void GCNTTIImpl :: getUnrollingPreferences ( Loop * L , ScalarEvolution & SE , TTI :: UnrollingPreferences & UP , OptimizationRemarkEmitter * ORE ) {

CommonTTI . getUnrollingPreferences ( L , SE , UP , ORE ) ;


<s> switch ( Opcode ) { case Instruction :: ExtractElement : case Instruction :: InsertElement : { unsigned EltSize = DL . getTypeSizeInBits ( cast < VectorType > ( ValTy ) -> getElementType ( ) ) ; if ( EltSize < <NUM_LIT> ) { if ( EltSize == <NUM_LIT> && Index == <NUM_LIT> && ST -> has16BitInsts ( ) ) return <NUM_LIT> ; return BaseT :: getVectorInstrCost ( Opcode , ValTy , Index ) ; } return Index == ~ <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ; } default :


<s> unsigned EltSize = DL . getTypeSizeInBits ( cast < VectorType > ( ValTy ) -> getElementType ( ) ) ; if ( EltSize < <NUM_LIT> ) { if ( EltSize == <NUM_LIT> && Index == <NUM_LIT> && ST -> has16BitInsts ( ) ) return <NUM_LIT> ; return BaseT :: getVectorInstrCost ( Opcode , ValTy , Index ) ; } return Index == ~ <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ; }


<s> switch ( ID ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; default : return false ; }


<s> switch ( ID ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ;


<s> return false ; } const ExtractValueInst * ExtValue = dyn_cast < ExtractValueInst > ( V ) ; if ( ! ExtValue ) return false ; const CallInst * CI = dyn_cast < CallInst > ( ExtValue -> getOperand ( <NUM_LIT> ) ) ; if ( ! CI ) return false ; if ( const IntrinsicInst * Intrinsic = dyn_cast < IntrinsicInst > ( CI ) ) { switch ( Intrinsic -> getIntrinsicID ( ) ) { default : return false ; case <STR_LIT> : case <STR_LIT> : { ArrayRef < unsigned > Indices = ExtValue -> getIndices ( ) ;

return Indices . size ( ) == <NUM_LIT> && Indices [ <NUM_LIT> ] == <NUM_LIT> ;


<s> for ( auto & TC : TargetConstraints ) { if ( TC . Type != InlineAsm :: isOutput ) continue ; if ( TargetOutputIdx != - <NUM_LIT> && TargetOutputIdx != OutputIdx ++ ) continue ; TLI -> ComputeConstraintToUse ( TC , SDValue ( ) ) ; const TargetRegisterClass * RC = TLI -> getRegForInlineAsmConstraint ( TRI , TC . ConstraintCode , TC . ConstraintVT ) . second ; if ( ! RC || ! TRI -> isSGPRClass ( RC ) ) return true ; } return false ;


<s> if ( TargetOutputIdx != - <NUM_LIT> && TargetOutputIdx != OutputIdx ++ ) continue ; TLI -> ComputeConstraintToUse ( TC , SDValue ( ) ) ; const TargetRegisterClass * RC = TLI -> getRegForInlineAsmConstraint ( TRI , TC . ConstraintCode , TC . ConstraintVT ) . second ;

if ( ! RC || ! TRI -> isSGPRClass ( RC ) ) return true ;


<s> bool GCNTTIImpl :: isLegalToVectorizeLoadChain ( unsigned ChainSizeInBytes , Align Alignment , unsigned AddrSpace ) const {

return isLegalToVectorizeMemChain ( ChainSizeInBytes , Alignment , AddrSpace ) ;


<s> return isLegalToVectorizeMemChain ( ChainSizeInBytes , Alignment , AddrSpace ) ;


<s> if ( const IntrinsicInst * Intrinsic = dyn_cast < IntrinsicInst > ( V ) ) return <STR_LIT> ( Intrinsic -> getIntrinsicID ( ) ) ; if ( const CallInst * CI = dyn_cast < CallInst > ( V ) ) { if ( CI -> isInlineAsm ( ) ) return isInlineAsmSourceOfDivergence ( CI ) ; return true ; } if ( isa < InvokeInst > ( V ) ) return true ;


<s> case <STR_LIT> : { unsigned TrueAS = IntrID == <STR_LIT> ? <ISA_LIT>AS :: LOCAL_ADDRESS : <ISA_LIT>AS :: PRIVATE_ADDRESS ; unsigned NewAS = NewV -> getType ( ) -> getPointerAddressSpace ( ) ; LLVMContext & Ctx = NewV -> getType ( ) -> getContext ( ) ; ConstantInt * NewVal = ( TrueAS == NewAS ) ? ConstantInt :: getTrue ( Ctx ) : ConstantInt :: getFalse ( Ctx ) ; return NewVal ; } case <STR_LIT> : { unsigned OldAS = OldV -> getType ( ) -> getPointerAddressSpace ( ) ; unsigned NewAS = NewV -> getType ( ) -> getPointerAddressSpace ( ) ; Value * MaskOp = II -> getArgOperand ( <NUM_LIT> ) ; Type * MaskTy = MaskOp -> getType ( ) ; bool DoTruncate = false ; const GCNTargetMachine & TM = static_cast < const GCNTargetMachine & > ( getTLI ( ) -> getTargetMachine ( ) ) ; if ( ! TM . isNoopAddrSpaceCast ( OldAS , NewAS ) ) { if ( DL . getPointerSizeInBits ( OldAS ) != <NUM_LIT> || DL . getPointerSizeInBits ( NewAS ) != <NUM_LIT> ) return nullptr ; KnownBits Known = computeKnownBits ( MaskOp , DL , <NUM_LIT> , nullptr , II ) ; if ( Known . countMinLeadingOnes ( ) < <NUM_LIT> ) return nullptr ; DoTruncate = true ; } IRBuilder < > B ( II ) ; if ( DoTruncate ) { MaskTy = B . getInt32Ty ( ) ;

MaskOp = B . CreateTrunc ( MaskOp , MaskTy ) ;


<s> return AS != <ISA_LIT>AS :: LOCAL_ADDRESS && AS != <ISA_LIT>AS :: REGION_ADDRESS && AS != <ISA_LIT>AS :: PRIVATE_ADDRESS ;


<s> unsigned getFlatAddressSpace ( ) const { if ( IsGraphics ) return - <NUM_LIT> ; return <ISA_LIT>AS :: FLAT_ADDRESS ;


<s> static inline int getHalfRateInstrCost ( TTI :: TargetCostKind CostKind ) {

return CostKind == TTI :: TCK_CodeSize ? <NUM_LIT> : <NUM_LIT> * TargetTransformInfo :: TCC_Basic ;


<s> int getInlinerVectorBonusPercent ( ) { return <NUM_LIT> ;


<s> return CostKind == TTI :: TCK_CodeSize ? <NUM_LIT> : <NUM_LIT> * TargetTransformInfo :: TCC_Basic ;


<s> <ISA_LIT>TTIImpl ( <ISA_LIT>TTIImpl && Arg ) : BaseT ( std :: move ( static_cast < BaseT & > ( Arg ) ) ) , ST ( std :: move ( Arg . ST ) ) , TLI ( std :: move ( Arg . TLI ) ) {


<s> assert ( isPowerOf2_32 ( TyWidth ) && <STR_LIT> ) ;

return ST -> hasBCNT ( TyWidth ) ? TTI :: PSK_FastHardware : TTI :: PSK_Software ;


<s> return ST -> hasBCNT ( TyWidth ) ? TTI :: PSK_FastHardware : TTI :: PSK_Software ;


<s> const <ISA_LIT>Subtarget * getST ( ) const {


<s> const <ISA_LIT>TargetLowering * getTLI ( ) const { return TLI ;


<s> const GCNSubtarget * CalleeST = static_cast < const GCNSubtarget * > ( TM . getSubtargetImpl ( * Callee ) ) ; const FeatureBitset & CallerBits = CallerST -> getFeatureBits ( ) ; const FeatureBitset & CalleeBits = CalleeST -> getFeatureBits ( ) ;

FeatureBitset RealCallerBits = CallerBits & ~ InlineFeatureIgnoreList ;


<s> return BaseT :: getArithmeticInstrCost ( Opcode , Ty , Opd1Info , Opd2Info , Opd1PropInfo , Opd2PropInfo ) ; } std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Ty ) ; int ISD = TLI -> InstructionOpcodeToISD ( Opcode ) ; unsigned NElts = LT . second . isVector ( ) ? LT . second . getVectorNumElements ( ) : <NUM_LIT> ; <STR_LIT> SLT = LT . second . getScalarType ( ) . SimpleTy ; switch ( ISD ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) return get64BitInstrCost ( ) * LT . first * NElts ; if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; return getFullRateInstrCost ( ) * LT . first * NElts ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) { return <NUM_LIT> * getFullRateInstrCost ( ) * LT . first * NElts ; } if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; return LT . first * NElts * getFullRateInstrCost ( ) ; case <STR_LIT> : { const int QuarterRateCost = getQuarterRateInstrCost ( ) ; if ( SLT == <STR_LIT> ) { const int FullRateCost = getFullRateInstrCost ( ) ; return ( <NUM_LIT> * QuarterRateCost + ( <NUM_LIT> * <NUM_LIT> ) * FullRateCost ) * LT . first * NElts ; } if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; return QuarterRateCost * NElts * LT . first ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) return LT . first * NElts * get64BitInstrCost ( ) ; if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; if ( SLT == <STR_LIT> || SLT == <STR_LIT> ) return LT . first * NElts * getFullRateInstrCost ( ) ; break ; case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) { int Cost = <NUM_LIT> * get64BitInstrCost ( ) + <NUM_LIT> * getQuarterRateInstrCost ( ) ; if ( ! ST -> hasUsableDivScaleConditionOutput ( ) ) Cost += <NUM_LIT> * getFullRateInstrCost ( ) ;


<s> EVT OrigTy = TLI -> getValueType ( DL , Ty ) ; if ( ! OrigTy . isSimple ( ) ) { return BaseT :: getArithmeticInstrCost ( Opcode , Ty , Opd1Info , Opd2Info , Opd1PropInfo , Opd2PropInfo ) ; } std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Ty ) ; int ISD = TLI -> InstructionOpcodeToISD ( Opcode ) ; unsigned NElts = LT . second . isVector ( ) ? LT . second . getVectorNumElements ( ) : <NUM_LIT> ; <STR_LIT> SLT = LT . second . getScalarType ( ) . SimpleTy ; switch ( ISD ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) return get64BitInstrCost ( ) * LT . first * NElts ; if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; return getFullRateInstrCost ( ) * LT . first * NElts ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) { return <NUM_LIT> * getFullRateInstrCost ( ) * LT . first * NElts ; } if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; return LT . first * NElts * getFullRateInstrCost ( ) ; case <STR_LIT> : { const int QuarterRateCost = getQuarterRateInstrCost ( ) ; if ( SLT == <STR_LIT> ) { const int FullRateCost = getFullRateInstrCost ( ) ; return ( <NUM_LIT> * QuarterRateCost + ( <NUM_LIT> * <NUM_LIT> ) * FullRateCost ) * LT . first * NElts ; }


<s> if ( IsPairwise || ! ST -> hasVOP3PInsts ( ) || OrigTy . getScalarSizeInBits ( ) != <NUM_LIT> ) return BaseT :: getArithmeticReductionCost ( Opcode , Ty , IsPairwise ) ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Ty ) ; return LT . first * getFullRateInstrCost ( ) ;


<s> int GCNTTIImpl :: getIntrinsicInstrCost ( <STR_LIT> ID , Type * RetTy , ArrayRef < Type * > Tys , FastMathFlags FMF , unsigned ScalarizationCostPassed ) { return getIntrinsicInstrCost < Type > ( ID , RetTy , Tys , FMF , ScalarizationCostPassed ) ;


<s> if ( AddrSpace == <ISA_LIT>AS :: FLAT_ADDRESS || AddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || AddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) return <NUM_LIT> ; if ( AddrSpace == <ISA_LIT>AS :: PRIVATE_ADDRESS ) return <NUM_LIT> * ST -> getMaxPrivateElementSize ( ) ;


<s> std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Ty ) ; return LT . first * getHalfRateInstrCost ( ) ;


<s> void <ISA_LIT>TTIImpl :: getUnrollingPreferences ( Loop * L , ScalarEvolution & SE , TTI :: UnrollingPreferences & UP ) { const Function & F = * L -> getHeader ( ) -> getParent ( ) ; UP . Threshold = <STR_LIT> ( F , <STR_LIT> , <NUM_LIT> ) ; UP . MaxCount = std :: numeric_limits < unsigned > :: max ( ) ; UP . Partial = true ; const unsigned MaxAlloca = ( <NUM_LIT> - <NUM_LIT> ) * <NUM_LIT> ; unsigned ThresholdPrivate = UnrollThresholdPrivate ; unsigned ThresholdLocal = UnrollThresholdLocal ; unsigned MaxBoost = std :: max ( ThresholdPrivate , ThresholdLocal ) ; for ( const BasicBlock * BB : L -> getBlocks ( ) ) { const DataLayout & DL = BB -> getModule ( ) -> getDataLayout ( ) ; unsigned LocalGEPsSeen = <NUM_LIT> ; if ( llvm :: any_of ( L -> getSubLoops ( ) , [ BB ] ( const Loop * SubLoop ) { return SubLoop -> contains ( BB ) ; } ) ) continue ; for ( const Instruction & I : * BB ) { if ( const BranchInst * Br = dyn_cast < BranchInst > ( & I ) ) { if ( UP . Threshold < MaxBoost && Br -> isConditional ( ) ) { BasicBlock * Succ0 = Br -> getSuccessor ( <NUM_LIT> ) ; BasicBlock * Succ1 = Br -> getSuccessor ( <NUM_LIT> ) ; if ( ( L -> contains ( Succ0 ) && L -> isLoopExiting ( Succ0 ) ) || ( L -> contains ( Succ1 ) && L -> isLoopExiting ( Succ1 ) ) ) continue ; if ( dependsOnLocalPhi ( L , Br -> getCondition ( ) ) ) { UP . Threshold += UnrollThresholdIf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << UP . Threshold << <STR_LIT> << * L << <STR_LIT> << * Br << '\n' ) ; if ( UP . Threshold >= MaxBoost ) return ; } } continue ; } const GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( & I ) ; if ( ! GEP ) continue ; unsigned AS = GEP -> getAddressSpace ( ) ; unsigned Threshold = <NUM_LIT> ; if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) Threshold = ThresholdPrivate ; else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: REGION_ADDRESS ) Threshold = ThresholdLocal ; else continue ; if ( UP . Threshold >= Threshold ) continue ; if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { const Value * Ptr = GEP -> getPointerOperand ( ) ;

const AllocaInst * Alloca = dyn_cast < AllocaInst > ( GetUnderlyingObject ( Ptr , DL ) ) ;


<s> if ( const LoadInst * Load = dyn_cast < LoadInst > ( V ) ) return Load -> getPointerAddressSpace ( ) == <ISA_LIT>AS :: PRIVATE_ADDRESS || Load -> getPointerAddressSpace ( ) == <ISA_LIT>AS :: FLAT_ADDRESS ;

if ( isa < AtomicRMWInst > ( V ) || isa < AtomicCmpXchgInst > ( V ) ) return true ;


<s> bool GCNTTIImpl :: isSourceOfDivergence ( const Value * V ) const { if ( const Argument * A = dyn_cast < Argument > ( V ) ) return ! isArgPassedInSGPR ( A ) ; if ( const LoadInst * Load = dyn_cast < LoadInst > ( V ) ) return Load -> getPointerAddressSpace ( ) == <ISA_LIT>AS :: PRIVATE_ADDRESS || Load -> getPointerAddressSpace ( ) == <ISA_LIT>AS :: FLAT_ADDRESS ; if ( isa < AtomicRMWInst > ( V ) || isa < AtomicCmpXchgInst > ( V ) ) return true ; if ( const IntrinsicInst * Intrinsic = dyn_cast < IntrinsicInst > ( V ) ) return <STR_LIT> ( Intrinsic -> getIntrinsicID ( ) ) ;

if ( isa < CallInst > ( V ) || isa < InvokeInst > ( V ) ) return true ;


<s> case <STR_LIT> : { unsigned TrueAS = IntrID == <STR_LIT> ? <ISA_LIT>AS :: LOCAL_ADDRESS : <ISA_LIT>AS :: PRIVATE_ADDRESS ; unsigned NewAS = NewV -> getType ( ) -> getPointerAddressSpace ( ) ; LLVMContext & Ctx = NewV -> getType ( ) -> getContext ( ) ; ConstantInt * NewVal = ( TrueAS == NewAS ) ? ConstantInt :: getTrue ( Ctx ) : ConstantInt :: getFalse ( Ctx ) ; II -> replaceAllUsesWith ( NewVal ) ; II -> eraseFromParent ( ) ; return true ;


<s> if ( const PHINode * PHI = dyn_cast < PHINode > ( V ) ) { if ( none_of ( L -> getSubLoops ( ) , [ PHI ] ( const Loop * SubLoop ) {

return SubLoop -> contains ( PHI ) ; } ) ) return true ;


<s> case <STR_LIT> : { if ( SLT == <STR_LIT> ) { return <NUM_LIT> * getFullRateInstrCost ( ) * LT . first * NElts ; } return LT . first * NElts * getFullRateInstrCost ( ) ; } case <STR_LIT> : { const int QuarterRateCost = getQuarterRateInstrCost ( ) ; if ( SLT == <STR_LIT> ) { const int FullRateCost = getFullRateInstrCost ( ) ; return ( <NUM_LIT> * QuarterRateCost + ( <NUM_LIT> * <NUM_LIT> ) * FullRateCost ) * LT . first * NElts ; } return QuarterRateCost * NElts * LT . first ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) return LT . first * NElts * get64BitInstrCost ( ) ; if ( SLT == <STR_LIT> || SLT == <STR_LIT> ) return LT . first * NElts * getFullRateInstrCost ( ) ; break ; case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) { int Cost = <NUM_LIT> * get64BitInstrCost ( ) + <NUM_LIT> * getQuarterRateInstrCost ( ) ; if ( ST -> getGeneration ( ) == <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) Cost += <NUM_LIT> * getFullRateInstrCost ( ) ; return LT . first * Cost * NElts ; } if ( SLT == <STR_LIT> || SLT == <STR_LIT> ) {

assert ( ! ST -> hasFP32Denormals ( ) && <STR_LIT> ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getCFInstrCost ( unsigned Opcode ) { switch ( Opcode ) { case Instruction :: Br : case Instruction :: Ret :


return BaseT :: getCFInstrCost ( Opcode ) ;


<s> if ( AddrSpace == AS . PRIVATE_ADDRESS ) return <NUM_LIT> * ST -> getMaxPrivateElementSize ( ) ; if ( ST -> getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS && ( AddrSpace == AS . PARAM_D_ADDRESS || AddrSpace == AS . PARAM_I_ADDRESS || ( AddrSpace >= AS . CONSTANT_BUFFER_0 && AddrSpace <= AS . CONSTANT_BUFFER_15 ) ) ) return <NUM_LIT> ; llvm_unreachable ( <STR_LIT> ) ;


<s> if ( AddrSpace == AS . PRIVATE_ADDRESS ) return <NUM_LIT> * ST -> getMaxPrivateElementSize ( ) ;

if ( ST -> getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS && ( AddrSpace == AS . PARAM_D_ADDRESS || AddrSpace == AS . PARAM_I_ADDRESS || ( AddrSpace >= AS . CONSTANT_BUFFER_0 && AddrSpace <= AS . CONSTANT_BUFFER_15 ) ) ) return <NUM_LIT> ;


<s> if ( Vec ) return <NUM_LIT> ; if ( ST -> getGeneration ( ) >= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return <NUM_LIT> ; return <NUM_LIT> * <NUM_LIT> ;


if ( ST -> getGeneration ( ) >= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return <NUM_LIT> ;


<s> unsigned <ISA_LIT>TTIImpl :: getRegisterBitWidth ( bool Vector ) const {


<s> if ( VT -> getNumElements ( ) == <NUM_LIT> && DL . getTypeSizeInBits ( VT -> getElementType ( ) ) == <NUM_LIT> ) { switch ( Kind ) { case TTI :: SK_Broadcast : case TTI :: SK_Reverse : case TTI :: SK_PermuteSingleSrc :


<s> else continue ; if ( UP . Threshold >= Threshold ) continue ; if ( AS == ASST . PRIVATE_ADDRESS ) { const Value * Ptr = GEP -> getPointerOperand ( ) ; const AllocaInst * Alloca = dyn_cast < AllocaInst > ( GetUnderlyingObject ( Ptr , DL ) ) ; if ( ! Alloca || ! Alloca -> isStaticAlloca ( ) ) continue ; Type * Ty = Alloca -> getAllocatedType ( ) ; unsigned AllocaSize = Ty -> isSized ( ) ? DL . getTypeAllocSize ( Ty ) : <NUM_LIT> ; if ( AllocaSize > MaxAlloca ) continue ; } else if ( AS == ASST . LOCAL_ADDRESS ) { LocalGEPsSeen ++ ; if ( LocalGEPsSeen > <NUM_LIT> || L -> getLoopDepth ( ) > <NUM_LIT> || ( ! isa < GlobalVariable > ( GEP -> getPointerOperand ( ) ) && ! isa < Argument > ( GEP -> getPointerOperand ( ) ) ) ) continue ; } bool HasLoopDef = false ; for ( const Value * Op : GEP -> operands ( ) ) { const Instruction * Inst = dyn_cast < Instruction > ( Op ) ; if ( ! Inst || L -> isLoopInvariant ( Op ) ) continue ; if ( any_of ( L -> getSubLoops ( ) , [ Inst ] ( const Loop * SubLoop ) { return SubLoop -> contains ( Inst ) ; } ) ) continue ; HasLoopDef = true ; break ;


<s> unsigned EltSize = DL . getTypeSizeInBits ( cast < VectorType > ( ValTy ) -> getElementType ( ) ) ; if ( EltSize < <NUM_LIT> ) { if ( EltSize == <NUM_LIT> && Index == <NUM_LIT> && ST -> has16BitInsts ( ) ) return <NUM_LIT> ; return BaseT :: getVectorInstrCost ( Opcode , ValTy , Index ) ;


<s> switch ( Opcode ) { case Instruction :: ExtractElement : case Instruction :: InsertElement : { unsigned EltSize = DL . getTypeSizeInBits ( cast < VectorType > ( ValTy ) -> getElementType ( ) ) ; if ( EltSize < <NUM_LIT> ) { if ( EltSize == <NUM_LIT> && Index == <NUM_LIT> && ST -> has16BitInsts ( ) ) return <NUM_LIT> ; return BaseT :: getVectorInstrCost ( Opcode , ValTy , Index ) ; } return Index == ~ <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ; } default : return BaseT :: getVectorInstrCost ( Opcode , ValTy , Index ) ; }


<s> if ( const IntrinsicInst * Intrinsic = dyn_cast < IntrinsicInst > ( V ) ) { switch ( Intrinsic -> getIntrinsicID ( ) ) { default :


<s> switch ( CC ) { case CallingConv :: <ISA_LIT>_KERNEL : case CallingConv :: SPIR_KERNEL : return true ; case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_HS : case CallingConv :: <ISA_LIT>_GS : case CallingConv :: <ISA_LIT>_PS :


<s> return isLegalToVectorizeMemChain ( ChainSizeInBytes , Alignment , AddrSpace ) ;


<s> bool <ISA_LIT>TTIImpl :: isLegalToVectorizeMemChain ( unsigned ChainSizeInBytes , unsigned Alignment , unsigned AddrSpace ) const {

if ( AddrSpace == ST -> get<ISA_LIT>AS ( ) . PRIVATE_ADDRESS ) {


<s> bool <ISA_LIT>TTIImpl :: isLegalToVectorizeStoreChain ( unsigned ChainSizeInBytes , unsigned Alignment , unsigned AddrSpace ) const {


<s> if ( const Argument * A = dyn_cast < Argument > ( V ) ) return ! isArgPassedInSGPR ( A ) ;

if ( const LoadInst * Load = dyn_cast < LoadInst > ( V ) ) return Load -> getPointerAddressSpace ( ) == ST -> get<ISA_LIT>AS ( ) . PRIVATE_ADDRESS ;


<s> if ( const IntrinsicInst * Intrinsic = dyn_cast < IntrinsicInst > ( V ) ) return isIntrinsicSourceOfDivergence ( Intrinsic ) ; if ( isa < CallInst > ( V ) || isa < InvokeInst > ( V ) ) return true ; return false ;


<s> const FeatureBitset & CallerBits = TM . getSubtargetImpl ( * Caller ) -> getFeatureBits ( ) ;

const FeatureBitset & CalleeBits = TM . getSubtargetImpl ( * Callee ) -> getFeatureBits ( ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) return get64BitInstrCost ( ) * LT . first * NElts ; return getFullRateInstrCost ( ) * LT . first * NElts ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) { return <NUM_LIT> * getFullRateInstrCost ( ) * LT . first * NElts ; } return LT . first * NElts * getFullRateInstrCost ( ) ; case <STR_LIT> : { const int QuarterRateCost = getQuarterRateInstrCost ( ) ; if ( SLT == <STR_LIT> ) { const int FullRateCost = getFullRateInstrCost ( ) ; return ( <NUM_LIT> * QuarterRateCost + ( <NUM_LIT> * <NUM_LIT> ) * FullRateCost ) * LT . first * NElts ; } return QuarterRateCost * NElts * LT . first ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) return LT . first * NElts * get64BitInstrCost ( ) ; if ( SLT == <STR_LIT> || SLT == <STR_LIT> ) return LT . first * NElts * getFullRateInstrCost ( ) ; break ; case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) { int Cost = <NUM_LIT> * get64BitInstrCost ( ) + <NUM_LIT> * getQuarterRateInstrCost ( ) ; if ( ST -> getGeneration ( ) == <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) Cost += <NUM_LIT> * getFullRateInstrCost ( ) ; return LT . first * Cost * NElts ; } if ( ! Args . empty ( ) && match ( Args [ <NUM_LIT> ] , PatternMatch :: m_FPOne ( ) ) ) { if ( ( SLT == <STR_LIT> && ! ST -> hasFP32Denormals ( ) ) || ( SLT == <STR_LIT> && ST -> has16BitInsts ( ) ) ) { return LT . first * getQuarterRateInstrCost ( ) * NElts ; } } if ( SLT == <STR_LIT> && ST -> has16BitInsts ( ) ) { int Cost = <NUM_LIT> * getFullRateInstrCost ( ) + <NUM_LIT> * getQuarterRateInstrCost ( ) ; return LT . first * Cost * NElts ; } if ( SLT == <STR_LIT> || SLT == <STR_LIT> ) { int Cost = <NUM_LIT> * getFullRateInstrCost ( ) + <NUM_LIT> * getQuarterRateInstrCost ( ) ;


<s> if ( ST -> getGeneration ( ) >= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return <NUM_LIT> ; return <NUM_LIT> * <NUM_LIT> ;


<s> unsigned <ISA_LIT>TTIImpl :: getHardwareNumberOfRegisters ( bool Vec ) const { if ( ST -> getGeneration ( ) >= <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return <NUM_LIT> ; return <NUM_LIT> * <NUM_LIT> ;


<s> unsigned <ISA_LIT>TTIImpl :: getMaxInterleaveFactor ( unsigned VF ) {

if ( VF == <NUM_LIT> ) return <NUM_LIT> ;


<s> if ( VF == <NUM_LIT> ) return <NUM_LIT> ;


<s> unsigned <ISA_LIT>TTIImpl :: getMinVectorRegisterBitWidth ( ) const { return <NUM_LIT> ;


<s> unsigned <ISA_LIT>TTIImpl :: getNumberOfRegisters ( bool Vec ) const { return getHardwareNumberOfRegisters ( Vec ) >> <NUM_LIT> ;


<s> unsigned <ISA_LIT>TTIImpl :: getRegisterBitWidth ( bool Vector ) const { return <NUM_LIT> ;


<s> Info . PtrVal = Inst -> getArgOperand ( <NUM_LIT> ) ; Info . Ordering = static_cast < AtomicOrdering > ( OrderingVal ) ; Info . ReadMem = true ; Info . WriteMem = true ; Info . IsVolatile = ! Volatile -> isNullValue ( ) ; return true ; } default : return false ; }


<s> const unsigned MaxAlloca = ( <NUM_LIT> - <NUM_LIT> ) * <NUM_LIT> ; unsigned ThresholdPrivate = UnrollThresholdPrivate ; unsigned ThresholdLocal = UnrollThresholdLocal ; unsigned MaxBoost = std :: max ( ThresholdPrivate , ThresholdLocal ) ; <ISA_LIT>AS ASST = ST -> get<ISA_LIT>AS ( ) ; for ( const BasicBlock * BB : L -> getBlocks ( ) ) { const DataLayout & DL = BB -> getModule ( ) -> getDataLayout ( ) ; unsigned LocalGEPsSeen = <NUM_LIT> ; if ( llvm :: any_of ( L -> getSubLoops ( ) , [ BB ] ( const Loop * SubLoop ) { return SubLoop -> contains ( BB ) ; } ) ) continue ; for ( const Instruction & I : * BB ) { if ( const BranchInst * Br = dyn_cast < BranchInst > ( & I ) ) { if ( UP . Threshold < MaxBoost && Br -> isConditional ( ) ) { if ( L -> isLoopExiting ( Br -> getSuccessor ( <NUM_LIT> ) ) || L -> isLoopExiting ( Br -> getSuccessor ( <NUM_LIT> ) ) ) continue ; if ( dependsOnLocalPhi ( L , Br -> getCondition ( ) ) ) { UP . Threshold += UnrollThresholdIf ; DEBUG ( dbgs ( ) << <STR_LIT> << UP . Threshold << <STR_LIT> << * L << <STR_LIT> << * Br << '\n' ) ; if ( UP . Threshold >= MaxBoost ) return ; } } continue ; } const GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( & I ) ; if ( ! GEP ) continue ; unsigned AS = GEP -> getAddressSpace ( ) ; unsigned Threshold = <NUM_LIT> ; if ( AS == ASST . PRIVATE_ADDRESS ) Threshold = ThresholdPrivate ;

else if ( AS == ASST . LOCAL_ADDRESS ) Threshold = ThresholdLocal ;


<s> } const GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( & I ) ; if ( ! GEP ) continue ; unsigned AS = GEP -> getAddressSpace ( ) ; unsigned Threshold = <NUM_LIT> ; if ( AS == ASST . PRIVATE_ADDRESS ) Threshold = ThresholdPrivate ; else if ( AS == ASST . LOCAL_ADDRESS ) Threshold = ThresholdLocal ; else continue ; if ( UP . Threshold >= Threshold ) continue ; if ( AS == ASST . PRIVATE_ADDRESS ) { const Value * Ptr = GEP -> getPointerOperand ( ) ; const AllocaInst * Alloca = dyn_cast < AllocaInst > ( GetUnderlyingObject ( Ptr , DL ) ) ; if ( ! Alloca || ! Alloca -> isStaticAlloca ( ) ) continue ; Type * Ty = Alloca -> getAllocatedType ( ) ; unsigned AllocaSize = Ty -> isSized ( ) ? DL . getTypeAllocSize ( Ty ) : <NUM_LIT> ; if ( AllocaSize > MaxAlloca ) continue ; } else if ( AS == ASST . LOCAL_ADDRESS ) { LocalGEPsSeen ++ ; if ( LocalGEPsSeen > <NUM_LIT> || L -> getLoopDepth ( ) > <NUM_LIT> || ( ! isa < GlobalVariable > ( GEP -> getPointerOperand ( ) ) && ! isa < Argument > ( GEP -> getPointerOperand ( ) ) ) ) continue ; } bool HasLoopDef = false ; for ( const Value * Op : GEP -> operands ( ) ) { const Instruction * Inst = dyn_cast < Instruction > ( Op ) ; if ( ! Inst || L -> isLoopInvariant ( Op ) ) continue ; if ( llvm :: any_of ( L -> getSubLoops ( ) , [ Inst ] ( const Loop * SubLoop ) { return SubLoop -> contains ( Inst ) ; } ) ) continue ; HasLoopDef = true ; break ; } if ( ! HasLoopDef ) continue ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) { return <NUM_LIT> * getFullRateInstrCost ( ) * LT . first * NElts ; } if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; return LT . first * NElts * getFullRateInstrCost ( ) ; case <STR_LIT> : { const int QuarterRateCost = getQuarterRateInstrCost ( CostKind ) ; if ( SLT == <STR_LIT> ) { const int FullRateCost = getFullRateInstrCost ( ) ; return ( <NUM_LIT> * QuarterRateCost + ( <NUM_LIT> * <NUM_LIT> ) * FullRateCost ) * LT . first * NElts ; } if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; return QuarterRateCost * NElts * LT . first ; } case <STR_LIT> : if ( CxtI && CxtI -> hasOneUse ( ) ) if ( const auto * FAdd = dyn_cast < BinaryOperator > ( * CxtI -> user_begin ( ) ) ) { const int OPC = TLI -> InstructionOpcodeToISD ( FAdd -> getOpcode ( ) ) ; if ( OPC == <STR_LIT> || OPC == <STR_LIT> ) { if ( ST -> hasMadMacF32Insts ( ) && SLT == <STR_LIT> && ! HasFP32Denormals ) return TargetTransformInfo :: TCC_Free ; if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> && ! HasFP64FP16Denormals ) return TargetTransformInfo :: TCC_Free ; const TargetOptions & Options = TLI -> getTargetMachine ( ) . Options ; if ( Options . AllowFPOpFusion == FPOpFusion :: Fast || Options . UnsafeFPMath || ( FAdd -> hasAllowContract ( ) && CxtI -> hasAllowContract ( ) ) ) return TargetTransformInfo :: TCC_Free ; } } LLVM_FALLTHROUGH ; case <STR_LIT> : case <STR_LIT> : if ( ST -> hasPackedFP32Ops ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; if ( SLT == <STR_LIT> ) return LT . first * NElts * get64BitInstrCost ( CostKind ) ; if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; if ( SLT == <STR_LIT> || SLT == <STR_LIT> ) return LT . first * NElts * getFullRateInstrCost ( ) ; break ; case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) { int Cost = <NUM_LIT> * get64BitInstrCost ( CostKind ) + getQuarterRateInstrCost ( CostKind ) + <NUM_LIT> * getHalfRateInstrCost ( CostKind ) ; if ( ! ST -> hasUsableDivScaleConditionOutput ( ) ) Cost += <NUM_LIT> * getFullRateInstrCost ( ) ; return LT . first * Cost * NElts ; } if ( ! Args . empty ( ) && match ( Args [ <NUM_LIT> ] , PatternMatch :: m_FPOne ( ) ) ) { if ( ( SLT == <STR_LIT> && ! HasFP32Denormals ) || ( SLT == <STR_LIT> && ST -> has16BitInsts ( ) ) ) {

return LT . first * getQuarterRateInstrCost ( CostKind ) * NElts ;


<s> unsigned GCNTTIImpl :: getHardwareNumberOfRegisters ( bool Vec ) const {


<s> if ( ! OrigTy . isSimple ( ) ) { if ( CostKind != TTI :: TCK_CodeSize ) return BaseT :: getIntrinsicInstrCost ( ICA , CostKind ) ; if ( ICA . isTypeBasedOnly ( ) ) return getTypeBasedIntrinsicInstrCost ( ICA , CostKind ) ; unsigned RetVF = ( RetTy -> isVectorTy ( ) ? cast < FixedVectorType > ( RetTy ) -> getNumElements ( ) : <NUM_LIT> ) ; const IntrinsicInst * I = ICA . getInst ( ) ; const SmallVectorImpl < const Value * > & Args = ICA . getArgs ( ) ; FastMathFlags FMF = ICA . getFlags ( ) ; InstructionCost ScalarizationCost = InstructionCost :: getInvalid ( ) ; if ( RetVF > <NUM_LIT> ) { ScalarizationCost = <NUM_LIT> ; if ( ! RetTy -> isVoidTy ( ) ) ScalarizationCost += getScalarizationOverhead ( cast < VectorType > ( RetTy ) , true , false ) ; ScalarizationCost += getOperandsScalarizationOverhead ( Args , ICA . getArgTypes ( ) ) ; } IntrinsicCostAttributes Attrs ( ICA . getID ( ) , RetTy , ICA . getArgTypes ( ) , FMF , I , ScalarizationCost ) ; return getIntrinsicInstrCost ( Attrs , CostKind ) ; } std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , RetTy ) ; unsigned NElts = LT . second . isVector ( ) ? LT . second . getVectorNumElements ( ) : <NUM_LIT> ; <STR_LIT> SLT = LT . second . getScalarType ( ) . SimpleTy ; if ( SLT == <STR_LIT> ) return LT . first * NElts * get64BitInstrCost ( CostKind ) ; if ( ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) || ( ST -> hasPackedFP32Ops ( ) && SLT == <STR_LIT> ) ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ;

unsigned InstRate = getQuarterRateInstrCost ( CostKind ) ;


<s> unsigned GCNTTIImpl :: getNumberOfRegisters ( unsigned RCID ) const { const <ISA_LIT>RegisterInfo * TRI = ST -> getRegisterInfo ( ) ; const TargetRegisterClass * RC = TRI -> getRegClass ( RCID ) ; unsigned NumVGPRs = ( TRI -> getRegSizeInBits ( * RC ) + <NUM_LIT> ) / <NUM_LIT> ; return getHardwareNumberOfRegisters ( false ) / NumVGPRs ;


<s> const TargetRegisterClass * RC = TRI -> getRegClass ( RCID ) ; unsigned NumVGPRs = ( TRI -> getRegSizeInBits ( * RC ) + <NUM_LIT> ) / <NUM_LIT> ;

return getHardwareNumberOfRegisters ( false ) / NumVGPRs ;


<s> case TTI :: SK_Broadcast : case TTI :: SK_Reverse : case TTI :: SK_PermuteSingleSrc : return <NUM_LIT> ; default : break ; } } } return BaseT :: getShuffleCost ( Kind , VT , Mask , Index , SubTp ) ;


<s> Kind = improveShuffleKindFromMask ( Kind , Mask ) ; if ( ST -> hasVOP3PInsts ( ) ) { if ( cast < FixedVectorType > ( VT ) -> getNumElements ( ) == <NUM_LIT> && DL . getTypeSizeInBits ( VT -> getElementType ( ) ) == <NUM_LIT> ) { switch ( Kind ) { case TTI :: SK_Broadcast : case TTI :: SK_Reverse : case TTI :: SK_PermuteSingleSrc : return <NUM_LIT> ; default : break ; } } }

return BaseT :: getShuffleCost ( Kind , VT , Mask , Index , SubTp ) ;


<s> const TargetMachine & TM = getTLI ( ) -> getTargetMachine ( ) ; const FeatureBitset & CallerBits = TM . getSubtargetImpl ( * Caller ) -> getFeatureBits ( ) ; const FeatureBitset & CalleeBits = TM . getSubtargetImpl ( * Callee ) -> getFeatureBits ( ) ; FeatureBitset RealCallerBits = CallerBits & ~ InlineFeatureIgnoreList ; FeatureBitset RealCalleeBits = CalleeBits & ~ InlineFeatureIgnoreList ; if ( ( RealCallerBits & RealCalleeBits ) != RealCalleeBits ) return false ; <STR_LIT> CallerMode ( * Caller ) ; <STR_LIT> CalleeMode ( * Callee ) ;

return CallerMode . isInlineCompatible ( CalleeMode ) ;


<s> int ISD = TLI -> InstructionOpcodeToISD ( Opcode ) ; unsigned NElts = LT . second . isVector ( ) ? LT . second . getVectorNumElements ( ) : <NUM_LIT> ; <STR_LIT> SLT = LT . second . getScalarType ( ) . SimpleTy ; switch ( ISD ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) return get64BitInstrCost ( ) * LT . first * NElts ; return getFullRateInstrCost ( ) * LT . first * NElts ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) { return <NUM_LIT> * getFullRateInstrCost ( ) * LT . first * NElts ; } return LT . first * NElts * getFullRateInstrCost ( ) ; case <STR_LIT> : { const int QuarterRateCost = getQuarterRateInstrCost ( ) ; if ( SLT == <STR_LIT> ) { const int FullRateCost = getFullRateInstrCost ( ) ; return ( <NUM_LIT> * QuarterRateCost + ( <NUM_LIT> * <NUM_LIT> ) * FullRateCost ) * LT . first * NElts ; } return QuarterRateCost * NElts * LT . first ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) return LT . first * NElts * get64BitInstrCost ( ) ; if ( SLT == <STR_LIT> || SLT == <STR_LIT> ) return LT . first * NElts * getFullRateInstrCost ( ) ;


<s> unsigned LocalGEPsSeen = <NUM_LIT> ; if ( llvm :: any_of ( L -> getSubLoops ( ) , [ BB ] ( const Loop * SubLoop ) { return SubLoop -> contains ( BB ) ; } ) ) continue ; for ( const Instruction & I : * BB ) { if ( const BranchInst * Br = dyn_cast < BranchInst > ( & I ) ) { if ( UP . Threshold < MaxBoost && Br -> isConditional ( ) ) { BasicBlock * Succ0 = Br -> getSuccessor ( <NUM_LIT> ) ; BasicBlock * Succ1 = Br -> getSuccessor ( <NUM_LIT> ) ; if ( ( L -> contains ( Succ0 ) && L -> isLoopExiting ( Succ0 ) ) || ( L -> contains ( Succ1 ) && L -> isLoopExiting ( Succ1 ) ) ) continue ; if ( dependsOnLocalPhi ( L , Br -> getCondition ( ) ) ) { UP . Threshold += UnrollThresholdIf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << UP . Threshold << <STR_LIT> << * L << <STR_LIT> << * Br << '\n' ) ; if ( UP . Threshold >= MaxBoost ) return ; } } continue ; } const GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( & I ) ; if ( ! GEP ) continue ; unsigned AS = GEP -> getAddressSpace ( ) ; unsigned Threshold = <NUM_LIT> ; if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) Threshold = ThresholdPrivate ; else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: REGION_ADDRESS ) Threshold = ThresholdLocal ; else continue ; if ( UP . Threshold >= Threshold ) continue ; if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { const Value * Ptr = GEP -> getPointerOperand ( ) ; const AllocaInst * Alloca = dyn_cast < AllocaInst > ( GetUnderlyingObject ( Ptr , DL ) ) ; if ( ! Alloca || ! Alloca -> isStaticAlloca ( ) ) continue ; Type * Ty = Alloca -> getAllocatedType ( ) ; unsigned AllocaSize = Ty -> isSized ( ) ? DL . getTypeAllocSize ( Ty ) : <NUM_LIT> ; if ( AllocaSize > MaxAlloca ) continue ; } else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: REGION_ADDRESS ) {


<s> } if ( const InsertElementInst * IE = dyn_cast < InsertElementInst > ( U ) ) { ConstantInt * CI = dyn_cast < ConstantInt > ( IE -> getOperand ( <NUM_LIT> ) ) ; unsigned Idx = - <NUM_LIT> ; if ( CI ) Idx = CI -> getZExtValue ( ) ; return getVectorInstrCost ( IE -> getOpcode ( ) , IE -> getType ( ) , Idx ) ; } if ( const IntrinsicInst * II = dyn_cast < IntrinsicInst > ( U ) ) { SmallVector < Value * , <NUM_LIT> > Args ( II -> arg_operands ( ) ) ; FastMathFlags FMF ; if ( auto * FPMO = dyn_cast < FPMathOperator > ( II ) ) FMF = FPMO -> getFastMathFlags ( ) ;

return getIntrinsicInstrCost ( II -> getIntrinsicID ( ) , II -> getType ( ) , Args , FMF ) ;


<s> EVT OrigTy = TLI -> getValueType ( DL , Ty ) ; if ( IsPairwise || ! ST -> hasVOP3PInsts ( ) || OrigTy . getScalarSizeInBits ( ) != <NUM_LIT> ) return BaseT :: getArithmeticReductionCost ( Opcode , Ty , IsPairwise ) ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Ty ) ;

return LT . first * getFullRateInstrCost ( ) ;


<s> EVT OrigTy = TLI -> getValueType ( DL , Ty ) ;

if ( IsPairwise || ! ST -> hasVOP3PInsts ( ) || OrigTy . getScalarSizeInBits ( ) != <NUM_LIT> ) return BaseT :: getArithmeticReductionCost ( Opcode , Ty , IsPairwise ) ;


<s> return CostKind == TTI :: TCK_CodeSize ? <NUM_LIT> : <NUM_LIT> * TargetTransformInfo :: TCC_Basic ;


<s> if ( ST -> getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS ) return <NUM_LIT> ; return <NUM_LIT> ; } if ( AddrSpace == AS . FLAT_ADDRESS ) return <NUM_LIT> ; if ( AddrSpace == AS . LOCAL_ADDRESS || AddrSpace == AS . REGION_ADDRESS ) return ST -> useDS128 ( ) ? <NUM_LIT> : <NUM_LIT> ; if ( AddrSpace == AS . PRIVATE_ADDRESS ) return <NUM_LIT> * ST -> getMaxPrivateElementSize ( ) ; if ( ST -> getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS && ( AddrSpace == AS . PARAM_D_ADDRESS || AddrSpace == AS . PARAM_I_ADDRESS || ( AddrSpace >= AS . CONSTANT_BUFFER_0 && AddrSpace <= AS . CONSTANT_BUFFER_15 ) ) ) return <NUM_LIT> ;


<s> if ( ST -> getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS && ( AddrSpace == AS . PARAM_D_ADDRESS || AddrSpace == AS . PARAM_I_ADDRESS || ( AddrSpace >= AS . CONSTANT_BUFFER_0 && AddrSpace <= AS . CONSTANT_BUFFER_15 ) ) ) return <NUM_LIT> ; llvm_unreachable ( <STR_LIT> ) ;


<s> if ( VecRegBitWidth > <NUM_LIT> && VecTy -> getScalarSizeInBits ( ) < <NUM_LIT> ) return <NUM_LIT> / LoadSize ;


<s> unsigned VecRegBitWidth = VF * LoadSize ; if ( VecRegBitWidth > <NUM_LIT> && VecTy -> getScalarSizeInBits ( ) < <NUM_LIT> ) return <NUM_LIT> / LoadSize ; return VF ;


<s> static inline int getQuarterRateInstrCost ( TTI :: TargetCostKind CostKind = TTI :: TCK_RecipThroughput ) {

return CostKind == TTI :: TCK_CodeSize ? <NUM_LIT> : <NUM_LIT> * TargetTransformInfo :: TCC_Basic ;


<s> unsigned <ISA_LIT>TTIImpl :: getStoreVectorFactor ( unsigned VF , unsigned StoreSize , unsigned ChainSizeInBytes , VectorType * VecTy ) const { unsigned VecRegBitWidth = VF * StoreSize ;

if ( VecRegBitWidth > <NUM_LIT> ) return <NUM_LIT> / StoreSize ;


<s> unsigned VecRegBitWidth = VF * StoreSize ; if ( VecRegBitWidth > <NUM_LIT> ) return <NUM_LIT> / StoreSize ; return VF ;


<s> unsigned OrderingVal = Ordering -> getZExtValue ( ) ; if ( OrderingVal > static_cast < unsigned > ( AtomicOrdering :: SequentiallyConsistent ) ) return false ; Info . PtrVal = Inst -> getArgOperand ( <NUM_LIT> ) ; Info . Ordering = static_cast < AtomicOrdering > ( OrderingVal ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { auto * Ordering = dyn_cast < ConstantInt > ( Inst -> getArgOperand ( <NUM_LIT> ) ) ; auto * Volatile = dyn_cast < ConstantInt > ( Inst -> getArgOperand ( <NUM_LIT> ) ) ; if ( ! Ordering || ! Volatile ) return false ;

unsigned OrderingVal = Ordering -> getZExtValue ( ) ;


<s> <ISA_LIT>AS ASST = ST -> get<ISA_LIT>AS ( ) ; for ( const BasicBlock * BB : L -> getBlocks ( ) ) { const DataLayout & DL = BB -> getModule ( ) -> getDataLayout ( ) ; unsigned LocalGEPsSeen = <NUM_LIT> ; if ( llvm :: any_of ( L -> getSubLoops ( ) , [ BB ] ( const Loop * SubLoop ) { return SubLoop -> contains ( BB ) ; } ) ) continue ; for ( const Instruction & I : * BB ) { if ( const BranchInst * Br = dyn_cast < BranchInst > ( & I ) ) { if ( UP . Threshold < MaxBoost && Br -> isConditional ( ) ) { if ( L -> isLoopExiting ( Br -> getSuccessor ( <NUM_LIT> ) ) || L -> isLoopExiting ( Br -> getSuccessor ( <NUM_LIT> ) ) ) continue ; if ( dependsOnLocalPhi ( L , Br -> getCondition ( ) ) ) { UP . Threshold += UnrollThresholdIf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << UP . Threshold << <STR_LIT> << * L << <STR_LIT> << * Br << '\n' ) ; if ( UP . Threshold >= MaxBoost ) return ; } } continue ; } const GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( & I ) ; if ( ! GEP ) continue ; unsigned AS = GEP -> getAddressSpace ( ) ; unsigned Threshold = <NUM_LIT> ; if ( AS == ASST . PRIVATE_ADDRESS ) Threshold = ThresholdPrivate ; else if ( AS == ASST . LOCAL_ADDRESS ) Threshold = ThresholdLocal ; else continue ; if ( UP . Threshold >= Threshold ) continue ; if ( AS == ASST . PRIVATE_ADDRESS ) { const Value * Ptr = GEP -> getPointerOperand ( ) ; const AllocaInst * Alloca = dyn_cast < AllocaInst > ( GetUnderlyingObject ( Ptr , DL ) ) ;

if ( ! Alloca || ! Alloca -> isStaticAlloca ( ) ) continue ;


<s> if ( const LoadInst * Load = dyn_cast < LoadInst > ( V ) ) return Load -> getPointerAddressSpace ( ) == ST -> get<ISA_LIT>AS ( ) . PRIVATE_ADDRESS ; if ( isa < AtomicRMWInst > ( V ) || isa < AtomicCmpXchgInst > ( V ) ) return true ; if ( const IntrinsicInst * Intrinsic = dyn_cast < IntrinsicInst > ( V ) ) return <STR_LIT> ( Intrinsic -> getIntrinsicID ( ) ) ;

if ( isa < CallInst > ( V ) || isa < InvokeInst > ( V ) ) return true ;


<s> if ( const IntrinsicInst * Intrinsic = dyn_cast < IntrinsicInst > ( V ) ) return <STR_LIT> ( Intrinsic -> getIntrinsicID ( ) ) ;


<s> bool GCNTTIImpl :: areInlineCompatible ( const Function * Caller , const Function * Callee ) const { const TargetMachine & TM = getTLI ( ) -> getTargetMachine ( ) ; const FeatureBitset & CallerBits = TM . getSubtargetImpl ( * Caller ) -> getFeatureBits ( ) ; const FeatureBitset & CalleeBits = TM . getSubtargetImpl ( * Callee ) -> getFeatureBits ( ) ;

FeatureBitset RealCallerBits = CallerBits & ~ InlineFeatureIgnoreList ;


<s> const FeatureBitset & CallerBits = TM . getSubtargetImpl ( * Caller ) -> getFeatureBits ( ) ; const FeatureBitset & CalleeBits = TM . getSubtargetImpl ( * Callee ) -> getFeatureBits ( ) ; FeatureBitset RealCallerBits = CallerBits & ~ InlineFeatureIgnoreList ;

FeatureBitset RealCalleeBits = CalleeBits & ~ InlineFeatureIgnoreList ;


<s> if ( AddrSpace == <ISA_LIT>AS :: FLAT_ADDRESS || AddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || AddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) return <NUM_LIT> ; if ( AddrSpace == <ISA_LIT>AS :: PRIVATE_ADDRESS ) return <NUM_LIT> * ST -> getMaxPrivateElementSize ( ) ;


<s> unsigned GCNTTIImpl :: getLoadStoreVecRegBitWidth ( unsigned AddrSpace ) const { if ( AddrSpace == <ISA_LIT>AS :: GLOBAL_ADDRESS || AddrSpace == <ISA_LIT>AS :: CONSTANT_ADDRESS || AddrSpace == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) { return <NUM_LIT> ; } if ( AddrSpace == <ISA_LIT>AS :: FLAT_ADDRESS || AddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || AddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) return <NUM_LIT> ; if ( AddrSpace == <ISA_LIT>AS :: PRIVATE_ADDRESS ) return <NUM_LIT> * ST -> getMaxPrivateElementSize ( ) ;


<s> if ( llvm :: any_of ( L -> getSubLoops ( ) , [ BB ] ( const Loop * SubLoop ) { return SubLoop -> contains ( BB ) ; } ) ) continue ; for ( const Instruction & I : * BB ) { if ( const BranchInst * Br = dyn_cast < BranchInst > ( & I ) ) { if ( UP . Threshold < MaxBoost && Br -> isConditional ( ) ) { if ( L -> isLoopExiting ( Br -> getSuccessor ( <NUM_LIT> ) ) || L -> isLoopExiting ( Br -> getSuccessor ( <NUM_LIT> ) ) ) continue ; if ( dependsOnLocalPhi ( L , Br -> getCondition ( ) ) ) { UP . Threshold += UnrollThresholdIf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << UP . Threshold << <STR_LIT> << * L << <STR_LIT> << * Br << '\n' ) ; if ( UP . Threshold >= MaxBoost ) return ; } } continue ; } const GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( & I ) ; if ( ! GEP ) continue ; unsigned AS = GEP -> getAddressSpace ( ) ; unsigned Threshold = <NUM_LIT> ; if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) Threshold = ThresholdPrivate ; else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) Threshold = ThresholdLocal ; else continue ; if ( UP . Threshold >= Threshold ) continue ; if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { const Value * Ptr = GEP -> getPointerOperand ( ) ; const AllocaInst * Alloca = dyn_cast < AllocaInst > ( GetUnderlyingObject ( Ptr , DL ) ) ; if ( ! Alloca || ! Alloca -> isStaticAlloca ( ) ) continue ; Type * Ty = Alloca -> getAllocatedType ( ) ; unsigned AllocaSize = Ty -> isSized ( ) ? DL . getTypeAllocSize ( Ty ) : <NUM_LIT> ; if ( AllocaSize > MaxAlloca ) continue ; } else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) { LocalGEPsSeen ++ ; if ( LocalGEPsSeen > <NUM_LIT> || L -> getLoopDepth ( ) > <NUM_LIT> || ( ! isa < GlobalVariable > ( GEP -> getPointerOperand ( ) ) && ! isa < Argument > ( GEP -> getPointerOperand ( ) ) ) ) continue ;


<s> unsigned <ISA_LIT>TTIImpl :: getMaxInterleaveFactor ( unsigned VF ) { return <NUM_LIT> ;


<s> return Vector ? <NUM_LIT> : <NUM_LIT> ;


<s> unsigned <ISA_LIT>TTIImpl :: getRegisterBitWidth ( bool Vector ) {


<s> UP . Partial = true ; for ( const BasicBlock * BB : L -> getBlocks ( ) ) { const DataLayout & DL = BB -> getModule ( ) -> getDataLayout ( ) ; for ( const Instruction & I : * BB ) { const GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( & I ) ;

if ( ! GEP || GEP -> getAddressSpace ( ) != <ISA_LIT>AS :: PRIVATE_ADDRESS ) continue ;


<s> int <ISA_LIT>TTIImpl :: getVectorInstrCost ( unsigned Opcode , Type * ValTy , unsigned Index ) { switch ( Opcode ) { case Instruction :: ExtractElement : return Index == ~ <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ;


<s> if ( ShaderType == ShaderType :: COMPUTE ) return true ; if ( F -> getAttributes ( ) . hasAttribute ( A -> getArgNo ( ) + <NUM_LIT> , Attribute :: InReg ) || F -> getAttributes ( ) . hasAttribute ( A -> getArgNo ( ) + <NUM_LIT> , Attribute :: ByVal ) ) return true ; return false ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; } StringRef Name = I -> getCalledFunction ( ) -> getName ( ) ; switch ( TII -> lookupName ( ( const char * ) Name . bytes_begin ( ) , Name . size ( ) ) ) { default : return false ; case <STR_LIT> : case <STR_LIT> : return true ;


<s> if ( const LoadInst * Load = dyn_cast < LoadInst > ( V ) ) return Load -> getPointerAddressSpace ( ) == <ISA_LIT>AS :: PRIVATE_ADDRESS ; if ( const IntrinsicInst * Intrinsic = dyn_cast < IntrinsicInst > ( V ) ) { const TargetMachine & TM = getTLI ( ) -> getTargetMachine ( ) ; return isIntrinsicSourceOfDivergence ( TM . getIntrinsicInfo ( ) , Intrinsic ) ; }


<s> <ISA_LIT>AS AS = ST -> get<ISA_LIT>AS ( ) ; if ( AddrSpace == AS . GLOBAL_ADDRESS || AddrSpace == AS . CONSTANT_ADDRESS || AddrSpace == AS . CONSTANT_ADDRESS_32BIT ) { if ( ST -> getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS ) return <NUM_LIT> ; return <NUM_LIT> ; } if ( AddrSpace == AS . FLAT_ADDRESS ) return <NUM_LIT> ; if ( AddrSpace == AS . LOCAL_ADDRESS || AddrSpace == AS . REGION_ADDRESS ) return <NUM_LIT> ;

if ( AddrSpace == AS . PRIVATE_ADDRESS ) return <NUM_LIT> * ST -> getMaxPrivateElementSize ( ) ;


<s> explicit <ISA_LIT>TTIImpl ( const <ISA_LIT>TargetMachine * TM , const Function & F ) : BaseT ( TM , F . getParent ( ) -> getDataLayout ( ) ) , TargetTriple ( TM -> getTargetTriple ( ) ) , ST ( static_cast < const GCNSubtarget * > ( TM -> getSubtargetImpl ( F ) ) ) , TLI ( ST -> getTargetLowering ( ) ) {


<s> const GCNSubtarget * CallerST = static_cast < const GCNSubtarget * > ( TM . getSubtargetImpl ( * Caller ) ) ; const GCNSubtarget * CalleeST = static_cast < const GCNSubtarget * > ( TM . getSubtargetImpl ( * Callee ) ) ;

const FeatureBitset & CallerBits = CallerST -> getFeatureBits ( ) ;


<s> explicit GCNTTIImpl ( const <ISA_LIT>TargetMachine * TM , const Function & F ) : BaseT ( TM , F . getParent ( ) -> getDataLayout ( ) ) , ST ( static_cast < const GCNSubtarget * > ( TM -> getSubtargetImpl ( F ) ) ) , TLI ( ST -> getTargetLowering ( ) ) , CommonTTI ( TM , F ) , IsGraphicsShader ( <STR_LIT> ( F . getCallingConv ( ) ) ) , HasFP32Denormals ( <STR_LIT> ( F ) . allFP32Denormals ( ) ) {


<s> case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) return get64BitInstrCost ( ) * LT . first * NElts ; if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; return getFullRateInstrCost ( ) * LT . first * NElts ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) { return <NUM_LIT> * getFullRateInstrCost ( ) * LT . first * NElts ; } if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; return LT . first * NElts * getFullRateInstrCost ( ) ; case <STR_LIT> : { const int QuarterRateCost = getQuarterRateInstrCost ( ) ; if ( SLT == <STR_LIT> ) { const int FullRateCost = getFullRateInstrCost ( ) ; return ( <NUM_LIT> * QuarterRateCost + ( <NUM_LIT> * <NUM_LIT> ) * FullRateCost ) * LT . first * NElts ; } if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; return QuarterRateCost * NElts * LT . first ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) return LT . first * NElts * get64BitInstrCost ( ) ; if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ;

if ( SLT == <STR_LIT> || SLT == <STR_LIT> ) return LT . first * NElts * getFullRateInstrCost ( ) ;


<s> if ( IsPairwise || ! ST -> hasVOP3PInsts ( ) || OrigTy . getScalarSizeInBits ( ) != <NUM_LIT> ) return BaseT :: getArithmeticReductionCost ( Opcode , Ty , IsPairwise , CostKind ) ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Ty ) ; return LT . first * getFullRateInstrCost ( ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getCFInstrCost ( unsigned Opcode , TTI :: TargetCostKind CostKind ) { if ( CostKind == TTI :: TCK_CodeSize || CostKind == TTI :: TCK_SizeAndLatency ) return Opcode == Instruction :: PHI ? <NUM_LIT> : <NUM_LIT> ; switch ( Opcode ) {


<s> unsigned <ISA_LIT>TTIImpl :: getHardwareNumberOfRegisters ( bool Vec ) const {


<s> return BaseT :: getIntrinsicInstrCost ( ICA , CostKind ) ; } std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , RetTy ) ; unsigned NElts = LT . second . isVector ( ) ? LT . second . getVectorNumElements ( ) : <NUM_LIT> ; <STR_LIT> SLT = LT . second . getScalarType ( ) . SimpleTy ; if ( SLT == <STR_LIT> ) return LT . first * NElts * get64BitInstrCost ( ) ;

if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ;


<s> unsigned <ISA_LIT>TTIImpl :: getMaxInterleaveFactor ( unsigned VF ) { if ( VF == <NUM_LIT> ) return <NUM_LIT> ; return <NUM_LIT> ;


<s> Type * GCNTTIImpl :: getMemcpyLoopLoweringType ( LLVMContext & Context , Value * Length , unsigned SrcAddrSpace , unsigned DestAddrSpace , unsigned SrcAlign , unsigned DestAlign ) const { unsigned MinAlign = std :: min ( SrcAlign , DestAlign ) ; if ( MinAlign == <NUM_LIT> ) return Type :: getInt16Ty ( Context ) ; if ( SrcAddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || SrcAddrSpace == <ISA_LIT>AS :: REGION_ADDRESS || DestAddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || DestAddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) { return FixedVectorType :: get ( Type :: getInt32Ty ( Context ) , <NUM_LIT> ) ; } return FixedVectorType :: get ( Type :: getInt32Ty ( Context ) , <NUM_LIT> ) ;


<s> Type * GCNTTIImpl :: getMemcpyLoopLoweringType ( LLVMContext & Context , Value * Length , unsigned SrcAddrSpace , unsigned DestAddrSpace , unsigned SrcAlign , unsigned DestAlign ) const { unsigned MinAlign = std :: min ( SrcAlign , DestAlign ) ; if ( MinAlign == <NUM_LIT> ) return Type :: getInt16Ty ( Context ) ; if ( SrcAddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || SrcAddrSpace == <ISA_LIT>AS :: REGION_ADDRESS || DestAddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || DestAddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) { return FixedVectorType :: get ( Type :: getInt32Ty ( Context ) , <NUM_LIT> ) ; }


<s> if ( IsPairwise || ! ST -> hasVOP3PInsts ( ) || OrigTy . getScalarSizeInBits ( ) != <NUM_LIT> ) return BaseT :: getMinMaxReductionCost ( Ty , CondTy , IsPairwise , IsUnsigned , CostKind ) ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Ty ) ; return LT . first * getHalfRateInstrCost ( ) ;


<s> EVT OrigTy = TLI -> getValueType ( DL , Ty ) ; if ( IsPairwise || ! ST -> hasVOP3PInsts ( ) || OrigTy . getScalarSizeInBits ( ) != <NUM_LIT> ) return BaseT :: getMinMaxReductionCost ( Ty , CondTy , IsPairwise , IsUnsigned , CostKind ) ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Ty ) ;

return LT . first * getHalfRateInstrCost ( ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getMinVectorRegisterBitWidth ( ) const {


<s> unsigned <ISA_LIT>TTIImpl :: getNumberOfRegisters ( bool Vec ) const { return getHardwareNumberOfRegisters ( Vec ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getRegisterBitWidth ( bool Vector ) const {


<s> if ( cast < FixedVectorType > ( VT ) -> getNumElements ( ) == <NUM_LIT> && DL . getTypeSizeInBits ( VT -> getElementType ( ) ) == <NUM_LIT> ) { switch ( Kind ) {


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { auto * Ordering = dyn_cast < ConstantInt > ( Inst -> getArgOperand ( <NUM_LIT> ) ) ; auto * Volatile = dyn_cast < ConstantInt > ( Inst -> getArgOperand ( <NUM_LIT> ) ) ; if ( ! Ordering || ! Volatile ) return false ; unsigned OrderingVal = Ordering -> getZExtValue ( ) ; if ( OrderingVal > static_cast < unsigned > ( AtomicOrdering :: SequentiallyConsistent ) ) return false ;

Info . PtrVal = Inst -> getArgOperand ( <NUM_LIT> ) ;


<s> case <STR_LIT> : case <STR_LIT> : { auto * Ordering = dyn_cast < ConstantInt > ( Inst -> getArgOperand ( <NUM_LIT> ) ) ; auto * Volatile = dyn_cast < ConstantInt > ( Inst -> getArgOperand ( <NUM_LIT> ) ) ; if ( ! Ordering || ! Volatile ) return false ; unsigned OrderingVal = Ordering -> getZExtValue ( ) ; if ( OrderingVal > static_cast < unsigned > ( AtomicOrdering :: SequentiallyConsistent ) ) return false ; Info . PtrVal = Inst -> getArgOperand ( <NUM_LIT> ) ; Info . Ordering = static_cast < AtomicOrdering > ( OrderingVal ) ; Info . ReadMem = true ; Info . WriteMem = true ; Info . IsVolatile = ! Volatile -> isNullValue ( ) ; return true ;


<s> CommonTTI . getUnrollingPreferences ( L , SE , UP ) ;


<s> case Instruction :: ExtractElement : case Instruction :: InsertElement : { unsigned EltSize = DL . getTypeSizeInBits ( cast < VectorType > ( ValTy ) -> getElementType ( ) ) ;


<s> int <ISA_LIT>TTIImpl :: getVectorInstrCost ( unsigned Opcode , Type * ValTy , unsigned Index ) { switch ( Opcode ) { case Instruction :: ExtractElement : case Instruction :: InsertElement : { unsigned EltSize = DL . getTypeSizeInBits ( cast < VectorType > ( ValTy ) -> getElementType ( ) ) ; if ( EltSize < <NUM_LIT> ) { return BaseT :: getVectorInstrCost ( Opcode , ValTy , Index ) ;


<s> case <STR_LIT> : return true ; default : return false ; }


<s> case CallingConv :: <ISA_LIT>_HS : case CallingConv :: <ISA_LIT>_ES : case CallingConv :: <ISA_LIT>_GS : case CallingConv :: <ISA_LIT>_PS : case CallingConv :: <ISA_LIT>_CS : return F -> getAttributes ( ) . hasParamAttribute ( A -> getArgNo ( ) , Attribute :: InReg ) || F -> getAttributes ( ) . hasParamAttribute ( A -> getArgNo ( ) , Attribute :: ByVal ) ; default : return false ; }


<s> bool GCNTTIImpl :: isInlineAsmSourceOfDivergence ( const CallInst * CI , ArrayRef < unsigned > Indices ) const { if ( Indices . size ( ) > <NUM_LIT> ) return true ; const DataLayout & DL = CI -> getModule ( ) -> getDataLayout ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST -> getRegisterInfo ( ) ; TargetLowering :: AsmOperandInfoVector TargetConstraints = TLI -> ParseConstraints ( DL , ST -> getRegisterInfo ( ) , * CI ) ; const int TargetOutputIdx = Indices . empty ( ) ? - <NUM_LIT> : Indices [ <NUM_LIT> ] ; int OutputIdx = <NUM_LIT> ; for ( auto & TC : TargetConstraints ) { if ( TC . Type != InlineAsm :: isOutput ) continue ; if ( TargetOutputIdx != - <NUM_LIT> && TargetOutputIdx != OutputIdx ++ ) continue ; TLI -> ComputeConstraintToUse ( TC , SDValue ( ) ) ; Register AssignedReg ; const TargetRegisterClass * RC ;

std :: tie ( AssignedReg , RC ) = TLI -> getRegForInlineAsmConstraint ( TRI , TC . ConstraintCode , TC . ConstraintVT ) ;


<s> return isLegalToVectorizeMemChain ( ChainSizeInBytes , Alignment , AddrSpace ) ;


<s> bool <ISA_LIT>TTIImpl :: isLegalToVectorizeMemChain ( unsigned ChainSizeInBytes , unsigned Alignment , unsigned AddrSpace ) const { return ( AddrSpace != <ISA_LIT>AS :: PRIVATE_ADDRESS ) ;


<s> bool <ISA_LIT>TTIImpl :: isLegalToVectorizeStoreChain ( unsigned ChainSizeInBytes , unsigned Alignment , unsigned AddrSpace ) const {

return isLegalToVectorizeMemChain ( ChainSizeInBytes , Alignment , AddrSpace ) ;


<s> explicit <ISA_LIT>TTIImpl ( const <ISA_LIT>TargetMachine * TM , const Function & F ) : BaseT ( TM , F . getParent ( ) -> getDataLayout ( ) ) , ST ( static_cast < const <ISA_LIT>Subtarget * > ( TM -> getSubtargetImpl ( F ) ) ) , TLI ( ST -> getTargetLowering ( ) ) , CommonTTI ( TM , F ) {


<s> LLVMContext & Ctx = NewV -> getType ( ) -> getContext ( ) ; ConstantInt * NewVal = ( TrueAS == NewAS ) ? ConstantInt :: getTrue ( Ctx ) : ConstantInt :: getFalse ( Ctx ) ; return NewVal ; } case <STR_LIT> : { unsigned OldAS = OldV -> getType ( ) -> getPointerAddressSpace ( ) ; unsigned NewAS = NewV -> getType ( ) -> getPointerAddressSpace ( ) ; Value * MaskOp = II -> getArgOperand ( <NUM_LIT> ) ; Type * MaskTy = MaskOp -> getType ( ) ; bool DoTruncate = false ; if ( ! getTLI ( ) -> isNoopAddrSpaceCast ( OldAS , NewAS ) ) { if ( DL . getPointerSizeInBits ( OldAS ) != <NUM_LIT> || DL . getPointerSizeInBits ( NewAS ) != <NUM_LIT> ) return nullptr ; KnownBits Known = computeKnownBits ( MaskOp , DL , <NUM_LIT> , nullptr , II ) ; if ( Known . countMinLeadingOnes ( ) < <NUM_LIT> ) return nullptr ; DoTruncate = true ; }


<s> Module * M = II -> getParent ( ) -> getParent ( ) -> getParent ( ) ; Type * DestTy = II -> getType ( ) ; Type * SrcTy = NewV -> getType ( ) ; Function * NewDecl = <STR_LIT> ( M , II -> getIntrinsicID ( ) , { DestTy , SrcTy } ) ; II -> setArgOperand ( <NUM_LIT> , NewV ) ; II -> setCalledFunction ( NewDecl ) ; return II ; } case <STR_LIT> : case <STR_LIT> : { unsigned TrueAS = IntrID == <STR_LIT> ? <ISA_LIT>AS :: LOCAL_ADDRESS : <ISA_LIT>AS :: PRIVATE_ADDRESS ; unsigned NewAS = NewV -> getType ( ) -> getPointerAddressSpace ( ) ; LLVMContext & Ctx = NewV -> getType ( ) -> getContext ( ) ; ConstantInt * NewVal = ( TrueAS == NewAS ) ? ConstantInt :: getTrue ( Ctx ) : ConstantInt :: getFalse ( Ctx ) ; return NewVal ; } case <STR_LIT> : { unsigned OldAS = OldV -> getType ( ) -> getPointerAddressSpace ( ) ; unsigned NewAS = NewV -> getType ( ) -> getPointerAddressSpace ( ) ; Value * MaskOp = II -> getArgOperand ( <NUM_LIT> ) ; Type * MaskTy = MaskOp -> getType ( ) ; bool DoTruncate = false ; if ( ! getTLI ( ) -> isNoopAddrSpaceCast ( OldAS , NewAS ) ) { if ( DL . getPointerSizeInBits ( OldAS ) != <NUM_LIT> || DL . getPointerSizeInBits ( NewAS ) != <NUM_LIT> ) return nullptr ; KnownBits Known = computeKnownBits ( MaskOp , DL , <NUM_LIT> , nullptr , II ) ; if ( Known . countMinLeadingOnes ( ) < <NUM_LIT> ) return nullptr ; DoTruncate = true ; } IRBuilder < > B ( II ) ; if ( DoTruncate ) { MaskTy = B . getInt32Ty ( ) ; MaskOp = B . CreateTrunc ( MaskOp , MaskTy ) ; } return B . CreateIntrinsic ( <STR_LIT> , { NewV -> getType ( ) , MaskTy } , { NewV , MaskOp } ) ; } default : return nullptr ;


<s> explicit GCNTTIImpl ( const <ISA_LIT>TargetMachine * TM , const Function & F ) : BaseT ( TM , F . getParent ( ) -> getDataLayout ( ) ) , ST ( static_cast < const GCNSubtarget * > ( TM -> getSubtargetImpl ( F ) ) ) , TLI ( ST -> getTargetLowering ( ) ) , CommonTTI ( TM , F ) , IsGraphicsShader ( <STR_LIT> ( F . getCallingConv ( ) ) ) , HasFP32Denormals ( ST -> hasFP32Denormals ( F ) ) {


<s> <ISA_LIT>AS AS = ST -> get<ISA_LIT>AS ( ) ; if ( AddrSpace == AS . GLOBAL_ADDRESS || AddrSpace == AS . CONSTANT_ADDRESS ) return <NUM_LIT> ; if ( AddrSpace == AS . LOCAL_ADDRESS || AddrSpace == AS . REGION_ADDRESS ) return <NUM_LIT> ; if ( AddrSpace == AS . PRIVATE_ADDRESS ) return <NUM_LIT> ; if ( ( AddrSpace == AS . PARAM_D_ADDRESS || AddrSpace == AS . PARAM_I_ADDRESS || ( AddrSpace >= AS . CONSTANT_BUFFER_0 && AddrSpace <= AS . CONSTANT_BUFFER_15 ) ) ) return <NUM_LIT> ; llvm_unreachable ( <STR_LIT> ) ;


<s> case <STR_LIT> : { auto * Ordering = dyn_cast < ConstantInt > ( Inst -> getArgOperand ( <NUM_LIT> ) ) ; auto * Volatile = dyn_cast < ConstantInt > ( Inst -> getArgOperand ( <NUM_LIT> ) ) ; if ( ! Ordering || ! Volatile ) return false ; unsigned OrderingVal = Ordering -> getZExtValue ( ) ; if ( OrderingVal > static_cast < unsigned > ( AtomicOrdering :: SequentiallyConsistent ) ) return false ; Info . PtrVal = Inst -> getArgOperand ( <NUM_LIT> ) ; Info . Ordering = static_cast < AtomicOrdering > ( OrderingVal ) ;


<s> bool GCNTTIImpl :: isAlwaysUniform ( const Value * V ) const { if ( const IntrinsicInst * Intrinsic = dyn_cast < IntrinsicInst > ( V ) ) { switch ( Intrinsic -> getIntrinsicID ( ) ) { default : return false ; case <STR_LIT> : case <STR_LIT> : return true ; }


<s> if ( AddrSpace == ST -> get<ISA_LIT>AS ( ) . PRIVATE_ADDRESS ) return false ;


<s> if ( const Argument * A = dyn_cast < Argument > ( V ) ) return ! isArgPassedInSGPR ( A ) ; if ( const LoadInst * Load = dyn_cast < LoadInst > ( V ) ) return Load -> getPointerAddressSpace ( ) == ST -> get<ISA_LIT>AS ( ) . PRIVATE_ADDRESS || Load -> getPointerAddressSpace ( ) == ST -> get<ISA_LIT>AS ( ) . FLAT_ADDRESS ; if ( isa < AtomicRMWInst > ( V ) || isa < AtomicCmpXchgInst > ( V ) ) return true ;

if ( const IntrinsicInst * Intrinsic = dyn_cast < IntrinsicInst > ( V ) ) return <STR_LIT> ( Intrinsic -> getIntrinsicID ( ) ) ;


<s> if ( const Argument * A = dyn_cast < Argument > ( V ) ) return ! isArgPassedInSGPR ( A ) ; if ( const LoadInst * Load = dyn_cast < LoadInst > ( V ) ) return Load -> getPointerAddressSpace ( ) == ST -> get<ISA_LIT>AS ( ) . PRIVATE_ADDRESS || Load -> getPointerAddressSpace ( ) == ST -> get<ISA_LIT>AS ( ) . FLAT_ADDRESS ; if ( isa < AtomicRMWInst > ( V ) || isa < AtomicCmpXchgInst > ( V ) ) return true ; if ( const IntrinsicInst * Intrinsic = dyn_cast < IntrinsicInst > ( V ) ) return <STR_LIT> ( Intrinsic -> getIntrinsicID ( ) ) ;


<s> if ( ST -> getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS && ( AddrSpace == <ISA_LIT>AS :: PARAM_D_ADDRESS || AddrSpace == <ISA_LIT>AS :: PARAM_I_ADDRESS || ( AddrSpace >= <ISA_LIT>AS :: CONSTANT_BUFFER_0 && AddrSpace <= <ISA_LIT>AS :: CONSTANT_BUFFER_15 ) ) ) return <NUM_LIT> ; llvm_unreachable ( <STR_LIT> ) ;


<s> switch ( Opcode ) { case Instruction :: ExtractElement : case Instruction :: InsertElement :

return Index == ~ <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ;


<s> case Instruction :: InsertElement : return Index == ~ <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ; default : return BaseT :: getVectorInstrCost ( Opcode , ValTy , Index ) ; }


<s> if ( F -> getAttributes ( ) . hasAttribute ( A -> getArgNo ( ) + <NUM_LIT> , Attribute :: InReg ) || F -> getAttributes ( ) . hasAttribute ( A -> getArgNo ( ) + <NUM_LIT> , Attribute :: ByVal ) ) return true ;


<s> static bool isArgPassedInSGPR ( const Argument * A ) { const Function * F = A -> getParent ( ) ; if ( ! <STR_LIT> ( F -> getCallingConv ( ) ) ) return true ;

if ( F -> getAttributes ( ) . hasAttribute ( A -> getArgNo ( ) + <NUM_LIT> , Attribute :: InReg ) || F -> getAttributes ( ) . hasAttribute ( A -> getArgNo ( ) + <NUM_LIT> , Attribute :: ByVal ) ) return true ;


<s> if ( isa < AtomicRMWInst > ( V ) || isa < AtomicCmpXchgInst > ( V ) ) return true ; if ( const IntrinsicInst * Intrinsic = dyn_cast < IntrinsicInst > ( V ) ) { const TargetMachine & TM = getTLI ( ) -> getTargetMachine ( ) ; return isIntrinsicSourceOfDivergence ( TM . getIntrinsicInfo ( ) , Intrinsic ) ; } if ( isa < CallInst > ( V ) || isa < InvokeInst > ( V ) ) return true ; return false ;


<s> if ( isa < AtomicRMWInst > ( V ) || isa < AtomicCmpXchgInst > ( V ) ) return true ; if ( const IntrinsicInst * Intrinsic = dyn_cast < IntrinsicInst > ( V ) ) { const TargetMachine & TM = getTLI ( ) -> getTargetMachine ( ) ; return isIntrinsicSourceOfDivergence ( TM . getIntrinsicInfo ( ) , Intrinsic ) ;


<s> explicit GCNTTIImpl ( const <ISA_LIT>TargetMachine * TM , const Function & F ) : BaseT ( TM , F . getParent ( ) -> getDataLayout ( ) ) , ST ( static_cast < const GCNSubtarget * > ( TM -> getSubtargetImpl ( F ) ) ) , TLI ( ST -> getTargetLowering ( ) ) , CommonTTI ( TM , F ) , IsGraphics ( <STR_LIT> ( F . getCallingConv ( ) ) ) , MaxVGPRs ( ST -> getMaxNumVGPRs ( std :: max ( ST -> getWavesPerEU ( F ) . first , ST -> getWavesPerEUForWorkGroup ( ST -> getFlatWorkGroupSizes ( F ) . second ) ) ) ) { <STR_LIT> Mode ( F ) ;

HasFP32Denormals = Mode . allFP32Denormals ( ) ;


<s> return ST -> hasHalfRate64Ops ( ) ? getHalfRateInstrCost ( CostKind ) : getQuarterRateInstrCost ( CostKind ) ;


<s> inline int get64BitInstrCost ( TTI :: TargetCostKind CostKind = TTI :: TCK_RecipThroughput ) const {

return ST -> hasHalfRate64Ops ( ) ? getHalfRateInstrCost ( CostKind ) : getQuarterRateInstrCost ( CostKind ) ;


<s> return <NUM_LIT> ; default : return BaseT :: getCFInstrCost ( Opcode , CostKind ) ; }


<s> case Instruction :: FPToUI : case Instruction :: FPTo<ISA_LIT> : case Instruction :: FPExt : case Instruction :: PtrToInt : case Instruction :: IntToPtr : case Instruction :: <ISA_LIT>ToFP : case Instruction :: UIToFP : case Instruction :: Trunc : case Instruction :: FPTrunc : case Instruction :: BitCast : case Instruction :: AddrSpaceCast : { return getCastInstrCost ( I -> getOpcode ( ) , I -> getType ( ) , I -> getOperand ( <NUM_LIT> ) -> getType ( ) , CostKind , I ) ; } case Instruction :: Add : case Instruction :: FAdd : case Instruction :: Sub : case Instruction :: FSub : case Instruction :: Mul : case Instruction :: FMul : case Instruction :: UDiv : case Instruction :: SDiv : case Instruction :: FDiv : case Instruction :: URem : case Instruction :: SRem : case Instruction :: FRem : case Instruction :: Shl : case Instruction :: LShr : case Instruction :: AShr : case Instruction :: And : case Instruction :: Or : case Instruction :: Xor : case Instruction :: FNeg : { return getArithmeticInstrCost ( I -> getOpcode ( ) , I -> getType ( ) , CostKind , TTI :: OK_AnyValue , TTI :: OK_AnyValue , TTI :: OP_None , TTI :: OP_None , Operands , I ) ;


<s> case Instruction :: Call : { if ( const IntrinsicInst * II = dyn_cast < IntrinsicInst > ( U ) ) { IntrinsicCostAttributes CostAttrs ( * II ) ; return getIntrinsicInstrCost ( CostAttrs , CostKind ) ; } else { return BaseT :: getUserCost ( U , Operands , CostKind ) ; } } case Instruction :: ShuffleVector : { const ShuffleVectorInst * Shuffle = cast < ShuffleVectorInst > ( I ) ; auto * Ty = cast < VectorType > ( Shuffle -> getType ( ) ) ; auto * SrcTy = cast < VectorType > ( Shuffle -> getOperand ( <NUM_LIT> ) -> getType ( ) ) ; int SubIndex ; if ( Shuffle -> isExtractSubvectorMask ( SubIndex ) ) return getShuffleCost ( TTI :: SK_ExtractSubvector , SrcTy , SubIndex , Ty ) ; if ( Shuffle -> changesLength ( ) ) return BaseT :: getUserCost ( U , Operands , CostKind ) ; if ( Shuffle -> isIdentity ( ) ) return <NUM_LIT> ; if ( Shuffle -> isReverse ( ) ) return getShuffleCost ( TTI :: SK_Reverse , Ty , <NUM_LIT> , nullptr ) ; if ( Shuffle -> isSelect ( ) ) return getShuffleCost ( TTI :: SK_Select , Ty , <NUM_LIT> , nullptr ) ; if ( Shuffle -> isTranspose ( ) ) return getShuffleCost ( TTI :: SK_Transpose , Ty , <NUM_LIT> , nullptr ) ; if ( Shuffle -> isZeroEltSplat ( ) ) return getShuffleCost ( TTI :: SK_Broadcast , Ty , <NUM_LIT> , nullptr ) ; if ( Shuffle -> isSingleSource ( ) ) return getShuffleCost ( TTI :: SK_PermuteSingleSrc , Ty , <NUM_LIT> , nullptr ) ; return getShuffleCost ( TTI :: SK_PermuteTwoSrc , Ty , <NUM_LIT> , nullptr ) ; } case Instruction :: ZExt : case Instruction :: SExt : case Instruction :: FPToUI : case Instruction :: FPTo<ISA_LIT> : case Instruction :: FPExt : case Instruction :: PtrToInt : case Instruction :: IntToPtr : case Instruction :: <ISA_LIT>ToFP : case Instruction :: UIToFP : case Instruction :: Trunc : case Instruction :: FPTrunc : case Instruction :: BitCast :


<s> explicit GCNTTIImpl ( const <ISA_LIT>TargetMachine * TM , const Function & F ) : BaseT ( TM , F . getParent ( ) -> getDataLayout ( ) ) , ST ( static_cast < const GCNSubtarget * > ( TM -> getSubtargetImpl ( F ) ) ) , TLI ( ST -> getTargetLowering ( ) ) , CommonTTI ( TM , F ) , IsGraphicsShader ( <STR_LIT> ( F . getCallingConv ( ) ) ) , HasFP32Denormals ( <STR_LIT> ( F ) . allFP32Denormals ( ) ) , MaxVGPRs ( ST -> getMaxNumVGPRs ( std :: max ( ST -> getWavesPerEU ( F ) . first , ST -> getWavesPerEUForWorkGroup ( ST -> getFlatWorkGroupSizes ( F ) . second ) ) ) ) {


<s> StringRef Name = I -> getCalledFunction ( ) -> getName ( ) ; switch ( TII -> lookupName ( ( const char * ) Name . bytes_begin ( ) , Name . size ( ) ) ) { default : return false ; case <STR_LIT> : case <STR_LIT> :


<s> case <STR_LIT> : return true ; } StringRef Name = I -> getCalledFunction ( ) -> getName ( ) ; switch ( TII -> lookupName ( ( const char * ) Name . bytes_begin ( ) , Name . size ( ) ) ) { default :


<s> if ( ST -> hasVOP3PInsts ( ) ) { if ( cast < FixedVectorType > ( VT ) -> getNumElements ( ) == <NUM_LIT> && DL . getTypeSizeInBits ( VT -> getElementType ( ) ) == <NUM_LIT> ) { switch ( Kind ) { case TTI :: SK_Broadcast : case TTI :: SK_Reverse : case TTI :: SK_PermuteSingleSrc : return <NUM_LIT> ; default :


<s> if ( ST -> getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS ) return <NUM_LIT> ; return <NUM_LIT> ; } if ( AddrSpace == AS . FLAT_ADDRESS || AddrSpace == AS . LOCAL_ADDRESS || AddrSpace == AS . REGION_ADDRESS ) return <NUM_LIT> ; if ( AddrSpace == AS . PRIVATE_ADDRESS ) return <NUM_LIT> * ST -> getMaxPrivateElementSize ( ) ; if ( ST -> getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS && ( AddrSpace == AS . PARAM_D_ADDRESS || AddrSpace == AS . PARAM_I_ADDRESS || ( AddrSpace >= AS . CONSTANT_BUFFER_0 && AddrSpace <= AS . CONSTANT_BUFFER_15 ) ) ) return <NUM_LIT> ; llvm_unreachable ( <STR_LIT> ) ;


<s> <ISA_LIT>AS AS = ST -> get<ISA_LIT>AS ( ) ; if ( AddrSpace == AS . GLOBAL_ADDRESS || AddrSpace == AS . CONSTANT_ADDRESS || AddrSpace == AS . CONSTANT_ADDRESS_32BIT ) { if ( ST -> getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS ) return <NUM_LIT> ; return <NUM_LIT> ; } if ( AddrSpace == AS . FLAT_ADDRESS || AddrSpace == AS . LOCAL_ADDRESS || AddrSpace == AS . REGION_ADDRESS ) return <NUM_LIT> ; if ( AddrSpace == AS . PRIVATE_ADDRESS ) return <NUM_LIT> * ST -> getMaxPrivateElementSize ( ) ;

if ( ST -> getGeneration ( ) <= <ISA_LIT>Subtarget :: NORTHERN_ISLANDS && ( AddrSpace == AS . PARAM_D_ADDRESS || AddrSpace == AS . PARAM_I_ADDRESS || ( AddrSpace >= AS . CONSTANT_BUFFER_0 && AddrSpace <= AS . CONSTANT_BUFFER_15 ) ) ) return <NUM_LIT> ;


<s> explicit GCNTTIImpl ( const <ISA_LIT>TargetMachine * TM , const Function & F ) : BaseT ( TM , F . getParent ( ) -> getDataLayout ( ) ) , ST ( static_cast < const GCNSubtarget * > ( TM -> getSubtargetImpl ( F ) ) ) , TLI ( ST -> getTargetLowering ( ) ) , CommonTTI ( TM , F ) , IsGraphicsShader ( <STR_LIT> ( F . getCallingConv ( ) ) ) , MaxVGPRs ( ST -> getMaxNumVGPRs ( std :: max ( ST -> getWavesPerEU ( F ) . first , ST -> getWavesPerEUForWorkGroup ( ST -> getFlatWorkGroupSizes ( F ) . second ) ) ) ) { <STR_LIT> Mode ( F ) ; HasFP32Denormals = Mode . allFP32Denormals ( ) ; HasFP64FP16Denormals = Mode . allFP64FP16Denormals ( ) ;


<s> } if ( auto * VTy = dyn_cast < VectorType > ( Ty ) ) { unsigned Num = cast < FixedVectorType > ( VTy ) -> getNumElements ( ) ; unsigned Cost = getArithmeticInstrCost ( Opcode , VTy -> getScalarType ( ) , CostKind , Opd1Info , Opd2Info , Opd1PropInfo , Opd2PropInfo , Args , CxtI ) ; return getScalarizationOverhead ( VTy , Args ) + Num * Cost ; } return OpCost ; } std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Ty ) ; int ISD = TLI -> InstructionOpcodeToISD ( Opcode ) ; unsigned NElts = LT . second . isVector ( ) ? LT . second . getVectorNumElements ( ) : <NUM_LIT> ; <STR_LIT> SLT = LT . second . getScalarType ( ) . SimpleTy ; switch ( ISD ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) return get64BitInstrCost ( CostKind ) * LT . first * NElts ; if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; return getFullRateInstrCost ( ) * LT . first * NElts ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) { return <NUM_LIT> * getFullRateInstrCost ( ) * LT . first * NElts ; } if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; return LT . first * NElts * getFullRateInstrCost ( ) ; case <STR_LIT> : { const int QuarterRateCost = getQuarterRateInstrCost ( CostKind ) ; if ( SLT == <STR_LIT> ) { const int FullRateCost = getFullRateInstrCost ( ) ; return ( <NUM_LIT> * QuarterRateCost + ( <NUM_LIT> * <NUM_LIT> ) * FullRateCost ) * LT . first * NElts ; } if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; return QuarterRateCost * NElts * LT . first ; } case <STR_LIT> : if ( CxtI && CxtI -> hasOneUse ( ) ) if ( const auto * FAdd = dyn_cast < BinaryOperator > ( * CxtI -> user_begin ( ) ) ) { const int OPC = TLI -> InstructionOpcodeToISD ( FAdd -> getOpcode ( ) ) ; if ( OPC == <STR_LIT> || OPC == <STR_LIT> ) { if ( ST -> hasMadMacF32Insts ( ) && SLT == <STR_LIT> && ! HasFP32Denormals ) return TargetTransformInfo :: TCC_Free ; if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> && ! HasFP64FP16Denormals ) return TargetTransformInfo :: TCC_Free ; const TargetOptions & Options = TLI -> getTargetMachine ( ) . Options ; if ( Options . AllowFPOpFusion == FPOpFusion :: Fast || Options . UnsafeFPMath || ( FAdd -> hasAllowContract ( ) && CxtI -> hasAllowContract ( ) ) ) return TargetTransformInfo :: TCC_Free ; } } LLVM_FALLTHROUGH ;


<s> unsigned RetVF = ( RetTy -> isVectorTy ( ) ? cast < FixedVectorType > ( RetTy ) -> getNumElements ( ) : <NUM_LIT> ) ; assert ( ( RetVF == <NUM_LIT> || VF == <NUM_LIT> ) && <STR_LIT> ) ; const IntrinsicInst * I = ICA . getInst ( ) ; const SmallVectorImpl < const Value * > & Args = ICA . getArgs ( ) ; FastMathFlags FMF = ICA . getFlags ( ) ; SmallVector < Type * , <NUM_LIT> > Types ; for ( const Value * Op : Args ) { Type * OpTy = Op -> getType ( ) ; assert ( VF == <NUM_LIT> || ! OpTy -> isVectorTy ( ) ) ; Types . push_back ( VF == <NUM_LIT> ? OpTy : FixedVectorType :: get ( OpTy , VF ) ) ; } if ( VF > <NUM_LIT> && ! RetTy -> isVoidTy ( ) ) RetTy = FixedVectorType :: get ( RetTy , VF ) ; unsigned ScalarizationCost = std :: numeric_limits < unsigned > :: max ( ) ; if ( RetVF > <NUM_LIT> || VF > <NUM_LIT> ) { ScalarizationCost = <NUM_LIT> ; if ( ! RetTy -> isVoidTy ( ) ) ScalarizationCost += getScalarizationOverhead ( cast < VectorType > ( RetTy ) , true , false ) ; ScalarizationCost += getOperandsScalarizationOverhead ( Args , VF ) ; } IntrinsicCostAttributes Attrs ( ICA . getID ( ) , RetTy , Types , FMF , ScalarizationCost , I ) ; return getIntrinsicInstrCost ( Attrs , CostKind ) ; } std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , RetTy ) ; unsigned NElts = LT . second . isVector ( ) ? LT . second . getVectorNumElements ( ) : <NUM_LIT> ; <STR_LIT> SLT = LT . second . getScalarType ( ) . SimpleTy ; if ( SLT == <STR_LIT> ) return LT . first * NElts * get64BitInstrCost ( CostKind ) ; if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; unsigned InstRate = getQuarterRateInstrCost ( CostKind ) ; if ( ICA . getID ( ) == <STR_LIT> ) { InstRate = ST -> hasFastFMAF32 ( ) ? getHalfRateInstrCost ( CostKind ) : getQuarterRateInstrCost ( CostKind ) ; } return LT . first * NElts * InstRate ;


<s> EVT OrigTy = TLI -> getValueType ( DL , Ty ) ; if ( IsPairwise || ! ST -> hasVOP3PInsts ( ) || OrigTy . getScalarSizeInBits ( ) != <NUM_LIT> ) return BaseT :: getMinMaxReductionCost ( Ty , CondTy , IsPairwise , IsUnsigned , CostKind ) ;

std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Ty ) ;


<s> CommonTTI . getPeelingPreferences ( L , SE , PP ) ;


<s> continue ; } const GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( & I ) ; if ( ! GEP ) continue ; unsigned AS = GEP -> getAddressSpace ( ) ; unsigned Threshold = <NUM_LIT> ; if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) Threshold = ThresholdPrivate ; else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: REGION_ADDRESS ) Threshold = ThresholdLocal ; else continue ; if ( UP . Threshold >= Threshold ) continue ; if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { const Value * Ptr = GEP -> getPointerOperand ( ) ; const AllocaInst * Alloca = dyn_cast < AllocaInst > ( getUnderlyingObject ( Ptr ) ) ; if ( ! Alloca || ! Alloca -> isStaticAlloca ( ) ) continue ; Type * Ty = Alloca -> getAllocatedType ( ) ; unsigned AllocaSize = Ty -> isSized ( ) ? DL . getTypeAllocSize ( Ty ) : <NUM_LIT> ; if ( AllocaSize > MaxAlloca ) continue ; } else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: REGION_ADDRESS ) { LocalGEPsSeen ++ ; if ( LocalGEPsSeen > <NUM_LIT> || L -> getLoopDepth ( ) > <NUM_LIT> || ( ! isa < GlobalVariable > ( GEP -> getPointerOperand ( ) ) && ! isa < Argument > ( GEP -> getPointerOperand ( ) ) ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * L << <STR_LIT> ) ; UP . Runtime = UnrollRuntimeLocal ;


<s> if ( UP . Threshold >= Threshold ) continue ; if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { const Value * Ptr = GEP -> getPointerOperand ( ) ; const AllocaInst * Alloca = dyn_cast < AllocaInst > ( getUnderlyingObject ( Ptr ) ) ; if ( ! Alloca || ! Alloca -> isStaticAlloca ( ) ) continue ; Type * Ty = Alloca -> getAllocatedType ( ) ; unsigned AllocaSize = Ty -> isSized ( ) ? DL . getTypeAllocSize ( Ty ) : <NUM_LIT> ; if ( AllocaSize > MaxAlloca ) continue ; } else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: REGION_ADDRESS ) { LocalGEPsSeen ++ ; if ( LocalGEPsSeen > <NUM_LIT> || L -> getLoopDepth ( ) > <NUM_LIT> || ( ! isa < GlobalVariable > ( GEP -> getPointerOperand ( ) ) && ! isa < Argument > ( GEP -> getPointerOperand ( ) ) ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * L << <STR_LIT> ) ; UP . Runtime = UnrollRuntimeLocal ; } bool HasLoopDef = false ; for ( const Value * Op : GEP -> operands ( ) ) { const Instruction * Inst = dyn_cast < Instruction > ( Op ) ; if ( ! Inst || L -> isLoopInvariant ( Op ) ) continue ; if ( llvm :: any_of ( L -> getSubLoops ( ) , [ Inst ] ( const Loop * SubLoop ) { return SubLoop -> contains ( Inst ) ; } ) ) continue ; HasLoopDef = true ; break ; } if ( ! HasLoopDef ) continue ;


<s> static bool isArgPassedInSGPR ( const Argument * A ) { const Function * F = A -> getParent ( ) ; CallingConv :: ID CC = F -> getCallingConv ( ) ;


<s> return isLegalToVectorizeMemChain ( ChainSizeInBytes , Alignment , AddrSpace ) ;


<s> bool <ISA_LIT>TTIImpl :: isLegalToVectorizeMemChain ( unsigned ChainSizeInBytes , Align Alignment , unsigned AddrSpace ) const {

return ( AddrSpace != <ISA_LIT>AS :: PRIVATE_ADDRESS ) ;


<s> return ( AddrSpace != <ISA_LIT>AS :: PRIVATE_ADDRESS ) ;


<s> InstructionCost <ISA_LIT>TTIImpl :: getCFInstrCost ( unsigned Opcode , TTI :: TargetCostKind CostKind , const Instruction * I ) { if ( CostKind == TTI :: TCK_CodeSize || CostKind == TTI :: TCK_SizeAndLatency ) return Opcode == Instruction :: PHI ? <NUM_LIT> : <NUM_LIT> ; switch ( Opcode ) { case Instruction :: Br : case Instruction :: Ret : return <NUM_LIT> ; default :

return BaseT :: getCFInstrCost ( Opcode , CostKind , I ) ;


<s> return TypeSize :: getFixed ( <NUM_LIT> ) ;


<s> TypeSize <ISA_LIT>TTIImpl :: getRegisterBitWidth ( TargetTransformInfo :: RegisterKind K ) const {

return TypeSize :: getFixed ( <NUM_LIT> ) ;


<s> if ( ! Alloca || ! Alloca -> isStaticAlloca ( ) ) continue ; Type * Ty = Alloca -> getAllocatedType ( ) ; unsigned AllocaSize = Ty -> isSized ( ) ? DL . getTypeAllocSize ( Ty ) : <NUM_LIT> ; if ( AllocaSize > MaxAlloca ) continue ; } else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: REGION_ADDRESS ) { LocalGEPsSeen ++ ; if ( LocalGEPsSeen > <NUM_LIT> || L -> getLoopDepth ( ) > <NUM_LIT> || ( ! isa < GlobalVariable > ( GEP -> getPointerOperand ( ) ) && ! isa < Argument > ( GEP -> getPointerOperand ( ) ) ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * L << <STR_LIT> ) ; UP . Runtime = UnrollRuntimeLocal ; } bool HasLoopDef = false ; for ( const Value * Op : GEP -> operands ( ) ) { const Instruction * Inst = dyn_cast < Instruction > ( Op ) ; if ( ! Inst || L -> isLoopInvariant ( Op ) ) continue ; if ( llvm :: any_of ( L -> getSubLoops ( ) , [ Inst ] ( const Loop * SubLoop ) { return SubLoop -> contains ( Inst ) ; } ) ) continue ; HasLoopDef = true ; break ; } if ( ! HasLoopDef ) continue ; UP . Threshold = Threshold ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Threshold << <STR_LIT> << * L << <STR_LIT> << * GEP << '\n' ) ; if ( UP . Threshold >= MaxBoost ) return ; } if ( L -> isInnermost ( ) && BB -> size ( ) < UnrollMaxBlockToAnalyze ) UP . MaxIterationsCountToAnalyze = <NUM_LIT> ; }


<s> } const GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( & I ) ; if ( ! GEP ) continue ; unsigned AS = GEP -> getAddressSpace ( ) ; unsigned Threshold = <NUM_LIT> ; if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) Threshold = ThresholdPrivate ; else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: REGION_ADDRESS ) Threshold = ThresholdLocal ; else continue ; if ( UP . Threshold >= Threshold ) continue ; if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { const Value * Ptr = GEP -> getPointerOperand ( ) ; const AllocaInst * Alloca = dyn_cast < AllocaInst > ( getUnderlyingObject ( Ptr ) ) ; if ( ! Alloca || ! Alloca -> isStaticAlloca ( ) ) continue ; Type * Ty = Alloca -> getAllocatedType ( ) ; unsigned AllocaSize = Ty -> isSized ( ) ? DL . getTypeAllocSize ( Ty ) : <NUM_LIT> ; if ( AllocaSize > MaxAlloca ) continue ; } else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: REGION_ADDRESS ) { LocalGEPsSeen ++ ; if ( LocalGEPsSeen > <NUM_LIT> || L -> getLoopDepth ( ) > <NUM_LIT> || ( ! isa < GlobalVariable > ( GEP -> getPointerOperand ( ) ) && ! isa < Argument > ( GEP -> getPointerOperand ( ) ) ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * L << <STR_LIT> ) ; UP . Runtime = UnrollRuntimeLocal ; } bool HasLoopDef = false ;

for ( const Value * Op : GEP -> operands ( ) ) {


<s> unsigned EltSize = DL . getTypeSizeInBits ( cast < VectorType > ( ValTy ) -> getElementType ( ) ) ; if ( EltSize < <NUM_LIT> ) { return BaseT :: getVectorInstrCost ( Opcode , ValTy , Index ) ;


<s> <ISA_LIT>TTIImpl :: <ISA_LIT>TTIImpl ( const <ISA_LIT>TargetMachine * TM , const Function & F ) : BaseT ( TM , F . getParent ( ) -> getDataLayout ( ) ) , ST ( static_cast < const <ISA_LIT>Subtarget * > ( TM -> getSubtargetImpl ( F ) ) ) , TLI ( ST -> getTargetLowering ( ) ) , CommonTTI ( TM , F ) {


<s> UP . Threshold = <NUM_LIT> ; UP . MaxCount = UINT_MAX ; UP . Partial = true ; const unsigned MaxAlloca = ( <NUM_LIT> - <NUM_LIT> ) * <NUM_LIT> ; for ( const BasicBlock * BB : L -> getBlocks ( ) ) { const DataLayout & DL = BB -> getModule ( ) -> getDataLayout ( ) ; for ( const Instruction & I : * BB ) { const GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( & I ) ; if ( ! GEP || GEP -> getAddressSpace ( ) != <ISA_LIT>AS :: PRIVATE_ADDRESS ) continue ; const Value * Ptr = GEP -> getPointerOperand ( ) ; const AllocaInst * Alloca = dyn_cast < AllocaInst > ( GetUnderlyingObject ( Ptr , DL ) ) ; if ( Alloca && Alloca -> isStaticAlloca ( ) ) { Type * Ty = Alloca -> getAllocatedType ( ) ; unsigned AllocaSize = Ty -> isSized ( ) ? DL . getTypeAllocSize ( Ty ) : <NUM_LIT> ; if ( AllocaSize > MaxAlloca ) continue ;


<s> for ( const Instruction & I : * BB ) { const GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( & I ) ; if ( ! GEP || GEP -> getAddressSpace ( ) != <ISA_LIT>AS :: PRIVATE_ADDRESS ) continue ; const Value * Ptr = GEP -> getPointerOperand ( ) ; const AllocaInst * Alloca = dyn_cast < AllocaInst > ( GetUnderlyingObject ( Ptr , DL ) ) ; if ( Alloca && Alloca -> isStaticAlloca ( ) ) { Type * Ty = Alloca -> getAllocatedType ( ) ; unsigned AllocaSize = Ty -> isSized ( ) ? DL . getTypeAllocSize ( Ty ) : <NUM_LIT> ;

if ( AllocaSize > MaxAlloca ) continue ;


<s> bool <ISA_LIT>TTIImpl :: isSourceOfDivergence ( const Value * V ) const { if ( const Argument * A = dyn_cast < Argument > ( V ) ) return ! isArgPassedInSGPR ( A ) ; if ( const LoadInst * Load = dyn_cast < LoadInst > ( V ) ) return Load -> getPointerAddressSpace ( ) == <ISA_LIT>AS :: PRIVATE_ADDRESS ; if ( isa < AtomicRMWInst > ( V ) || isa < AtomicCmpXchgInst > ( V ) ) return true ; if ( const IntrinsicInst * Intrinsic = dyn_cast < IntrinsicInst > ( V ) ) return isIntrinsicSourceOfDivergence ( Intrinsic ) ; if ( isa < CallInst > ( V ) || isa < InvokeInst > ( V ) ) return true ; return false ;


<s> SmallVector < Type * , <NUM_LIT> > Types ; for ( const Value * Op : Args ) { Type * OpTy = Op -> getType ( ) ; assert ( VF == <NUM_LIT> || ! OpTy -> isVectorTy ( ) ) ; Types . push_back ( VF == <NUM_LIT> ? OpTy : FixedVectorType :: get ( OpTy , VF ) ) ; } if ( VF > <NUM_LIT> && ! RetTy -> isVoidTy ( ) ) RetTy = FixedVectorType :: get ( RetTy , VF ) ; unsigned ScalarizationCost = std :: numeric_limits < unsigned > :: max ( ) ; if ( RetVF > <NUM_LIT> || VF > <NUM_LIT> ) { ScalarizationCost = <NUM_LIT> ; if ( ! RetTy -> isVoidTy ( ) ) ScalarizationCost += getScalarizationOverhead ( cast < VectorType > ( RetTy ) , true , false ) ; ScalarizationCost += getOperandsScalarizationOverhead ( Args , VF ) ; } IntrinsicCostAttributes Attrs ( ICA . getID ( ) , RetTy , Types , FMF , ScalarizationCost , I ) ; return getIntrinsicInstrCost ( Attrs , CostKind ) ; } std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , RetTy ) ; unsigned NElts = LT . second . isVector ( ) ? LT . second . getVectorNumElements ( ) : <NUM_LIT> ; <STR_LIT> SLT = LT . second . getScalarType ( ) . SimpleTy ; if ( SLT == <STR_LIT> ) return LT . first * NElts * get64BitInstrCost ( CostKind ) ; if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; unsigned InstRate = getQuarterRateInstrCost ( CostKind ) ;


<s> explicit GCNTTIImpl ( const <ISA_LIT>TargetMachine * TM , const Function & F ) : BaseT ( TM , F . getParent ( ) -> getDataLayout ( ) ) , ST ( static_cast < const <ISA_LIT>Subtarget * > ( TM -> getSubtargetImpl ( F ) ) ) , TLI ( ST -> getTargetLowering ( ) ) , CommonTTI ( TM , F ) , IsGraphicsShader ( <STR_LIT> ( F . getCallingConv ( ) ) ) {


<s> LLVM_FALLTHROUGH ; case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) return LT . first * NElts * get64BitInstrCost ( ) ; if ( ST -> has16BitInsts ( ) && SLT == <STR_LIT> ) NElts = ( NElts + <NUM_LIT> ) / <NUM_LIT> ; if ( SLT == <STR_LIT> || SLT == <STR_LIT> ) return LT . first * NElts * getFullRateInstrCost ( ) ; break ; case <STR_LIT> : case <STR_LIT> : if ( SLT == <STR_LIT> ) { int Cost = <NUM_LIT> * get64BitInstrCost ( ) + <NUM_LIT> * getQuarterRateInstrCost ( ) ; if ( ! ST -> hasUsableDivScaleConditionOutput ( ) ) Cost += <NUM_LIT> * getFullRateInstrCost ( ) ; return LT . first * Cost * NElts ; } if ( ! Args . empty ( ) && match ( Args [ <NUM_LIT> ] , PatternMatch :: m_FPOne ( ) ) ) { if ( ( SLT == <STR_LIT> && ! HasFP32Denormals ) || ( SLT == <STR_LIT> && ST -> has16BitInsts ( ) ) ) { return LT . first * getQuarterRateInstrCost ( ) * NElts ; } } if ( SLT == <STR_LIT> && ST -> has16BitInsts ( ) ) { int Cost = <NUM_LIT> * getFullRateInstrCost ( ) + <NUM_LIT> * getQuarterRateInstrCost ( ) ; return LT . first * Cost * NElts ; } if ( SLT == <STR_LIT> || SLT == <STR_LIT> ) { int Cost = <NUM_LIT> * getFullRateInstrCost ( ) + <NUM_LIT> * getQuarterRateInstrCost ( ) ; if ( ! HasFP32Denormals ) { Cost += <NUM_LIT> * getFullRateInstrCost ( ) ; } return LT . first * NElts * Cost ; } break ; case <STR_LIT> :

return TLI -> isFNegFree ( SLT ) ? <NUM_LIT> : NElts ;


<s> int GCNTTIImpl :: getIntrinsicInstrCost ( <STR_LIT> ID , Type * RetTy , ArrayRef < Type * > Tys , FastMathFlags FMF , unsigned ScalarizationCostPassed , const Instruction * I ) {

return getIntrinsicInstrCost < Type > ( ID , RetTy , Tys , FMF , ScalarizationCostPassed , I ) ;


<s> default : return false ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; } } if ( const CallInst * CI = dyn_cast < CallInst > ( V ) ) { if ( isa < InlineAsm > ( CI -> getCalledValue ( ) ) ) return ! isInlineAsmSourceOfDivergence ( CI ) ; return false ; } const ExtractValueInst * ExtValue = dyn_cast < ExtractValueInst > ( V ) ; if ( ! ExtValue ) return false ; const CallInst * CI = dyn_cast < CallInst > ( ExtValue -> getOperand ( <NUM_LIT> ) ) ; if ( ! CI ) return false ; if ( const IntrinsicInst * Intrinsic = dyn_cast < IntrinsicInst > ( CI ) ) { switch ( Intrinsic -> getIntrinsicID ( ) ) { default : return false ; case <STR_LIT> : case <STR_LIT> : { ArrayRef < unsigned > Indices = ExtValue -> getIndices ( ) ; return Indices . size ( ) == <NUM_LIT> && Indices [ <NUM_LIT> ] == <NUM_LIT> ; } }


<s> bool GCNTTIImpl :: isInlineAsmSourceOfDivergence ( const CallInst * CI , ArrayRef < unsigned > Indices ) const { if ( Indices . size ( ) > <NUM_LIT> ) return true ; const DataLayout & DL = CI -> getModule ( ) -> getDataLayout ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST -> getRegisterInfo ( ) ; ImmutableCallSite CS ( CI ) ; TargetLowering :: AsmOperandInfoVector TargetConstraints = TLI -> ParseConstraints ( DL , ST -> getRegisterInfo ( ) , CS ) ; const int TargetOutputIdx = Indices . empty ( ) ? - <NUM_LIT> : Indices [ <NUM_LIT> ] ; int OutputIdx = <NUM_LIT> ;


<s> bool GCNTTIImpl :: isSourceOfDivergence ( const Value * V ) const { if ( const Argument * A = dyn_cast < Argument > ( V ) ) return ! isArgPassedInSGPR ( A ) ; if ( const LoadInst * Load = dyn_cast < LoadInst > ( V ) ) return Load -> getPointerAddressSpace ( ) == <ISA_LIT>AS :: PRIVATE_ADDRESS || Load -> getPointerAddressSpace ( ) == <ISA_LIT>AS :: FLAT_ADDRESS ; if ( isa < AtomicRMWInst > ( V ) || isa < AtomicCmpXchgInst > ( V ) ) return true ; if ( const IntrinsicInst * Intrinsic = dyn_cast < IntrinsicInst > ( V ) ) return <STR_LIT> ( Intrinsic -> getIntrinsicID ( ) ) ; if ( const CallInst * CI = dyn_cast < CallInst > ( V ) ) { if ( isa < InlineAsm > ( CI -> getCalledValue ( ) ) ) return isInlineAsmSourceOfDivergence ( CI ) ; return true ; }

if ( isa < InvokeInst > ( V ) ) return true ;


<s> if ( isa < AtomicRMWInst > ( V ) || isa < AtomicCmpXchgInst > ( V ) ) return true ; if ( const IntrinsicInst * Intrinsic = dyn_cast < IntrinsicInst > ( V ) ) return <STR_LIT> ( Intrinsic -> getIntrinsicID ( ) ) ; if ( const CallInst * CI = dyn_cast < CallInst > ( V ) ) {

if ( isa < InlineAsm > ( CI -> getCalledValue ( ) ) ) return isInlineAsmSourceOfDivergence ( CI ) ;


<s> explicit <ISA_LIT>TTIImpl ( const <ISA_LIT>TargetMachine * TM , const Function & F ) : BaseT ( TM , F . getParent ( ) -> getDataLayout ( ) ) , ST ( TM -> getSubtargetImpl ( F ) ) , TLI ( ST -> getTargetLowering ( ) ) {


<s> void <ISA_LIT>TTIImpl :: getUnrollingPreferences ( Loop * L , ScalarEvolution & SE , TTI :: UnrollingPreferences & UP , OptimizationRemarkEmitter * ORE ) {

CommonTTI . getUnrollingPreferences ( L , SE , UP , ORE ) ;


<s> CommonTTI . getUnrollingPreferences ( L , SE , UP , ORE ) ;