<s> unsigned SubVecSize = DL . getTypeAllocSizeInBits ( SubVecTy ) ; if ( NumElts % Factor == <NUM_LIT> && ( SubVecSize == <NUM_LIT> || SubVecSize == <NUM_LIT> ) ) return Factor ; } return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor ,


<s> return BaseT :: getShuffleCost ( Kind , Tp , Index , SubTp ) ; } if ( Kind == TTI :: SK_Alternate ) { static const CostTblEntry < <STR_LIT> > NEONAltShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( NEONAltShuffleTbl , <STR_LIT> , LT .

second ) ) return LT . first * Entry -> Cost ;


<s> unsigned <STR_LIT> ( unsigned Opcode , Type * ValTy , Type * CondTy ) const { int ISD = TLI -> InstructionOpcodeToISD


<s> unsigned <STR_LIT> ( unsigned Opcode , Type * ValTy , Type * CondTy ) const { int ISD = TLI -> InstructionOpcodeToISD ( Opcode ) ; if ( ST -> hasNEON ( ) && ValTy ->


<s> bool isLegalMaskedScatter ( Type * Ty , MaybeAlign Alignment ) { return


<s> return ST -> isMClass ( ) && ST -> isThumb2 ( ) && L -> getNumBlocks (


<s> if ( isa < FPExtInst > ( & I ) || isa < FPTruncInst > ( & I ) ) return false ; if ( isa < SExtInst > ( & I ) || isa < ZExtInst > ( & I ) ) if ( ! I . getOperand ( <NUM_LIT> ) -> hasOneUse ( ) || ! isa < LoadInst > ( I . getOperand ( <NUM_LIT> ) ) ) return false ; if ( isa < TruncInst > ( & I ) ) if ( ! I . hasOneUse ( ) || ! isa < StoreInst > ( * I . user_begin ( ) ) ) return false ; return


<s> if ( isa < ICmpInst > ( & I ) && ++ ICmpCount > <NUM_LIT> ) return false ; if ( auto * II = dyn_cast < IntrinsicInst > ( & I ) ) if ( ( II -> getIntrinsicID ( ) == <STR_LIT> || II -> getIntrinsicID ( ) == <STR_LIT> || II -> getIntrinsicID ( ) == <STR_LIT> || II -> getIntrinsicID ( ) == <STR_LIT> ) && ++ ICmpCount > <NUM_LIT> ) return false ; if ( isa < FCmpInst > ( & I


<s> if ( isa < PHINode > ( & I ) ) continue ; if ( ! canTailPredicateInstruction ( I , ICmpCount ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; I . dump ( ) ) ; return false ; } Type * T = I . getType ( ) ; if ( T -> isPointerTy ( ) ) T = T -> getPointerElementType ( ) ; if ( T -> getScalarSizeInBits ( ) > <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; T -> dump ( ) ) ; return false ; } if ( isa < StoreInst > ( I ) || isa < LoadInst > ( I ) ) { Value * Ptr = getLoadStorePointerOperand ( & I ) ; Type * AccessTy = getLoadStoreType ( & I ) ; int64_t NextStride = getPtrStride ( PSE , AccessTy , Ptr , L ) ; if ( NextStride == <NUM_LIT> ) { continue ; } else if ( NextStride == - <NUM_LIT> || ( NextStride == <NUM_LIT> && MVEMaxSupportedInterleaveFactor >= <NUM_LIT> ) || ( NextStride == <NUM_LIT> && MVEMaxSupportedInterleaveFactor >= <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } else if ( EnableMaskedGatherScatters ) { const SCEV * PtrScev = PSE . getSE ( )


<s> unsigned NumVectorInstToHideOverhead = <NUM_LIT> ; int MaxMergeDistance = <NUM_LIT> ; if ( ST -> hasNEON ( ) ) { if ( Ty -> isVectorTy ( ) && SE && ! BaseT :: isConstantStridedAccessLessThan ( SE , Ptr , MaxMergeDistance + <NUM_LIT> ) ) return NumVectorInstToHideOverhead ; return <NUM_LIT>


<s> std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ; static const CostTblEntry CostTblAdd [ ] { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , } ; if ( const auto * Entry = CostTableLookup ( CostTblAdd , ISD , LT . second ) ) return Entry -> Cost * ST -> getMVEVectorCostFactor (


<s> if ( CostKind == TTI :: TCK_RecipThroughput && ( ST -> hasNEON ( ) || ST -> hasMVEIntegerOps ( ) ) ) { return <NUM_LIT> ; } return BaseT :: getCFInstrCost ( Opcode , CostKind , I


<s> if ( ST -> hasMVEIntegerOps ( ) && ValTy -> isVectorTy ( ) && ( Opcode == Instruction :: ICmp || Opcode == Instruction :: FCmp ) && cast < FixedVectorType > ( ValTy ) -> getNumElements ( ) > <NUM_LIT> ) { FixedVectorType * VecValTy = cast < FixedVectorType > ( ValTy ) ; FixedVectorType * VecCondTy = dyn_cast_or_null < FixedVectorType > ( CondTy ) ; if ( ! VecCondTy ) VecCondTy = cast < FixedVectorType > ( CmpInst :: makeCmpResultType ( VecValTy ) ) ; if ( Opcode == Instruction :: FCmp && ! ST -> hasMVEFloatOps ( ) ) { return BaseT :: getScalarizationOverhead ( VecValTy , false , true ) + BaseT :: getScalarizationOverhead ( VecCondTy , true , false ) + VecValTy -> getNumElements ( ) * getCmpSelInstrCost ( Opcode , ValTy -> getScalarType ( ) , VecCondTy -> getScalarType ( ) , VecPred , CostKind , I ) ; } std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ; int BaseCost = ST -> getMVEVectorCostFactor ( CostKind ) ; if ( LT . second . getVectorNumElements ( ) > <NUM_LIT> ) { if ( LT . first > <NUM_LIT> ) return LT . first * BaseCost + BaseT :: getScalarizationOverhead ( VecCondTy , true , false ) ; return BaseCost ; } } int BaseCost = <NUM_LIT> ; if ( ST -> hasMVEIntegerOps ( ) && ValTy -> isVectorTy ( ) ) BaseCost =


<s> InstructionCost <ISA_LIT>TTIImpl :: getExtendedAddReductionCost ( bool IsMLA , bool IsUnsigned , Type * ResTy , VectorType * ValTy , TTI :: TargetCostKind CostKind ) { EVT ValVT = TLI -> getValueType ( DL , ValTy ) ; EVT ResVT = TLI -> getValueType ( DL , ResTy ) ; if ( ST -> hasMVEIntegerOps ( ) && ValVT .

isSimple ( ) && ResVT . isSimple ( ) ) {


<s> unsigned NumElems = VTy -> getNumElements ( ) ; unsigned EltSize = VTy -> getScalarSizeInBits ( ) ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , DataTy ) ; InstructionCost VectorCost = NumElems * LT . first * ST -> getMVEVectorCostFactor ( CostKind ) ; InstructionCost ScalarCost = NumElems * LT . first + BaseT :: getScalarizationOverhead ( VTy , true , false ) + BaseT :: getScalarizationOverhead ( VTy , false , true ) ; if ( EltSize < <NUM_LIT> || Alignment < EltSize / <NUM_LIT> ) return ScalarCost ; unsigned ExtSize = EltSize ; if ( I != nullptr ) { if ( ( I -> getOpcode ( ) == Instruction :: Load || match ( I , m_Intrinsic < <STR_LIT> > ( ) ) ) && I -> hasOneUse ( ) ) { const User * Us = * I -> users ( ) . begin ( ) ; if ( isa < ZExtInst > ( Us ) || isa < SExtInst > ( Us ) ) { unsigned TypeSize = cast < Instruction > ( Us ) -> getType ( ) -> getScalarSizeInBits ( ) ; if ( ( ( TypeSize == <NUM_LIT> && ( EltSize == <NUM_LIT> || EltSize == <NUM_LIT> ) ) || ( TypeSize == <NUM_LIT> && EltSize == <NUM_LIT> ) ) && TypeSize * NumElems == <NUM_LIT> ) { ExtSize = TypeSize ; } } } TruncInst * T ; if ( ( I -> getOpcode ( ) == Instruction :: Store || match ( I , m_Intrinsic < <STR_LIT> > ( ) ) ) && ( T = dyn_cast < TruncInst > ( I -> getOperand ( <NUM_LIT> ) ) ) ) { unsigned TypeSize = T -> getOperand ( <NUM_LIT> ) -> getType ( ) -> getScalarSizeInBits ( ) ; if ( ( ( EltSize == <NUM_LIT> && TypeSize == <NUM_LIT> ) || ( EltSize == <NUM_LIT> && ( TypeSize == <NUM_LIT> || TypeSize == <NUM_LIT> ) ) ) && TypeSize * NumElems == <NUM_LIT> ) ExtSize = TypeSize ; } } if ( ExtSize * NumElems != <NUM_LIT> || NumElems < <NUM_LIT> ) return ScalarCost ; if ( ExtSize == <NUM_LIT> )


<s> if ( NumElts % Factor == <NUM_LIT> && TLI -> isLegalInterleavedAccessType ( Factor , SubVecTy , Alignment , DL ) ) return Factor * BaseCost * TLI -> getNumInterleavedAccesses ( SubVecTy , DL ) ; if ( ST -> hasMVEIntegerOps ( ) && Factor == <NUM_LIT> && NumElts / Factor > <NUM_LIT> && VecTy -> isIntOrIntVectorTy ( ) && DL . getTypeSizeInBits ( SubVecTy ) . getFixedSize ( ) <= <NUM_LIT> ) return <NUM_LIT> * BaseCost ; } return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment

, AddressSpace , CostKind , UseMaskForCond , UseMaskForGaps ) ;


<s> InstructionCost <ISA_LIT>TTIImpl :: getIntImmCodeSizeCost ( unsigned Opcode , unsigned Idx , const


<s> unsigned Bits = Ty -> getPrimitiveSizeInBits ( ) ; if ( Bits == <NUM_LIT> || Imm . getActiveBits ( ) >= <NUM_LIT> ) return <NUM_LIT> ; int64_t SImmVal = Imm . getSExtValue ( ) ; uint64_t ZImmVal = Imm . getZExtValue ( ) ; if ( ! ST -> isThumb ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ; } if ( ST -> isThumb2 ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ; } if ( Bits == <NUM_LIT> || ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) ) return <NUM_LIT> ; if ( ( ~ SImmVal < <NUM_LIT> ) || <STR_LIT> ( ZImmVal ) ) return <NUM_LIT> ; return


<s> int64_t SImmVal = Imm . getSExtValue ( ) ; uint64_t ZImmVal = Imm . getZExtValue ( ) ; if ( ! ST -> isThumb ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ; } if ( ST -> isThumb2 ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( )


<s> if ( Imm == <NUM_LIT> || Imm == <NUM_LIT> ) return <NUM_LIT> ; return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( ~ Imm , Ty , CostKind ) ) ; } if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( - Imm , Ty , CostKind ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == <NUM_LIT> ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; if ( ST -> isThumb ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; } if ( Opcode == Instruction :: Xor && Imm . isAllOnes ( ) ) return <NUM_LIT> ; if ( Inst && ( ( ST -> hasV6Ops ( ) && ! ST -> isThumb ( ) ) || ST -> isThumb2 ( ) ) && Ty -> getIntegerBitWidth ( ) <= <NUM_LIT> ) { if ( isSSATMinMaxPattern ( Inst , Imm ) || ( isa < ICmpInst > ( Inst ) && Inst -> hasOneUse ( ) && isSSATMinMaxPattern ( cast < Instruction

> ( * Inst -> user_begin ( ) ) , Imm ) ) ) return <NUM_LIT> ;


<s> if ( Opcode == Instruction :: GetElementPtr && Idx != <NUM_LIT> ) return <NUM_LIT> ; if ( Opcode == Instruction :: And ) { if ( Imm == <NUM_LIT> || Imm == <NUM_LIT> ) return <NUM_LIT> ; return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( ~ Imm , Ty , CostKind ) ) ; } if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( - Imm , Ty , CostKind ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == <NUM_LIT> ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; if ( ST -> isThumb ( ) && NegImm < <NUM_LIT> << <NUM_LIT> )


<s> if ( ST -> hasMVEIntegerOps ( ) ) { if ( Opcode == Instruction :: Load && isLegalMaskedLoad ( Src , Alignment ) ) return ST -> getMVEVectorCostFactor ( CostKind ) ; if ( Opcode == Instruction :: Store && isLegalMaskedStore ( Src , Alignment ) ) return ST


<s> InstructionCost <ISA_LIT>TTIImpl :: getMemoryOpCost ( unsigned Opcode , Type * Src , MaybeAlign Alignment , unsigned AddressSpace , TTI :: TargetCostKind CostKind , const Instruction * I ) { if ( CostKind != TTI :: TCK_RecipThroughput ) return <NUM_LIT> ; if ( TLI -> getValueType ( DL , Src , true ) == <STR_LIT> ) return BaseT :: getMemoryOpCost ( Opcode , Src , Alignment , AddressSpace , CostKind ) ; if ( ST -> hasNEON ( ) && Src -> isVectorTy ( ) && ( Alignment && * Alignment != Align ( <NUM_LIT> ) ) && cast < VectorType > ( Src ) -> getElementType ( ) -> isDoubleTy ( ) ) { std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Src ) ; return LT . first * <NUM_LIT> ; } if ( ST -> hasMVEFloatOps ( ) && isa < FixedVectorType > ( Src ) && I && ( ( Opcode == Instruction :: Load && I -> hasOneUse ( ) && isa < FPExtInst > ( * I -> user_begin

( ) ) ) || ( Opcode == Instruction :: Store && isa < FPTruncInst > ( I -> getOperand ( <NUM_LIT> ) ) ) ) ) {


<s> } else if ( const auto * MS = dyn_cast < MemSetInst > ( I ) ) { ConstantInt * C = dyn_cast < ConstantInt > ( MS -> getLength ( ) ) ; if ( ! C ) return - <NUM_LIT> ; const unsigned Size = C -> getValue ( ) . getZExtValue ( ) ; const Align DstAlign = * MS -> getDestAlign ( ) ; MOp = MemOp :: Set ( Size , false , DstAlign , false , false ) ; DstAddrSpace = MS -> getDestAddressSpace ( ) ; } else llvm_unreachable ( <STR_LIT> ) ; unsigned Limit , Factor = <NUM_LIT> ; switch ( I -> getIntrinsicID ( ) ) { case <STR_LIT> : Limit = TLI -> getMaxStoresPerMemcpy ( F -> hasMinSize ( ) ) ; break ; case <STR_LIT> : Limit = TLI -> getMaxStoresPerMemmove ( F -> hasMinSize ( ) ) ; break ; case <STR_LIT> : Limit = TLI -> getMaxStoresPerMemset ( F -> hasMinSize (


<s> void <ISA_LIT>TTIImpl :: getPeelingPreferences ( Loop * L , ScalarEvolution & SE , TTI ::


<s> if ( ST -> isMClass ( ) && ST -> isThumb2 ( ) && L -> getNumBlocks ( ) == <NUM_LIT> ) return TTI :: AMK_PreIndexed ; return TTI


<s> if ( const auto * Entry = CostTableLookup ( NEONSelShuffleTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost ; } } if ( ST -> hasMVEIntegerOps ( ) ) { if ( Kind == TTI :: SK_Broadcast ) { static const CostTblEntry MVEDupTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( MVEDupTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost * ST -> getMVEVectorCostFactor ( TTI :: TCK_RecipThroughput ) ; } if ( ! Mask . empty ( ) ) { std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( Mask . size ( ) <= LT . second . getVectorNumElements ( ) && ( isVREVMask ( Mask , LT . second , <NUM_LIT> ) || isVREVMask ( Mask , LT . second , <NUM_LIT> ) || isVREVMask ( Mask , LT . second , <NUM_LIT> ) ) ) return ST -> getMVEVectorCostFactor ( TTI :: TCK_RecipThroughput ) * LT . first ; } } int BaseCost = ST -> hasMVEIntegerOps ( ) && Tp -> isVectorTy ( ) ? ST -> getMVEVectorCostFactor ( TTI :: TCK_RecipThroughput ) : <NUM_LIT> ; return BaseCost * BaseT :: getShuffleCost ( Kind , Tp , Mask


<s> Value * Arg = II . getArgOperand ( <NUM_LIT> ) ; Value * ArgArg ; if ( match ( Arg , PatternMatch :: m_Intrinsic < <STR_LIT> > ( PatternMatch :: m_Value ( ArgArg ) ) ) ) { return IC . replaceInstUsesWith ( II , ArgArg ) ; } if ( ! II . getMetadata ( LLVMContext :: MD_range ) ) { Type * IntTy32 = Type :: getInt32Ty ( II . getContext ( ) ) ; Metadata * M [ ] = { ConstantAsMetadata :: get ( ConstantInt :: get ( IntTy32 , <NUM_LIT> ) ) , ConstantAsMetadata :: get ( ConstantInt :: get ( IntTy32 , <NUM_LIT> ) ) } ; II . setMetadata ( LLVMContext :: MD_range , MDNode :: get ( II . getContext ( ) , M ) ) ; return & II ; } break ; } case <STR_LIT> : case <STR_LIT> : { unsigned CarryOp = ( II . getIntrinsicID ( ) == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; assert ( II . getArgOperand ( CarryOp ) -> getType ( ) -> getScalarSizeInBits ( ) == <NUM_LIT> && <STR_LIT> ) ; KnownBits CarryKnown ( <NUM_LIT> ) ; if ( IC . SimplifyDemandedBits ( & II , CarryOp , APInt :: getOneBitSet ( <NUM_LIT> , <NUM_LIT> ) , CarryKnown ) ) { return & II ; } break ; } case <STR_LIT> : { Instruction * I = cast < Instruction > ( & II ) ; if ( I -> hasOneUse ( ) ) { auto * User = cast < Instruction > ( * I -> user_begin ( ) ) ; Value * OpZ ; if ( match ( User , m_c_Add ( m_Specific ( I ) , m_Value ( OpZ ) ) ) && match ( I -> getOperand ( <NUM_LIT> ) , m_Zero ( ) ) ) { Value * OpX = I -> getOperand ( <NUM_LIT> ) ; Value * OpY = I -> getOperand ( <NUM_LIT> ) ; Type * OpTy = OpX -> getType ( ) ; IC . Builder . SetInsertPoint ( User ) ; Value * V = IC . Builder . CreateIntrinsic ( <STR_LIT> , { OpTy } , { I -> getOperand ( <NUM_LIT> ) , I -> getOperand ( <NUM_LIT> ) , I -> getOperand ( <NUM_LIT> ) , OpZ , OpX , OpY } ) ; IC . replaceInstUsesWith ( *


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { Align MemAlign = getKnownAlignment ( II . getArgOperand ( <NUM_LIT> ) , IC . getDataLayout ( ) , & II , & IC . getAssumptionCache ( ) , & IC . getDominatorTree ( ) ) ; unsigned AlignArg = II . arg_size ( ) - <NUM_LIT> ; Value * AlignArgOp = II . getArgOperand ( AlignArg ) ; MaybeAlign Align = cast < ConstantInt > ( AlignArgOp ) -> getMaybeAlignValue ( ) ; if ( Align && * Align < MemAlign ) { return IC . replaceOperand ( II , AlignArg , ConstantInt :: get ( Type :: getInt32Ty ( II . getContext ( ) ) , MemAlign . value ( ) , false ) ) ; } break ; } case <STR_LIT> : { Value * Arg = II . getArgOperand ( <NUM_LIT> ) ; Value * ArgArg ; if ( match ( Arg , PatternMatch :: m_Intrinsic < <STR_LIT> > ( PatternMatch :: m_Value ( ArgArg ) ) ) && II . getType ( ) == ArgArg -> getType ( ) ) { return IC . replaceInstUsesWith ( II , ArgArg ) ; } Constant * XorMask ; if ( match ( Arg , m_Xor ( PatternMatch :: m_Intrinsic < <STR_LIT> > ( PatternMatch :: m_Value ( ArgArg ) ) , PatternMatch :: m_Constant ( XorMask ) ) ) && II . getType ( ) == ArgArg -> getType ( ) ) { if ( auto * CI = dyn_cast < ConstantInt > ( XorMask ) ) { if ( CI -> getValue ( ) . trunc ( <NUM_LIT> ) . isAllOnes ( ) ) { auto TrueVector = IC . Builder . CreateVectorSplat ( cast < FixedVectorType > ( II . getType ( ) ) -> getNumElements ( ) , IC . Builder . getTrue ( ) ) ; return BinaryOperator :: Create ( Instruction :: Xor , ArgArg , TrueVector ) ; } } } KnownBits ScalarKnown ( <NUM_LIT> ) ; if ( IC . SimplifyDemandedBits ( & II , <NUM_LIT> , APInt :: getLowBitsSet ( <NUM_LIT> , <NUM_LIT> ) , ScalarKnown , <NUM_LIT> ) ) { return & II ; } break ; } case <STR_LIT> : { Value * Arg = II . getArgOperand ( <NUM_LIT> ) ; Value * ArgArg ; if ( match ( Arg , PatternMatch :: m_Intrinsic < <STR_LIT> > ( PatternMatch :: m_Value ( ArgArg ) ) ) ) { return IC . replaceInstUsesWith ( II , ArgArg ) ; } if ( ! II . getMetadata ( LLVMContext :: MD_range ) ) { Type * IntTy32 = Type :: getInt32Ty ( II . getContext ( ) ) ; Metadata * M [


<s> if ( ! EnableMaskedGatherScatters || ! ST -> hasMVEIntegerOps ( ) ) return false ; if ( isa < VectorType > ( Ty ) ) return false ; unsigned EltWidth = Ty -> getScalarSizeInBits ( ) ; return ( ( EltWidth == <NUM_LIT> && Alignment >= <NUM_LIT> ) || ( EltWidth == <NUM_LIT> && Alignment

>= <NUM_LIT> ) || EltWidth == <NUM_LIT> ) ;


<s> if ( VecWidth != <NUM_LIT> && VecTy -> getElementType ( ) -> isFloatingPointTy ( ) ) return false ; } unsigned EltWidth = DataTy -> getScalarSizeInBits ( ) ; return ( EltWidth == <NUM_LIT> && Alignment >= <NUM_LIT> ) || ( EltWidth == <NUM_LIT> && Alignment

>= <NUM_LIT> ) || ( EltWidth == <NUM_LIT> ) ;


<s> } if ( VT . isInteger ( ) && VT . getSizeInBits ( ) >= <NUM_LIT> ) { switch ( ISD ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; } } if ( ! VT . isFloatingPoint ( ) ) return false ; if ( TLI -> useSoftFloat ( ) ) { switch ( I . getOpcode ( ) ) { default : return true ; case Instruction :: Alloca : case Instruction :: Load : case Instruction :: Store : case Instruction :: Select : case Instruction :: PHI : return false ; } } if ( I . getType ( ) -> isDoubleTy ( ) && ! ST


<s> if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; return


<s> UndefElts &= APInt :: getSplat ( NumElts , ! IsTop ? APInt :: getLowBitsSet ( <NUM_LIT> , <NUM_LIT> ) : APInt :: getHighBitsSet ( <NUM_LIT> , <NUM_LIT> ) ) ; return None ; } ; switch ( II . getIntrinsicID ( ) ) { default : break ; case <STR_LIT> : SimplifyNarrowInstrTopBottom ( <NUM_LIT> ) ; break ; case <STR_LIT> : SimplifyNarrowInstrTopBottom ( <NUM_LIT> ) ; break ; case <STR_LIT> : SimplifyNarrowInstrTopBottom ( <NUM_LIT> )


<s> unsigned Alignment = IntrAlign -> getLimitedValue ( ) < MemAlign ? MemAlign : IntrAlign -> getLimitedValue ( ) ; if ( ! isPowerOf2_32 ( Alignment ) ) return nullptr ; auto * BCastInst = Builder . CreateBitCast ( II . getArgOperand ( <NUM_LIT> ) , PointerType :: get ( II . getType ( ) , <NUM_LIT> ) ) ; return Builder . CreateAlignedLoad ( II . getType ( ) , BCastInst , Align ( Alignment )


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; T -> dump ( ) ) ; return false ; } if ( isa < StoreInst > ( I ) || isa < LoadInst > ( I ) ) { Value * Ptr = getLoadStorePointerOperand ( & I ) ; Type * AccessTy = getLoadStoreType ( & I ) ; int64_t NextStride = getPtrStride ( PSE , AccessTy , Ptr , L ) ; if ( NextStride == <NUM_LIT> ) { continue ; } else if ( NextStride == - <NUM_LIT> || ( NextStride == <NUM_LIT> && MVEMaxSupportedInterleaveFactor >= <NUM_LIT> ) || ( NextStride == <NUM_LIT> && MVEMaxSupportedInterleaveFactor >= <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } else if ( EnableMaskedGatherScatters ) { const SCEV * PtrScev = PSE . getSE ( ) -> getSCEV ( Ptr ) ; if ( auto AR = dyn_cast < SCEVAddRecExpr > ( PtrScev ) ) { const SCEV * Step = AR -> getStepRecurrence ( * PSE . getSE ( ) ) ; if ( PSE . getSE ( ) -> isLoopInvariant ( Step , L ) ) continue ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } } } LLVM_DEBUG ( dbgs ( ) <<


<s> } if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( - Imm , Ty , CostKind ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == <NUM_LIT> ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; if ( ST -> isThumb ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; } if ( Opcode == Instruction :: Xor && Imm . isAllOnes ( ) ) return <NUM_LIT> ; if ( Inst && ( ( ST -> hasV6Ops ( ) && ! ST -> isThumb ( ) ) || ST -> isThumb2 ( ) ) && Ty -> getIntegerBitWidth ( ) <= <NUM_LIT> ) { if ( isSSATMinMaxPattern ( Inst , Imm ) || ( isa < ICmpInst > ( Inst ) && Inst -> hasOneUse ( ) && isSSATMinMaxPattern ( cast < Instruction > ( * Inst -> user_begin ( ) ) , Imm ) ) ) return <NUM_LIT> ; } if ( Inst && ST -> hasVFP2Base ( ) && isFPSatMinMaxPattern ( Inst , Imm ) ) return <NUM_LIT> ; if ( Inst && Opcode == Instruction :: ICmp && Idx == <NUM_LIT> && Imm


<s> int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; if ( ST -> isThumb ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; } if ( Opcode == Instruction :: Xor && Imm . isAllOnes ( ) ) return <NUM_LIT> ; if ( Inst && ( ( ST -> hasV6Ops ( ) && ! ST -> isThumb ( ) ) || ST -> isThumb2 ( ) ) && Ty -> getIntegerBitWidth ( ) <= <NUM_LIT> ) { if ( isSSATMinMaxPattern ( Inst , Imm ) || ( isa < ICmpInst > ( Inst ) && Inst -> hasOneUse ( ) && isSSATMinMaxPattern ( cast < Instruction > ( * Inst -> user_begin ( )


<s> static bool isFPSatMinMaxPattern ( Instruction * Inst , const APInt & Imm ) { if ( Imm . getBitWidth ( ) != <NUM_LIT> || Imm != APInt ::

getHighBitsSet ( <NUM_LIT> , <NUM_LIT> ) ) return false ;


<s> if ( ! EnableMaskedGatherScatters || ! ST -> hasMVEIntegerOps ( ) ) return false ; unsigned EltWidth = Ty -> getScalarSizeInBits ( ) ; return ( ( EltWidth == <NUM_LIT> && Alignment >= <NUM_LIT> ) || (

EltWidth == <NUM_LIT> && Alignment >= <NUM_LIT> ) || EltWidth == <NUM_LIT> ) ;


<s> if ( ! EnableMaskedGatherScatters || ! ST -> hasMVEIntegerOps ( ) ) return false ; unsigned EltWidth = Ty -> getScalarSizeInBits ( ) ; return ( ( EltWidth == <NUM_LIT> && Alignment >= <NUM_LIT> ) || ( EltWidth

== <NUM_LIT> && Alignment >= <NUM_LIT> ) || EltWidth == <NUM_LIT> ) ;


<s> unsigned <STR_LIT> ( Type * Ty , bool IsComplex ) const { unsigned NumVectorInstToHideOverhead = <NUM_LIT>


<s> unsigned <STR_LIT> ( unsigned Opcode , Type * Ty , OperandValueKind Op1Info , OperandValueKind Op2Info ) const { int ISDOpcode = TLI -> InstructionOpcodeToISD ( Opcode ) ; std :: pair < unsigned , MVT > LT = TLI -> getTypeLegalizationCost ( Ty ) ; const unsigned FunctionCallDivCost = <NUM_LIT> ; const unsigned ReciprocalDivCost = <NUM_LIT> ; static const CostTblEntry < MVT > CostTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , } ; int Idx = - <NUM_LIT> ; if ( ST -> hasNEON ( ) ) Idx = CostTableLookup < MVT > ( CostTbl , array_lengthof ( CostTbl ) , ISDOpcode , LT . second ) ; if ( Idx != - <NUM_LIT> ) return LT . first * CostTbl [ Idx ] . Cost ; return TargetTransformInfo :: getArithmeticInstrCost ( Opcode , Ty ,


<s> unsigned <STR_LIT> ( unsigned Opcode , Type * Ty , OperandValueKind Op1Info , OperandValueKind Op2Info ) const { int ISDOpcode = TLI -> InstructionOpcodeToISD ( Opcode ) ; std :: pair < unsigned , MVT > LT = TLI -> getTypeLegalizationCost ( Ty ) ; const unsigned FunctionCallDivCost = <NUM_LIT> ; const unsigned ReciprocalDivCost = <NUM_LIT> ; static const CostTblEntry < MVT > CostTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , } ; int Idx = - <NUM_LIT> ; if ( ST -> hasNEON ( ) ) Idx = CostTableLookup < MVT > ( CostTbl , array_lengthof ( CostTbl ) , ISDOpcode , LT . second ) ; if ( Idx != - <NUM_LIT> ) return LT .


<s> BasicBlock * BB = L -> getLoopLatch ( ) ; if ( BB -> getParent ( ) -> optForSize ( ) ) return ; unsigned Cost = <NUM_LIT> ; for ( auto & I : * BB ) { if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { ImmutableCallSite CS ( & I ) ; if ( const Function * F = CS . getCalledFunction ( ) ) { if ( ! isLoweredToCall ( F ) )


<s> if ( isa < FCmpInst > ( & I ) ) return false ; if ( isa < FPExtInst > ( & I ) || isa < FPTruncInst > ( & I ) ) return false ; if ( isa < SExtInst > ( & I ) || isa < ZExtInst > ( & I ) ) if ( ! I . getOperand ( <NUM_LIT> ) -> hasOneUse ( ) || ! isa < LoadInst > ( I . getOperand ( <NUM_LIT> ) ) ) return false ; if ( isa < TruncInst > ( & I ) ) if ( ! I . hasOneUse ( ) || ! isa < StoreInst > ( * I . user_begin ( ) ) ) return false ; return


|| DisableTailPredication ) return false ;


<s> int <ISA_LIT>TTIImpl :: getAddressComputationCost ( Type * Ty , ScalarEvolution * SE , const SCEV * Ptr ) { unsigned NumVectorInstToHideOverhead = <NUM_LIT> ; int MaxMergeDistance = <NUM_LIT> ; if ( ST -> hasNEON ( ) ) { if ( Ty -> isVectorTy ( ) && SE && ! BaseT :: isConstantStridedAccessLessThan ( SE , Ptr , MaxMergeDistance + <NUM_LIT> ) ) return NumVectorInstToHideOverhead ; return <NUM_LIT>


<s> { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , } ; if ( const auto * Entry = CostTableLookup ( CostTbl , ISDOpcode , LT . second ) ) return LT . first * Entry -> Cost ; int Cost = BaseT :: getArithmeticInstrCost ( Opcode , Ty , CostKind , Op1Info , Op2Info , Opd1PropInfo , Opd2PropInfo ) ; if ( LT . second == <STR_LIT> && Op2Info == TargetTransformInfo :: OK_UniformConstantValue ) Cost += <NUM_LIT> ; return


<s> int ISD = TLI -> InstructionOpcodeToISD ( Opcode ) ; if ( ST -> hasNEON ( ) && ValTy -> isVectorTy ( ) && ISD == <STR_LIT> ) { static const TypeConversionCostTblEntry NEONVectorSelectTbl [ ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; EVT SelCondTy = TLI -> getValueType ( DL , CondTy ) ; EVT SelValTy = TLI -> getValueType ( DL , ValTy ) ; if ( SelCondTy . isSimple ( ) && SelValTy . isSimple ( ) ) { if ( const auto * Entry = ConvertCostTableLookup ( NEONVectorSelectTbl , ISD , SelCondTy . getSimpleVT ( ) , SelValTy . getSimpleVT ( ) ) ) return Entry -> Cost ; } std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ; return LT . first


<s> assert ( DataTy -> isVectorTy ( ) && <STR_LIT> ) ; auto * VTy = cast < FixedVectorType > ( DataTy ) ; unsigned NumElems = VTy -> getNumElements ( ) ; unsigned EltSize = VTy -> getScalarSizeInBits ( ) ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , DataTy ) ; unsigned VectorCost = NumElems * LT . first ; unsigned ScalarCost = NumElems * LT . first + BaseT :: getScalarizationOverhead ( VTy , { } ) ; if ( Alignment < EltSize / <NUM_LIT> ) return ScalarCost ; unsigned ExtSize = EltSize ; if ( I != nullptr ) { if ( ( I -> getOpcode ( ) == Instruction :: Load || match ( I , m_Intrinsic < <STR_LIT> > ( ) ) ) && I -> hasOneUse ( ) ) { const User * Us = * I -> users ( ) . begin ( ) ; if ( isa < ZExtInst > ( Us ) || isa < SExtInst > ( Us ) ) { unsigned TypeSize = cast < Instruction > ( Us ) -> getType ( ) -> getScalarSizeInBits ( ) ; if ( ( ( TypeSize == <NUM_LIT> && ( EltSize == <NUM_LIT> || EltSize == <NUM_LIT> ) ) || ( TypeSize == <NUM_LIT> && EltSize == <NUM_LIT> ) ) && TypeSize * NumElems == <NUM_LIT> ) { ExtSize = TypeSize ; } } } TruncInst * T ; if ( ( I -> getOpcode ( ) == Instruction :: Store || match ( I , m_Intrinsic < <STR_LIT> > ( ) ) ) && ( T = dyn_cast < TruncInst > ( I -> getOperand ( <NUM_LIT> ) ) ) ) { unsigned TypeSize = T -> getOperand ( <NUM_LIT> ) -> getType ( ) -> getScalarSizeInBits ( ) ; if ( ( ( EltSize == <NUM_LIT> && TypeSize == <NUM_LIT> ) || ( EltSize == <NUM_LIT> && ( TypeSize == <NUM_LIT> || TypeSize == <NUM_LIT> ) ) ) && TypeSize * NumElems == <NUM_LIT> ) ExtSize = TypeSize ; } } if ( ExtSize * NumElems != <NUM_LIT> || NumElems < <NUM_LIT> ) return ScalarCost ; if ( ExtSize == <NUM_LIT> ) return VectorCost ; if ( ExtSize != <NUM_LIT> && ExtSize != <NUM_LIT> ) return ScalarCost ; if ( auto BC = dyn_cast < BitCastInst > ( Ptr ) ) Ptr = BC -> getOperand ( <NUM_LIT> ) ; if ( auto * GEP = dyn_cast < GetElementPtrInst > ( Ptr ) ) { if ( GEP -> getNumOperands ( ) != <NUM_LIT> ) return ScalarCost ; unsigned Scale = DL . getTypeAllocSize ( GEP -> getResultElementType ( ) ) ; if ( Scale != <NUM_LIT> && Scale * <NUM_LIT> != ExtSize ) return ScalarCost ; if ( auto ZExt = dyn_cast < ZExtInst > ( GEP -> getOperand ( <NUM_LIT> ) ) ) { if ( ZExt -> getOperand ( <NUM_LIT> ) -> getType ( ) -> getScalarSizeInBits ( ) <= ExtSize ) return VectorCost ; } return ScalarCost


<s> auto * SubVecTy = FixedVectorType :: get ( VecTy -> getScalarType ( ) , NumElts / Factor ) ; int BaseCost = ST -> hasMVEIntegerOps ( ) ? ST -> getMVEVectorCostFactor ( ) : <NUM_LIT> ; if ( NumElts % Factor == <NUM_LIT> && TLI -> isLegalInterleavedAccessType ( Factor , SubVecTy , DL ) ) return Factor * BaseCost * TLI -> getNumInterleavedAccesses ( SubVecTy , DL ) ; if ( ST -> hasMVEIntegerOps ( ) && Factor == <NUM_LIT> && NumElts / Factor > <NUM_LIT> && VecTy -> isIntOrIntVectorTy ( ) && DL . getTypeSizeInBits ( SubVecTy ) <= <NUM_LIT> ) return <NUM_LIT> * BaseCost ; } return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment , AddressSpace , CostKind


<s> assert ( isa < VectorType > ( VecTy ) && <STR_LIT> ) ; bool EltIs64Bits = DL . getTypeSizeInBits ( VecTy -> getScalarType ( ) ) == <NUM_LIT> ; if ( Factor <= TLI -> getMaxSupportedInterleaveFactor ( ) && ! EltIs64Bits && ! UseMaskForCond && ! UseMaskForGaps ) { unsigned NumElts = cast < FixedVectorType > ( VecTy ) -> getNumElements ( ) ; auto * SubVecTy = FixedVectorType :: get ( VecTy -> getScalarType ( ) , NumElts / Factor ) ; int BaseCost = ST -> hasMVEIntegerOps ( ) ? ST -> getMVEVectorCostFactor ( ) : <NUM_LIT> ; if ( NumElts % Factor == <NUM_LIT> && TLI -> isLegalInterleavedAccessType ( Factor , SubVecTy , DL ) ) return Factor * BaseCost * TLI -> getNumInterleavedAccesses ( SubVecTy , DL ) ; if ( ST -> hasMVEIntegerOps ( ) && Factor == <NUM_LIT> && NumElts / Factor > <NUM_LIT> && VecTy -> isIntOrIntVectorTy ( ) && DL . getTypeSizeInBits ( SubVecTy ) <= <NUM_LIT> ) return <NUM_LIT> * BaseCost ; } return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment , AddressSpace , CostKind ,


<s> int64_t SImmVal = Imm . getSExtValue ( ) ; uint64_t ZImmVal = Imm . getZExtValue ( ) ; if ( ! ST -> isThumb ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ; } if ( ST -> isThumb2 ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~


<s> int <ISA_LIT>TTIImpl :: getIntImmCost ( const APInt & Imm , Type * Ty , TTI :: TargetCostKind CostKind ) { assert ( Ty -> isIntegerTy ( ) ) ; unsigned Bits = Ty -> getPrimitiveSizeInBits ( ) ; if ( Bits == <NUM_LIT> || Imm . getActiveBits ( ) >= <NUM_LIT> ) return <NUM_LIT> ; int64_t SImmVal = Imm . getSExtValue ( ) ; uint64_t ZImmVal = Imm . getZExtValue ( ) ; if ( ! ST -> isThumb ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ; } if ( ST -> isThumb2 ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ; } if ( Bits == <NUM_LIT> || ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) ) return <NUM_LIT> ; if ( ( ~ SImmVal < <NUM_LIT> ) || <STR_LIT> ( ZImmVal ) ) return <NUM_LIT> ; return <NUM_LIT>


<s> int <ISA_LIT>TTIImpl :: getMemcpyCost ( const Instruction * I ) { const MemCpyInst * MI = dyn_cast < MemCpyInst > ( I ) ; assert ( MI && <STR_LIT> ) ; ConstantInt * C = dyn_cast < ConstantInt > ( MI -> getLength ( ) ) ; const unsigned LibCallCost = <NUM_LIT> ; if ( ! C ) return


<s> if ( CostKind != TTI :: TCK_RecipThroughput ) return <NUM_LIT> ; if ( TLI -> getValueType ( DL , Src , true ) == <STR_LIT> ) return BaseT :: getMemoryOpCost ( Opcode , Src , Alignment , AddressSpace , CostKind ) ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Src ) ; if ( ST -> hasNEON ( ) && Src -> isVectorTy ( ) && ( Alignment && * Alignment != Align ( <NUM_LIT> ) ) && cast < VectorType > ( Src ) -> getElementType ( ) -> isDoubleTy ( ) ) { return LT . first * <NUM_LIT> ; } int BaseCost = ST -> hasMVEIntegerOps ( ) && Src -> isVectorTy

( ) ? ST -> getMVEVectorCostFactor ( ) : <NUM_LIT> ;


<s> if ( Kind == TTI :: SK_Broadcast ) { static const CostTblEntry NEONDupTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( NEONDupTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost ; } if ( Kind == TTI :: SK_Reverse ) { static const CostTblEntry NEONShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( NEONShuffleTbl , <STR_LIT> , LT . second ) ) return


<s> { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( NEONSelShuffleTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost ; } } if ( ST -> hasMVEIntegerOps ( ) ) { if ( Kind == TTI :: SK_Broadcast ) { static const CostTblEntry MVEDupTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( MVEDupTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost * ST -> getMVEVectorCostFactor ( ) ; } } int BaseCost = ST -> hasMVEIntegerOps ( ) && Tp -> isVectorTy ( ) ? ST -> getMVEVectorCostFactor ( ) : <NUM_LIT> ; return BaseCost * BaseT :: getShuffleCost ( Kind , Tp , Index


<s> if ( ST -> hasMVEIntegerOps ( ) && ( Opcode == Instruction :: InsertElement || Opcode == Instruction :: ExtractElement ) ) { return std :: max ( BaseT :: getVectorInstrCost ( Opcode , ValTy , Index ) , ST -> getMVEVectorCostFactor ( ) ) * cast < FixedVectorType > ( ValTy ) -> getNumElements ( ) / <NUM_LIT> ; } return BaseT :: getVectorInstrCost ( Opcode


<s> if ( isa < VectorType > ( Ty ) ) return false ; unsigned EltWidth = Ty -> getScalarSizeInBits ( ) ; return ( ( EltWidth == <NUM_LIT> && ( ! Alignment || * Alignment >= <NUM_LIT> ) ) || ( EltWidth == <NUM_LIT> &&

( ! Alignment || * Alignment >= <NUM_LIT> ) ) || EltWidth == <NUM_LIT> ) ;


<s> bool <ISA_LIT>TTIImpl :: isLegalMaskedGather ( Type * Ty , MaybeAlign Alignment ) { if ( ! EnableMaskedGatherScatters || ! ST -> hasMVEIntegerOps ( ) ) return false ; if ( isa < VectorType > ( Ty ) ) return false ; unsigned EltWidth = Ty -> getScalarSizeInBits ( ) ; return ( ( EltWidth == <NUM_LIT> && ( ! Alignment || * Alignment >= <NUM_LIT> ) ) || ( EltWidth == <NUM_LIT> && (

! Alignment || * Alignment >= <NUM_LIT> ) ) || EltWidth == <NUM_LIT> ) ;


<s> if ( VecTy -> getNumElements ( ) == <NUM_LIT> ) return false ; unsigned VecWidth = DataTy -> getPrimitiveSizeInBits ( ) ; if ( VecWidth != <NUM_LIT> && VecTy -> getElementType ( ) -> isFloatingPointTy ( ) ) return false ; } unsigned EltWidth = DataTy -> getScalarSizeInBits (


<s> bool isLegalMaskedScatter ( Type * Ty , MaybeAlign


<s> bool isLegalMaskedScatter ( Type * Ty ,


<s> bool isLegalMaskedStore ( Type * DataTy , MaybeAlign Alignment


<s> bool isLegalMaskedStore ( Type * DataTy , MaybeAlign Alignment ) { return isLegalMaskedLoad ( DataTy ,


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( F -> getReturnType ( ) -> isDoubleTy ( ) && ! ST -> hasFP64 ( ) ) return true ; if ( F -> getReturnType ( ) -> isHalfTy ( ) && ! ST -> hasFullFP16 ( ) ) return true ; return ! ST -> hasFP<ISA_LIT>v8Base ( ) && ! ST -> hasVFP2Base ( ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return ! ST -> hasMVEIntegerOps ( ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT>


<s> if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; if ( L -> getNumBlocks ( ) > <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } assert ( L -> empty ( ) && <STR_LIT> ) ; HardwareLoopInfo HWLoopInfo ( L ) ; if ( ! HWLoopInfo . canAnalyze ( * LI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false


<s> if ( ST -> isROPI ( ) || ST -> isRWPI (


<s> return ST -> isMClass ( ) && ST -> isThumb2 ( ) && L -> getNumBlocks


<s> bool <ISA_LIT>TTIImpl :: shouldFavorBackedgeIndex ( const Loop * L ) const { if ( L -> getHeader ( ) -> getParent ( ) -> hasOptSize ( )


<s> bool <ISA_LIT>TTIImpl :: useReductionIntrinsic ( unsigned Opcode , Type * Ty ,


<s> if ( ! ST -> isThumb2 ( ) ) return ; SmallVector < BasicBlock * , <NUM_LIT> > ExitingBlocks ; L -> getExitingBlocks ( ExitingBlocks ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << L -> getNumBlocks ( ) << <STR_LIT> << <STR_LIT> << ExitingBlocks . size ( ) << <STR_LIT> ) ; if ( ExitingBlocks . size ( ) > <NUM_LIT> ) return ; if ( ST -> hasBranchPredictor ( ) && L -> getNumBlocks ( ) > <NUM_LIT> ) return ; unsigned Cost = <NUM_LIT> ; for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { ImmutableCallSite CS ( & I ) ; if ( const Function * F = CS . getCalledFunction ( ) ) { if ( ! isLoweredToCall ( F ) ) continue ; } return ; } SmallVector < const Value * , <NUM_LIT> > Operands ( I . value_op_begin ( ) , I . value_op_end ( ) ) ; Cost += getUserCost ( & I , Operands ) ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ) ; UP . Partial =


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << L -> getNumBlocks ( ) << <STR_LIT> << <STR_LIT> << ExitingBlocks . size ( ) << <STR_LIT> ) ; if ( ExitingBlocks . size ( ) > <NUM_LIT> ) return ; if ( ST -> hasBranchPredictor ( ) && L -> getNumBlocks ( ) > <NUM_LIT> ) return ; unsigned Cost = <NUM_LIT> ; for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { ImmutableCallSite CS ( & I ) ; if ( const Function * F = CS . getCalledFunction ( ) ) { if ( ! isLoweredToCall ( F ) ) continue ; } return ; } SmallVector < const Value * , <NUM_LIT> > Operands ( I . value_op_begin ( ) , I . value_op_end ( ) ) ; Cost += getUserCost ( & I , Operands ) ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ) ; UP . Partial = true ; UP . Runtime = true ; UP . UnrollRemainder =


<s> if ( ID == & TargetTransformInfo :: ID ) return ( TargetTransformInfo *


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { TargetTransformInfo :: getAnalysisUsage ( AU


<s> std :: pair < unsigned , MVT > LT = TLI -> getTypeLegalizationCost ( Ty ) ; const unsigned FunctionCallDivCost = <NUM_LIT> ; const unsigned ReciprocalDivCost = <NUM_LIT> ; static const CostTblEntry < <STR_LIT> > CostTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , } ; int Idx = - <NUM_LIT> ; if ( ST -> hasNEON ( ) ) Idx = CostTableLookup ( CostTbl , ISDOpcode ,


<s> unsigned <STR_LIT> ( unsigned Opcode , Type * ValTy , Type * CondTy ) const { int ISD = TLI -> InstructionOpcodeToISD ( Opcode ) ; if ( ST -> hasNEON ( ) && ValTy -> isVectorTy ( ) && ISD == <STR_LIT> ) { static const TypeConversionCostTblEntry < <STR_LIT> > NEONVectorSelectTbl [ ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; EVT SelCondTy = TLI -> getValueType ( CondTy ) ; EVT SelValTy = TLI -> getValueType ( ValTy ) ; if ( SelCondTy . isSimple ( ) && SelValTy . isSimple ( ) ) { int Idx = ConvertCostTableLookup ( NEONVectorSelectTbl , ISD , SelCondTy . getSimpleVT ( ) , SelValTy . getSimpleVT


<s> if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ; } if ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) return <NUM_LIT> ; if ( ( ~ ZImmVal < <NUM_LIT> )

|| <STR_LIT> ( ZImmVal ) ) return <NUM_LIT> ;


<s> if ( ST -> isThumb2 ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ; } if ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) return


<s> unsigned getMaximumUnrollFactor ( ) const override { if ( ST -> isCortexA15 ( ) || ST -> isSwift ( ) ) return <NUM_LIT> ; return


<s> if ( Kind != SK_Reverse ) return TargetTransformInfo :: getShuffleCost ( Kind , Tp , Index , SubTp ) ; static const CostTblEntry < <STR_LIT> > NEONShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } ,


<s> unsigned <STR_LIT> ( ShuffleKind Kind , Type * Tp , int Index , Type * SubTp ) const { if ( Kind != SK_Reverse ) return TargetTransformInfo :: getShuffleCost ( Kind , Tp , Index , SubTp ) ; static const CostTblEntry < <STR_LIT> > NEONShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < unsigned , MVT > LT = TLI -> getTypeLegalizationCost ( Tp ) ; int Idx = CostTableLookup ( NEONShuffleTbl , <STR_LIT> , LT . second ) ; if ( Idx == - <NUM_LIT> ) return TargetTransformInfo :: getShuffleCost ( Kind , Tp ,


<s> unsigned <ISA_LIT>TTIImpl :: getAddressComputationCost ( Type * Ty , bool IsComplex ) { unsigned NumVectorInstToHideOverhead


<s> unsigned <ISA_LIT>TTIImpl :: getAddressComputationCost ( Type * Ty , bool IsComplex ) { unsigned NumVectorInstToHideOverhead =


<s> if ( ST -> hasNEON ( ) && ValTy -> isVectorTy ( ) && ISD == <STR_LIT> ) { static const TypeConversionCostTblEntry < <STR_LIT> > NEONVectorSelectTbl [ ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; EVT SelCondTy = TLI -> getValueType ( CondTy ) ; EVT SelValTy = TLI -> getValueType ( ValTy ) ; if ( SelCondTy . isSimple ( ) && SelValTy . isSimple ( ) ) { int Idx = ConvertCostTableLookup ( NEONVectorSelectTbl , ISD , SelCondTy . getSimpleVT ( ) , SelValTy . getSimpleVT


<s> if ( ST -> isThumb2 ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ; } if ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) return <NUM_LIT> ; if ( ( ~ ZImmVal < <NUM_LIT> ) || <STR_LIT> ( ZImmVal )


<s> unsigned <ISA_LIT>TTIImpl :: getIntImmCost ( const APInt & Imm , Type * Ty ) { assert ( Ty -> isIntegerTy ( ) ) ; unsigned Bits = Ty -> getPrimitiveSizeInBits ( ) ; if ( Bits == <NUM_LIT> || Bits > <NUM_LIT> ) return <NUM_LIT> ; int32_t SImmVal = Imm . getSExtValue ( ) ; uint32_t ZImmVal = Imm . getZExtValue ( ) ; if ( ! ST -> isThumb ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> :


<s> if ( Src -> isVectorTy ( ) && Alignment != <NUM_LIT> && Src -> getVectorElementType ( ) -> isDoubleTy ( ) ) { return LT . first


<s> if ( Src -> isVectorTy ( ) && Alignment != <NUM_LIT> && Src -> getVectorElementType ( ) -> isDoubleTy ( ) ) { return LT .


<s> unsigned <ISA_LIT>TTIImpl :: getShuffleCost ( TTI :: ShuffleKind Kind , Type * Tp , int Index , Type * SubTp ) { if ( Kind != TTI :: SK_Reverse && Kind != TTI :: SK_Alternate ) return BaseT :: getShuffleCost ( Kind , Tp , Index , SubTp ) ; if ( Kind == TTI :: SK_Reverse ) { static const CostTblEntry < <STR_LIT> > NEONShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < unsigned , MVT > LT = TLI -> getTypeLegalizationCost ( Tp ) ; int Idx = CostTableLookup ( NEONShuffleTbl , <STR_LIT> , LT . second ) ; if ( Idx == - <NUM_LIT> ) return BaseT :: getShuffleCost ( Kind , Tp , Index , SubTp ) ; return LT . first * NEONShuffleTbl [ Idx ] . Cost ; } if ( Kind == TTI :: SK_Alternate ) { static const CostTblEntry < <STR_LIT> > NEONAltShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < unsigned , MVT > LT = TLI -> getTypeLegalizationCost ( Tp ) ; int Idx = CostTableLookup ( NEONAltShuffleTbl , <STR_LIT> , LT . second ) ; if ( Idx == - <NUM_LIT> ) return BaseT :: getShuffleCost ( Kind , Tp , Index


<s> if ( ( Opcode == Instruction :: InsertElement || Opcode == Instruction :: ExtractElement ) && ValTy -> getVectorElementType ( ) -> isIntegerTy ( ) ) return <NUM_LIT> ; return BaseT :: getVectorInstrCost ( Opcode , ValTy , Index


<s> bool <ISA_LIT>TTIImpl :: emitGetActiveLaneMask ( Loop * L , LoopInfo * LI , ScalarEvolution & SE


<s> const unsigned ReciprocalDivCost = <NUM_LIT> ; static const CostTblEntry CostTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , } ; if ( const auto * Entry = CostTableLookup ( CostTbl , ISDOpcode , LT . second ) ) return LT . first * Entry -> Cost ; int Cost = BaseT :: getArithmeticInstrCost ( Opcode , Ty , CostKind , Op1Info , Op2Info , Opd1PropInfo , Opd2PropInfo ) ; if ( LT . second == <STR_LIT> && Op2Info == TargetTransformInfo :: OK_UniformConstantValue ) Cost += <NUM_LIT> ; return Cost ; } auto LooksLikeAFreeShift = [ & ] ( ) { if ( ST -> isThumb1Only ( ) || Ty -> isVectorTy ( ) ) return false ; if ( ! CxtI || ! CxtI -> hasOneUse ( ) || ! CxtI -> isShift ( ) ) return false ; if ( Op2Info != TargetTransformInfo :: OK_UniformConstantValue ) return false ; switch ( cast < Instruction > ( CxtI -> user_back ( ) ) -> getOpcode ( ) ) { case Instruction :: Add : case Instruction :: Sub : case Instruction :: And : case Instruction :: Xor : case Instruction ::


<s> if ( ST -> hasNEON ( ) && Src -> isVectorTy ( ) && ( Alignment && * Alignment != Align ( <NUM_LIT> ) ) && cast < VectorType > ( Src ) -> getElementType ( ) -> isDoubleTy ( ) ) { return LT . first


<s> assert ( isa < VectorType > ( Ty ) && <STR_LIT> ) ; unsigned ScalarBits = Ty -> getScalarSizeInBits ( ) ; if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; switch ( Opcode ) { case Instruction :: FAdd : case Instruction :: FMul : case Instruction :: And : case Instruction ::


<s> const unsigned FunctionCallDivCost = <NUM_LIT> ; const unsigned ReciprocalDivCost = <NUM_LIT> ; static const CostTblEntry CostTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , } ; if ( ST -> hasNEON ( )


<s> if ( ! isLoweredToCall ( F ) ) continue ; } return ; } if ( I . getType ( ) -> isVectorTy ( ) ) return ; SmallVector < const Value * , <NUM_LIT> > Operands ( I . value_op_begin ( ) , I . value_op_end ( ) ) ; Cost += getUserCost ( & I , Operands ) ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ) ; UP . Partial = true ; UP . Runtime = true ; UP . UpperBound = true ; UP . UnrollRemainder = true ; UP . DefaultUnrollRuntimeCount = <NUM_LIT> ; UP . UnrollAndJam = true ; UP . UnrollAndJamInnerLoopThreshold =


<s> bool <ISA_LIT>TTIImpl :: isLegalMaskedLoad ( Type * DataTy ) { if ( ! EnableMaskedLoadStores || ! ST -> hasMVEIntegerOps ( ) ) return


<s> if ( ! EnableMaskedLoadStores || ! ST -> hasMVEIntegerOps ( ) ) return false ; if ( DataTy -> isVectorTy ( ) ) { unsigned VecWidth = DataTy -> getPrimitiveSizeInBits ( ) ; if ( VecWidth != <NUM_LIT> ) return


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; SmallVector < Instruction * , <NUM_LIT> > LiveOuts ; LiveOuts = llvm :: findDefsUsedOutsideOfLoop ( L ) ; bool ReductionsDisabled = EnableTailPredication == TailPredication :: EnabledNoReductions || EnableTailPredication == TailPredication :: ForceEnabledNoReductions ; for ( auto * I : LiveOuts ) { if ( ! I -> getType ( ) -> isIntegerTy ( ) && ! I -> getType ( ) -> isFloatTy ( ) && ! I -> getType ( ) -> isHalfTy ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } if ( ReductionsDisabled ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } } PredicatedScalarEvolution PSE = LAI -> getPSE ( ) ; SmallVector < Instruction * , <NUM_LIT> > LoadStores ; int ICmpCount = <NUM_LIT> ; for ( BasicBlock * BB : L -> blocks ( ) ) { for ( Instruction & I : BB -> instructionsWithoutDebug ( ) ) { if ( isa < PHINode > ( & I ) ) continue ; if ( ! canTailPredicateInstruction ( I , ICmpCount ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; I . dump ( ) ) ; return false ; } Type * T = I . getType ( ) ; if ( T -> isPointerTy ( ) ) T = T -> getPointerElementType ( ) ; if ( T -> getScalarSizeInBits ( ) > <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; T -> dump ( ) ) ; return false ; } if ( isa < StoreInst > ( I ) || isa < LoadInst > ( I ) ) { Value * Ptr = isa < LoadInst > ( I ) ? I . getOperand ( <NUM_LIT> ) : I . getOperand ( <NUM_LIT> ) ; int64_t NextStride = getPtrStride ( PSE , Ptr , L ) ; if ( NextStride == <NUM_LIT> ) { continue ; } else if ( NextStride == - <NUM_LIT> || ( NextStride == <NUM_LIT> && MVEMaxSupportedInterleaveFactor >= <NUM_LIT> ) || ( NextStride == <NUM_LIT> && MVEMaxSupportedInterleaveFactor >= <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } else if ( EnableMaskedGatherScatters


<s> if ( T -> isPointerTy ( ) ) T = T -> getPointerElementType ( ) ; if ( T -> getScalarSizeInBits ( ) > <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; T -> dump ( ) ) ; return false ; } if ( isa < StoreInst > ( I ) || isa < LoadInst > ( I ) ) { Value * Ptr = isa < LoadInst > ( I ) ? I . getOperand ( <NUM_LIT> ) : I . getOperand ( <NUM_LIT> ) ; int64_t NextStride = getPtrStride ( PSE , Ptr , L ) ; if ( NextStride == <NUM_LIT> ) { continue ; } else if ( NextStride == - <NUM_LIT> || ( NextStride == <NUM_LIT> && MVEMaxSupportedInterleaveFactor >= <NUM_LIT> ) || ( NextStride == <NUM_LIT> && MVEMaxSupportedInterleaveFactor >= <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } else if ( EnableMaskedGatherScatters ) { const SCEV * PtrScev = replaceSymbolicStrideSCEV ( PSE , llvm :: ValueToValueMap ( ) , Ptr ) ; if ( auto AR = dyn_cast < SCEVAddRecExpr > ( PtrScev ) ) { const SCEV * Step = AR -> getStepRecurrence ( * PSE . getSE ( ) ) ; if ( PSE . getSE ( ) -> isLoopInvariant ( Step , L ) ) continue ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT>


<s> std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ; static const CostTblEntry CostTblAdd [ ] { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , } ; if ( const auto * Entry = CostTableLookup ( CostTblAdd , ISD , LT . second ) ) return Entry -> Cost * ST -> getMVEVectorCostFactor ( ) * LT . first ; return BaseT :: getArithmeticReductionCost ( Opcode ,


<s> if ( CostKind == TTI :: TCK_RecipThroughput && ( ST -> hasNEON ( ) || ST -> hasMVEIntegerOps ( ) ) ) { return


<s> return Cost ; } if ( ST -> hasNEON ( ) && ValTy -> isVectorTy ( ) && ISD == <STR_LIT> && CondTy ) { static const TypeConversionCostTblEntry NEONVectorSelectTbl [ ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; EVT SelCondTy = TLI -> getValueType ( DL , CondTy ) ; EVT SelValTy = TLI -> getValueType ( DL , ValTy ) ; if ( SelCondTy . isSimple ( ) && SelValTy . isSimple ( ) ) { if ( const auto * Entry = ConvertCostTableLookup ( NEONVectorSelectTbl , ISD , SelCondTy . getSimpleVT ( ) , SelValTy . getSimpleVT ( ) ) ) return Entry -> Cost ; } std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ; return LT . first ; } int BaseCost =


<s> } if ( ST -> hasNEON ( ) && ValTy -> isVectorTy ( ) && ISD == <STR_LIT> && CondTy ) { static const TypeConversionCostTblEntry NEONVectorSelectTbl [ ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; EVT SelCondTy = TLI -> getValueType ( DL , CondTy ) ; EVT SelValTy = TLI -> getValueType ( DL , ValTy ) ; if ( SelCondTy . isSimple ( ) && SelValTy . isSimple ( ) ) { if ( const auto * Entry = ConvertCostTableLookup ( NEONVectorSelectTbl , ISD , SelCondTy . getSimpleVT ( ) , SelValTy . getSimpleVT ( ) ) ) return Entry -> Cost ; } std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ; return LT .


<s> if ( I != nullptr ) { if ( ( I -> getOpcode ( ) == Instruction :: Load || match ( I , m_Intrinsic < <STR_LIT> > ( ) ) ) && I -> hasOneUse ( ) ) { const User * Us = * I -> users ( ) . begin ( ) ; if ( isa < ZExtInst > ( Us ) || isa < SExtInst > ( Us ) ) { unsigned TypeSize = cast < Instruction > ( Us ) -> getType ( ) -> getScalarSizeInBits ( ) ; if ( ( ( TypeSize == <NUM_LIT> && ( EltSize == <NUM_LIT> || EltSize == <NUM_LIT> ) ) || ( TypeSize == <NUM_LIT> && EltSize == <NUM_LIT> ) ) && TypeSize * NumElems == <NUM_LIT> ) { ExtSize = TypeSize ; } } } TruncInst * T ; if ( ( I -> getOpcode ( ) == Instruction :: Store || match ( I , m_Intrinsic < <STR_LIT> > ( ) ) ) && ( T = dyn_cast < TruncInst > ( I -> getOperand ( <NUM_LIT> ) ) ) ) { unsigned TypeSize = T -> getOperand ( <NUM_LIT> ) -> getType ( ) -> getScalarSizeInBits ( ) ; if ( ( ( EltSize == <NUM_LIT> && TypeSize == <NUM_LIT> ) || ( EltSize == <NUM_LIT> && ( TypeSize == <NUM_LIT> || TypeSize == <NUM_LIT> ) ) ) && TypeSize * NumElems == <NUM_LIT> ) ExtSize = TypeSize ; } } if ( ExtSize * NumElems != <NUM_LIT> || NumElems < <NUM_LIT> ) return ScalarCost ; if ( ExtSize == <NUM_LIT> ) return VectorCost ; if ( ExtSize != <NUM_LIT> && ExtSize != <NUM_LIT> ) return ScalarCost ; if ( const auto * BC = dyn_cast < BitCastInst > ( Ptr ) ) Ptr = BC -> getOperand ( <NUM_LIT> ) ; if ( const auto * GEP = dyn_cast < GetElementPtrInst > ( Ptr ) ) { if ( GEP -> getNumOperands ( ) != <NUM_LIT> ) return ScalarCost ; unsigned Scale = DL . getTypeAllocSize ( GEP -> getResultElementType ( ) ) ; if ( Scale != <NUM_LIT> && Scale * <NUM_LIT> != ExtSize ) return ScalarCost ; if ( const auto * ZExt = dyn_cast < ZExtInst > ( GEP -> getOperand


<s> assert ( isa < VectorType > ( VecTy ) && <STR_LIT> ) ; bool EltIs64Bits = DL . getTypeSizeInBits ( VecTy -> getScalarType ( ) ) == <NUM_LIT> ; if ( Factor <= TLI -> getMaxSupportedInterleaveFactor ( ) && ! EltIs64Bits && ! UseMaskForCond && ! UseMaskForGaps ) { unsigned NumElts = cast < FixedVectorType > ( VecTy ) -> getNumElements ( ) ; auto * SubVecTy = FixedVectorType :: get ( VecTy -> getScalarType ( ) , NumElts / Factor ) ; int BaseCost = ST -> hasMVEIntegerOps ( ) ? ST -> getMVEVectorCostFactor ( ) : <NUM_LIT> ; if ( NumElts % Factor == <NUM_LIT> && TLI -> isLegalInterleavedAccessType ( Factor , SubVecTy , DL ) ) return Factor * BaseCost * TLI -> getNumInterleavedAccesses ( SubVecTy , DL ) ; if ( ST -> hasMVEIntegerOps ( ) && Factor == <NUM_LIT> && NumElts / Factor > <NUM_LIT> && VecTy -> isIntOrIntVectorTy ( ) && DL . getTypeSizeInBits ( SubVecTy ) . getFixedSize ( ) <= <NUM_LIT> ) return <NUM_LIT> * BaseCost ; } return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment


<s> int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; if ( ST -> isThumb ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; } if ( Opcode == Instruction :: Xor && Imm . isAllOnesValue ( ) ) return <NUM_LIT> ; if ( Inst && ( ( ST -> hasV6Ops ( ) && ! ST -> isThumb ( ) ) || ST -> isThumb2 ( ) ) && Ty ->


<s> } if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( - Imm , Ty , CostKind ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == <NUM_LIT> ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; if ( ST -> isThumb ( ) && NegImm < <NUM_LIT> <<


<s> unsigned <ISA_LIT>TTIImpl :: getMaskedMemoryOpCost ( unsigned Opcode , Type * Src , Align Alignment , unsigned AddressSpace , TTI :: TargetCostKind CostKind ) { if ( ST -> hasMVEIntegerOps ( ) ) { if ( Opcode == Instruction :: Load && isLegalMaskedLoad ( Src , Alignment ) ) return ST -> getMVEVectorCostFactor ( ) ; if ( Opcode == Instruction :: Store && isLegalMaskedStore ( Src , Alignment ) ) return ST -> getMVEVectorCostFactor ( ) ; } if ( ! isa < FixedVectorType > ( Src ) ) return BaseT :: getMaskedMemoryOpCost ( Opcode , Src , Alignment , AddressSpace , CostKind ) ; return cast < FixedVectorType > ( Src ) -> getNumElements (


<s> int NumOps = getNumMemOps ( cast < IntrinsicInst > ( I )


<s> return LT . first * <NUM_LIT> ; } if ( ST -> hasMVEFloatOps ( ) && isa < FixedVectorType > ( Src ) && I && ( ( Opcode == Instruction :: Load && I -> hasOneUse ( ) && isa < FPExtInst > ( * I -> user_begin ( ) ) ) || ( Opcode == Instruction :: Store && isa < FPTruncInst > ( I -> getOperand ( <NUM_LIT> ) ) ) ) ) { FixedVectorType * SrcVTy = cast < FixedVectorType > ( Src ) ; Type * DstTy = Opcode == Instruction :: Load ? ( * I -> user_begin ( ) ) -> getType ( ) : cast < Instruction > ( I -> getOperand ( <NUM_LIT> )

) -> getOperand ( <NUM_LIT> ) -> getType ( ) ;


<s> UP . OptSizeThreshold = <NUM_LIT> ; UP . PartialOptSizeThreshold = <NUM_LIT> ; if ( L -> getHeader ( ) -> getParent ( ) -> hasOptSize ( ) ) return ; if ( ! ST -> isThumb2 ( ) ) return ; SmallVector < BasicBlock * , <NUM_LIT> > ExitingBlocks ; L -> getExitingBlocks ( ExitingBlocks ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << L -> getNumBlocks ( ) << <STR_LIT> << <STR_LIT> << ExitingBlocks . size ( ) << <STR_LIT> ) ; if ( ExitingBlocks . size ( ) > <NUM_LIT> ) return ; if ( ST -> hasBranchPredictor ( ) && L -> getNumBlocks ( ) > <NUM_LIT> ) return ; if ( getBooleanLoopAttribute ( L , <STR_LIT> ) ) return ; unsigned Cost = <NUM_LIT> ; for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( I . getType ( ) -> isVectorTy ( ) ) return ; if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { if ( const Function * F = cast < CallBase > ( I ) . getCalledFunction ( ) ) { if ( ! isLoweredToCall ( F ) ) continue ; } return ; } SmallVector < const Value * , <NUM_LIT> > Operands ( I . operand_values ( ) ) ; Cost += getUserCost ( & I , Operands , TargetTransformInfo :: TCK_SizeAndLatency ) ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ) ; UP . Partial = true ; UP . Runtime = true ; UP . UpperBound = true ; UP . UnrollRemainder = true ; UP . DefaultUnrollRuntimeCount = <NUM_LIT> ; UP . UnrollAndJam


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { Align MemAlign = getKnownAlignment ( II . getArgOperand ( <NUM_LIT> ) , IC . getDataLayout ( ) , & II , & IC . getAssumptionCache ( ) , & IC . getDominatorTree ( ) ) ; unsigned AlignArg = II . getNumArgOperands ( ) - <NUM_LIT> ; Value * AlignArgOp = II . getArgOperand ( AlignArg ) ; MaybeAlign Align = cast < ConstantInt > ( AlignArgOp ) -> getMaybeAlignValue ( ) ; if ( Align && * Align < MemAlign ) { return IC . replaceOperand ( II , AlignArg , ConstantInt :: get ( Type :: getInt32Ty ( II . getContext ( ) ) , MemAlign . value ( ) , false ) ) ; } break ; } case <STR_LIT> : { Value * Arg = II . getArgOperand ( <NUM_LIT> ) ; Value * ArgArg ; if ( match ( Arg , PatternMatch :: m_Intrinsic < <STR_LIT> > ( PatternMatch :: m_Value ( ArgArg ) ) ) && II . getType ( ) == ArgArg -> getType ( ) ) { return IC . replaceInstUsesWith ( II , ArgArg ) ; } Constant * XorMask ; if ( match ( Arg , m_Xor ( PatternMatch :: m_Intrinsic < <STR_LIT> > ( PatternMatch :: m_Value ( ArgArg ) ) , PatternMatch :: m_Constant ( XorMask ) ) ) && II . getType ( ) == ArgArg -> getType ( ) ) { if ( auto * CI = dyn_cast < ConstantInt > ( XorMask ) ) { if ( CI -> getValue ( ) . trunc ( <NUM_LIT> ) . isAllOnesValue ( ) ) { auto TrueVector = IC . Builder . CreateVectorSplat ( cast < FixedVectorType > ( II . getType ( ) ) -> getNumElements ( ) , IC . Builder . getTrue ( ) ) ; return BinaryOperator :: Create ( Instruction :: Xor , ArgArg , TrueVector ) ; } } } KnownBits ScalarKnown ( <NUM_LIT> ) ; if ( IC . SimplifyDemandedBits ( & II , <NUM_LIT> , APInt :: getLowBitsSet ( <NUM_LIT> , <NUM_LIT> ) , ScalarKnown , <NUM_LIT> ) ) { return & II ; } break ; } case <STR_LIT> : { Value * Arg = II . getArgOperand ( <NUM_LIT> ) ; Value * ArgArg ; if ( match ( Arg , PatternMatch :: m_Intrinsic < <STR_LIT> > ( PatternMatch :: m_Value ( ArgArg ) ) ) ) { return IC . replaceInstUsesWith ( II , ArgArg ) ; } if ( ! II . getMetadata ( LLVMContext


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto IsHardwareLoopIntrinsic = [ ] ( Instruction & I ) { if ( auto * Call = dyn_cast < IntrinsicInst > ( & I ) ) { switch ( Call -> getIntrinsicID ( ) ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; } } return false ; } ; bool IsTailPredLoop = false ; auto ScanLoop = [ & ] ( Loop * L ) { for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( maybeLoweredToCall ( I ) || IsHardwareLoopIntrinsic ( I ) || isa < InlineAsm > ( I ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << I << <STR_LIT> ) ; return false ; } if ( auto * II = dyn_cast < IntrinsicInst > ( & I ) ) IsTailPredLoop |= II -> getIntrinsicID ( ) == <STR_LIT> || II -> getIntrinsicID ( ) == <STR_LIT> || II -> getIntrinsicID ( ) == <STR_LIT> || II -> getIntrinsicID ( ) == <STR_LIT> || II -> getIntrinsicID ( ) == <STR_LIT> ; } } return true ; } ; for ( auto Inner : * L ) if ( ! ScanLoop ( Inner ) ) return false ; if ( ! ScanLoop ( L ) ) return false ; LLVMContext & C = L -> getHeader ( ) -> getContext ( ) ; HWLoopInfo . CounterInReg = true ; HWLoopInfo . IsNestingLegal = false ; HWLoopInfo . PerformEntryTest = AllowWLSLoops && ! IsTailPredLoop ; HWLoopInfo . CountType = Type :: getInt32Ty ( C ) ; HWLoopInfo . LoopDecrement = ConstantInt :: get (


<s> if ( MinSPF == SPF_SMIN && PatternMatch :: match ( MinRHS , PatternMatch :: m_ConstantInt ( MinC ) ) && MinC -> getValue ( ) == ( ( - Imm ) - <NUM_LIT> ) ) return true ; } return false ; } ; if ( isSSatMin ( Inst -> getOperand ( <NUM_LIT> ) ) || ( Inst -> hasNUses ( <NUM_LIT> ) && ( isSSatMin ( * Inst -> user_begin ( ) ) || isSSatMin ( * ( ++ Inst -> user_begin ( ) ) ) ) ) ) return true ; } return false


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( F -> getReturnType ( ) -> isDoubleTy ( ) && ! ST -> hasFP64 ( ) ) return true ; if ( F -> getReturnType ( ) -> isHalfTy ( ) && ! ST -> hasFullFP16 ( ) ) return true ; return ! ST -> hasFP<ISA_LIT>v8Base ( ) && ! ST -> hasVFP2Base ( ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return ! ST -> hasMVEIntegerOps ( ) ; case <STR_LIT> : case


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( F -> getReturnType ( ) -> isDoubleTy ( ) && ! ST -> hasFP64 ( ) ) return true ; if ( F -> getReturnType ( ) -> isHalfTy ( ) && ! ST -> hasFullFP16 ( ) ) return true ; return ! ST -> hasFP<ISA_LIT>v8Base ( ) && ! ST -> hasVFP2Base ( ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return ! ST -> hasMVEIntegerOps (


<s> for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( I . getType ( ) -> isVectorTy ( ) ) return ; if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { if ( const Function * F = cast < CallBase > ( I ) . getCalledFunction ( ) ) { if ( ! isLoweredToCall ( F ) ) continue ; } return ; } SmallVector < const Value * , <NUM_LIT> > Operands ( I . value_op_begin ( ) , I . value_op_end ( ) ) ; Cost += getUserCost ( & I , Operands , TargetTransformInfo :: TCK_SizeAndLatency ) ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ) ; UP . Partial = true ; UP . Runtime = true ; UP . UpperBound = true ; UP . UnrollRemainder = true ; UP . DefaultUnrollRuntimeCount = <NUM_LIT> ; UP . UnrollAndJam =


<s> bool isLegalMaskedGather ( Type * Ty , MaybeAlign Alignment ) { return false


<s> break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; } } return false ; } ; auto ScanLoop = [ & ] ( Loop * L ) { for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( maybeLoweredToCall ( I ) || IsHardwareLoopIntrinsic ( I ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << I << <STR_LIT> ) ; return false ; } } } return true ; } ; for ( auto Inner : * L ) if ( ! ScanLoop ( Inner ) ) return false ; if ( ! ScanLoop ( L ) ) return false ; LLVMContext & C = L -> getHeader ( ) -> getContext ( ) ; HWLoopInfo . CounterInReg = true ; HWLoopInfo . IsNestingLegal = false ; HWLoopInfo . PerformEntryTest = true ; HWLoopInfo . CountType = Type :: getInt32Ty


<s> EVT ResVT = TLI -> getValueType ( DL , ResTy ) ; if ( ST -> hasMVEIntegerOps ( ) && ValVT . isSimple ( ) && ResVT . isSimple ( ) ) { std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ; if ( ( LT . second == <STR_LIT> && ResVT . getSizeInBits ( ) <= <NUM_LIT> ) || ( LT . second == <STR_LIT> && ResVT . getSizeInBits ( ) <= ( IsMLA ? <NUM_LIT> : <NUM_LIT> ) ) || ( LT . second == <STR_LIT> && ResVT . getSizeInBits ( ) <= <NUM_LIT> ) ) return ST -> getMVEVectorCostFactor


<s> if ( Imm == <NUM_LIT> || Imm == <NUM_LIT> ) return <NUM_LIT> ; return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( ~ Imm , Ty , CostKind ) ) ; } if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( - Imm , Ty , CostKind ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == <NUM_LIT> ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; if ( ST -> isThumb ( ) &&


<s> if ( ST -> isThumb2 ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; if ( ST -> isThumb ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; } if ( Opcode == Instruction :: Xor && Imm . isAllOnesValue ( ) ) return <NUM_LIT> ; if ( Inst && ( ( ST -> hasV6Ops ( ) && ! ST -> isThumb ( ) ) || ST -> isThumb2 ( ) ) && Ty -> getIntegerBitWidth ( ) <= <NUM_LIT> ) { if ( isSSATMinMaxPattern ( Inst , Imm ) || ( isa < ICmpInst > ( Inst ) && Inst -> hasOneUse ( ) && isSSATMinMaxPattern ( cast < Instruction > ( * Inst -> user_begin ( ) ) , Imm ) ) ) return <NUM_LIT> ; } return getIntImmCost ( Imm , Ty , CostKind


<s> unsigned SrcAddrSpace = ~ <NUM_LIT> ; const Function * F = I -> getParent ( ) -> getParent ( ) ; if ( const auto * MC = dyn_cast < MemTransferInst > ( I ) ) { ConstantInt * C = dyn_cast < ConstantInt > ( MC -> getLength ( ) ) ; if ( ! C ) return - <NUM_LIT> ; const unsigned Size = C -> getValue ( ) . getZExtValue ( ) ; const Align DstAlign = * MC -> getDestAlign ( ) ; const Align SrcAlign = * MC -> getSourceAlign ( ) ; MOp = MemOp :: Copy ( Size , false , DstAlign , SrcAlign , false , false ) ; DstAddrSpace = MC -> getDestAddressSpace ( ) ; SrcAddrSpace = MC -> getSourceAddressSpace ( ) ; } else if ( const auto * MS = dyn_cast < MemSetInst > ( I ) ) { ConstantInt * C = dyn_cast < ConstantInt > ( MS -> getLength ( ) ) ; if ( ! C ) return - <NUM_LIT> ; const unsigned Size = C -> getValue ( ) . getZExtValue ( ) ; const Align DstAlign = * MS -> getDestAlign ( ) ; MOp = MemOp :: Set ( Size , false , DstAlign , false , false ) ; DstAddrSpace = MS -> getDestAddressSpace ( ) ; } else llvm_unreachable (


<s> SmallVector < const Value * , <NUM_LIT> > Operands ( I . operand_values ( ) ) ; Cost += getUserCost ( & I , Operands , TargetTransformInfo :: TCK_SizeAndLatency ) ; } } unsigned UnrollCount = <NUM_LIT> ; if ( ST -> isThumb1Only ( ) ) { unsigned ExitingValues = <NUM_LIT> ; SmallVector < BasicBlock * , <NUM_LIT> > ExitBlocks ; L -> getExitBlocks ( ExitBlocks ) ; for ( auto * Exit : ExitBlocks ) { unsigned LiveOuts = count_if ( Exit -> phis ( ) , [ ] ( auto & PH ) { return PH . getNumOperands ( ) != <NUM_LIT> || ! isa < GetElementPtrInst > ( PH . getOperand ( <NUM_LIT> ) ) ; } ) ; ExitingValues = ExitingValues < LiveOuts ? LiveOuts : ExitingValues ; } if ( ExitingValues ) UnrollCount /= ExitingValues ; if ( UnrollCount <= <NUM_LIT> ) return ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << UnrollCount << <STR_LIT> ) ; UP . Partial = true ; UP . Runtime = true ; UP . UnrollRemainder = true ; UP . DefaultUnrollRuntimeCount = UnrollCount ; UP . UnrollAndJam = true ; UP . UnrollAndJamInnerLoopThreshold = <NUM_LIT> ; if ( Cost < <NUM_LIT> ) UP


<s> UP . UpperBound = true ; if ( ! ST -> isMClass ( ) ) return BasicTTIImplBase :: getUnrollingPreferences ( L , SE , UP ) ; UP . OptSizeThreshold = <NUM_LIT> ; UP . PartialOptSizeThreshold = <NUM_LIT> ; if ( L -> getHeader ( ) -> getParent ( ) -> hasOptSize ( ) ) return ; SmallVector < BasicBlock * , <NUM_LIT> > ExitingBlocks ; L -> getExitingBlocks ( ExitingBlocks ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << L -> getNumBlocks ( ) << <STR_LIT> << <STR_LIT> << ExitingBlocks . size ( ) << <STR_LIT> ) ; if ( ExitingBlocks . size ( ) > <NUM_LIT> ) return ; if ( ST -> hasBranchPredictor ( ) && L -> getNumBlocks ( ) > <NUM_LIT> ) return ; if ( getBooleanLoopAttribute ( L , <STR_LIT> ) ) return ; InstructionCost Cost


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { Align MemAlign = getKnownAlignment ( II . getArgOperand ( <NUM_LIT> ) , IC . getDataLayout ( ) , & II , & IC . getAssumptionCache ( ) , & IC . getDominatorTree ( ) ) ; unsigned AlignArg = II . getNumArgOperands ( ) - <NUM_LIT> ; Value * AlignArgOp = II . getArgOperand ( AlignArg ) ; MaybeAlign Align = cast < ConstantInt > ( AlignArgOp ) -> getMaybeAlignValue ( ) ; if ( Align && * Align < MemAlign ) { return IC . replaceOperand ( II , AlignArg , ConstantInt :: get ( Type :: getInt32Ty ( II . getContext ( ) ) , MemAlign . value ( ) , false ) ) ; } break ; } case <STR_LIT> : { Value * Arg = II . getArgOperand ( <NUM_LIT> ) ; Value * ArgArg ; if ( match ( Arg , PatternMatch :: m_Intrinsic < <STR_LIT> > ( PatternMatch :: m_Value ( ArgArg ) ) ) && II . getType ( ) == ArgArg -> getType ( ) ) { return IC . replaceInstUsesWith ( II , ArgArg ) ; } Constant * XorMask ; if ( match ( Arg , m_Xor ( PatternMatch :: m_Intrinsic < <STR_LIT> > ( PatternMatch :: m_Value ( ArgArg ) ) , PatternMatch :: m_Constant ( XorMask ) ) ) && II . getType ( ) == ArgArg -> getType ( ) ) { if ( auto * CI = dyn_cast < ConstantInt > ( XorMask ) ) { if ( CI -> getValue ( ) . trunc ( <NUM_LIT> ) . isAllOnesValue ( ) ) { auto TrueVector = IC . Builder . CreateVectorSplat ( cast < FixedVectorType > ( II . getType ( ) ) -> getNumElements ( ) , IC . Builder . getTrue ( ) ) ; return BinaryOperator :: Create ( Instruction :: Xor , ArgArg , TrueVector ) ; } } } KnownBits ScalarKnown ( <NUM_LIT> ) ; if ( IC . SimplifyDemandedBits ( & II , <NUM_LIT> , APInt :: getLowBitsSet ( <NUM_LIT> , <NUM_LIT> ) , ScalarKnown , <NUM_LIT> ) ) { return & II ; } break ; } case <STR_LIT> : { Value * Arg = II . getArgOperand ( <NUM_LIT> ) ; Value * ArgArg ; if ( match ( Arg , PatternMatch :: m_Intrinsic < <STR_LIT> > ( PatternMatch :: m_Value ( ArgArg ) ) ) ) { return IC . replaceInstUsesWith ( II , ArgArg ) ; } if ( ! II . getMetadata ( LLVMContext


<s> int <ISA_LIT>TTIImpl :: getCmpSelInstrCost ( unsigned Opcode , Type * ValTy , Type * CondTy , TTI :: TargetCostKind CostKind , const Instruction * I ) { int ISD = TLI -> InstructionOpcodeToISD ( Opcode ) ; if ( CostKind == TTI :: TCK_CodeSize && ISD == <STR_LIT> && ST -> isThumb ( ) && ! ValTy -> isVectorTy ( ) ) { if ( TLI -> getValueType ( DL , ValTy , true ) == <STR_LIT> ) return TTI :: TCC_Expensive ; int Cost = TLI -> getTypeLegalizationCost ( DL , ValTy ) . first ; ++ Cost ; if ( ValTy -> isIntegerTy ( <NUM_LIT> ) ) ++ Cost ; return Cost


<s> ExtSize = TypeSize ; } } } TruncInst * T ; if ( ( I -> getOpcode ( ) == Instruction :: Store || match ( I , m_Intrinsic < <STR_LIT> > ( ) ) ) && ( T = dyn_cast < TruncInst > ( I -> getOperand ( <NUM_LIT> ) ) ) ) { unsigned TypeSize = T -> getOperand ( <NUM_LIT> ) -> getType ( ) -> getScalarSizeInBits ( ) ; if ( ( ( EltSize == <NUM_LIT> && TypeSize == <NUM_LIT> ) || ( EltSize == <NUM_LIT> && ( TypeSize == <NUM_LIT> || TypeSize == <NUM_LIT> ) ) ) && TypeSize * NumElems == <NUM_LIT> ) ExtSize = TypeSize ; } } if ( ExtSize * NumElems != <NUM_LIT> || NumElems < <NUM_LIT> ) return ScalarCost ; if ( ExtSize == <NUM_LIT> ) return VectorCost ; if ( ExtSize != <NUM_LIT> && ExtSize != <NUM_LIT> ) return ScalarCost ; if ( const auto * BC = dyn_cast < BitCastInst > ( Ptr ) ) Ptr = BC -> getOperand ( <NUM_LIT> ) ; if ( const auto * GEP = dyn_cast < GetElementPtrInst > ( Ptr ) ) { if ( GEP -> getNumOperands ( ) != <NUM_LIT> ) return ScalarCost ; unsigned Scale = DL . getTypeAllocSize (


<s> } return true ; } switch ( I . getOpcode ( ) ) { default : break ; case Instruction :: FPToSI : case Instruction :: FPToUI : case Instruction :: SIToFP : case Instruction :: UIToFP : case Instruction :: FPTrunc : case Instruction :: FPExt : return ! ST -> hasFP<ISA_LIT>v8Base ( ) ; } if ( VT . isInteger ( ) && VT . getSizeInBits ( ) >= <NUM_LIT> ) { switch ( ISD ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; } } if ( ! VT . isFloatingPoint ( ) ) return false ; if ( TLI -> useSoftFloat ( ) ) { switch ( I . getOpcode ( ) ) { default : return true ; case Instruction :: Alloca : case Instruction :: Load : case Instruction :: Store : case Instruction :: Select : case Instruction :: PHI : return false ; } } if ( I . getType ( ) -> isDoubleTy ( ) && ! ST -> hasFP64 ( ) ) return true ; if ( I . getType ( ) -> isHalfTy ( ) && ! ST -> hasFullFP16 ( ) ) return true ; return


<s> { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , } ; if ( const auto * Entry = CostTableLookup ( CostTbl , ISDOpcode , LT . second ) ) return LT . first * Entry -> Cost ; int Cost = BaseT :: getArithmeticInstrCost ( Opcode , Ty , Op1Info , Op2Info , Opd1PropInfo , Opd2PropInfo ) ; if ( LT . second == <STR_LIT> && Op2Info == TargetTransformInfo :: OK_UniformConstantValue ) Cost += <NUM_LIT> ; return Cost ; } auto LooksLikeAFreeShift = [ & ]


<s> if ( NumElts % Factor == <NUM_LIT> && TLI -> isLegalInterleavedAccessType ( Factor , SubVecTy , DL ) ) return Factor * BaseCost * TLI -> getNumInterleavedAccesses ( SubVecTy , DL ) ; if ( ST -> hasMVEIntegerOps ( ) && Factor == <NUM_LIT> && NumElts / Factor > <NUM_LIT> && VecTy -> isIntOrIntVectorTy ( ) && DL . getTypeSizeInBits ( SubVecTy ) <= <NUM_LIT> ) return <NUM_LIT> * BaseCost ; } return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment , AddressSpace ,


<s> unsigned NumElts = cast < VectorType > ( VecTy ) -> getNumElements ( ) ; auto * SubVecTy = VectorType :: get ( VecTy -> getScalarType ( ) , NumElts / Factor ) ; int BaseCost = ST -> hasMVEIntegerOps ( ) ? ST -> getMVEVectorCostFactor ( ) : <NUM_LIT> ; if ( NumElts % Factor == <NUM_LIT> && TLI -> isLegalInterleavedAccessType ( Factor , SubVecTy , DL ) ) return Factor

* BaseCost * TLI -> getNumInterleavedAccesses ( SubVecTy , DL ) ;


<s> std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Src ) ; if ( ST -> hasNEON ( ) && Src -> isVectorTy ( ) && ( Alignment && * Alignment != Align ( <NUM_LIT> ) ) && cast < VectorType > ( Src ) -> getElementType ( ) -> isDoubleTy ( ) ) { return LT . first


<s> case Instruction :: FCmp : return false ; case Instruction :: ICmp : case Instruction :: Add : return ScalarBits < <NUM_LIT> && ( ScalarBits * cast < VectorType > ( Ty ) -> getNumElements ( ) ) % <NUM_LIT> == <NUM_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ; } return false


<s> assert ( Ty -> isIntegerTy ( ) ) ; unsigned Bits = Ty -> getPrimitiveSizeInBits ( ) ; if ( Bits == <NUM_LIT> || Bits > <NUM_LIT> ) return <NUM_LIT> ; int64_t SImmVal = Imm . getSExtValue ( ) ; uint64_t ZImmVal = Imm . getZExtValue ( ) ; if ( ! ST -> isThumb ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ?


<s> if ( ! ST -> isThumb ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ; } if ( ST -> isThumb2 ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT>


<s> if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { ImmutableCallSite CS ( & I ) ; if ( const Function * F = CS . getCalledFunction ( ) ) { if ( ! isLoweredToCall ( F ) ) continue ; } return ; } SmallVector < const Value * , <NUM_LIT> > Operands ( I . value_op_begin ( ) , I . value_op_end ( ) ) ; Cost += getUserCost ( & I , Operands ) ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ) ; UP . Partial = true ; UP . Runtime =


<s> if ( ! ST -> isThumb2 ( ) ) return ; SmallVector < BasicBlock * , <NUM_LIT> > ExitingBlocks ; L -> getExitingBlocks ( ExitingBlocks ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << L -> getNumBlocks ( ) << <STR_LIT> << <STR_LIT> << ExitingBlocks . size ( ) << <STR_LIT> ) ; if ( ExitingBlocks . size ( ) > <NUM_LIT> ) return ; if ( ST -> hasBranchPredictor ( ) && L -> getNumBlocks ( ) > <NUM_LIT> ) return ; unsigned Cost = <NUM_LIT> ; for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { ImmutableCallSite CS ( & I ) ; if ( const Function * F = CS . getCalledFunction ( ) ) { if ( ! isLoweredToCall ( F ) ) continue ; } return ; } SmallVector < const Value * , <NUM_LIT> > Operands ( I . value_op_begin ( ) , I . value_op_end ( ) ) ; Cost += getUserCost ( & I , Operands ) ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ) ; UP . Partial = true ; UP . Runtime = true ; UP . UnrollRemainder = true ; UP . DefaultUnrollRuntimeCount = <NUM_LIT> ; UP . UnrollAndJam = true ; UP . UnrollAndJamInnerLoopThreshold


<s> EVT ValVT = TLI -> getValueType ( DL , ValTy ) ; int ISD = TLI -> InstructionOpcodeToISD ( Opcode ) ; if ( ! ST -> hasMVEIntegerOps ( ) || ! ValVT . isSimple ( ) || ISD != <STR_LIT> ) return BaseT :: getArithmeticReductionCost ( Opcode , ValTy , IsPairwiseForm , CostKind ) ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ; static const CostTblEntry CostTblAdd [ ] { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , } ; if ( const auto * Entry = CostTableLookup ( CostTblAdd , ISD , LT . second ) ) return Entry -> Cost * ST -> getMVEVectorCostFactor ( CostKind ) * LT . first ; return BaseT :: getArithmeticReductionCost ( Opcode , ValTy , IsPairwiseForm ,


<s> if ( Opcode == Instruction :: And ) { if ( Imm == <NUM_LIT> || Imm == <NUM_LIT> ) return <NUM_LIT> ; return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( ~ Imm , Ty , CostKind ) ) ; } if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( - Imm , Ty , CostKind ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == <NUM_LIT> ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; if ( ST -> isThumb ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; } if ( Opcode == Instruction :: Xor && Imm . isAllOnesValue