<s> unsigned <STR_LIT> ( unsigned Opcode , Type * Ty , OperandValueKind Op1Info , OperandValueKind Op2Info ) const { int ISDOpcode = TLI -> InstructionOpcodeToISD ( Opcode ) ; std :: pair < unsigned , MVT > LT = TLI -> getTypeLegalizationCost ( Ty ) ; const unsigned FunctionCallDivCost = <NUM_LIT> ; const unsigned ReciprocalDivCost = <NUM_LIT> ; static const CostTblEntry < MVT > CostTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , } ; int Idx = - <NUM_LIT> ; if ( ST -> hasNEON ( ) ) Idx = CostTableLookup < MVT > ( CostTbl , array_lengthof ( CostTbl ) , ISDOpcode , LT . second ) ; if ( Idx != - <NUM_LIT> ) return LT . first * CostTbl [ Idx ] . Cost ;

return TargetTransformInfo :: getArithmeticInstrCost ( Opcode , Ty , Op1Info , Op2Info ) ;


<s> unsigned <STR_LIT> ( unsigned Opcode , Type * Ty , OperandValueKind Op1Info , OperandValueKind Op2Info ) const { int ISDOpcode = TLI -> InstructionOpcodeToISD ( Opcode ) ; std :: pair < unsigned , MVT > LT = TLI -> getTypeLegalizationCost ( Ty ) ; const unsigned FunctionCallDivCost = <NUM_LIT> ; const unsigned ReciprocalDivCost = <NUM_LIT> ; static const CostTblEntry < MVT > CostTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , } ; int Idx = - <NUM_LIT> ; if ( ST -> hasNEON ( ) ) Idx = CostTableLookup < MVT > ( CostTbl , array_lengthof ( CostTbl ) , ISDOpcode , LT . second ) ;

if ( Idx != - <NUM_LIT> ) return LT . first * CostTbl [ Idx ] . Cost ;


<s> BasicBlock * BB = L -> getLoopLatch ( ) ; if ( BB -> getParent ( ) -> optForSize ( ) ) return ; unsigned Cost = <NUM_LIT> ; for ( auto & I : * BB ) { if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { ImmutableCallSite CS ( & I ) ; if ( const Function * F = CS . getCalledFunction ( ) ) {

if ( ! isLoweredToCall ( F ) ) continue ;


<s> const FeatureBitset & CalleeBits = TM . getSubtargetImpl ( * Callee ) -> getFeatureBits ( ) ; bool MatchExact = ( CallerBits & ~ InlineFeaturesAllowed ) == ( CalleeBits & ~ InlineFeaturesAllowed ) ; bool MatchSubset = ( ( CallerBits & CalleeBits ) & InlineFeaturesAllowed ) == ( CalleeBits & InlineFeaturesAllowed ) ; return MatchExact && MatchSubset ;


<s> if ( isa < FCmpInst > ( & I ) ) return false ; if ( isa < FPExtInst > ( & I ) || isa < FPTruncInst > ( & I ) ) return false ; if ( isa < SExtInst > ( & I ) || isa < ZExtInst > ( & I ) ) if ( ! I . getOperand ( <NUM_LIT> ) -> hasOneUse ( ) || ! isa < LoadInst > ( I . getOperand ( <NUM_LIT> ) ) ) return false ; if ( isa < TruncInst > ( & I ) ) if ( ! I . hasOneUse ( ) || ! isa < StoreInst > ( * I . user_begin ( ) ) ) return false ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; T -> dump ( ) ) ; return false ; } if ( isa < StoreInst > ( I ) || isa < LoadInst > ( I ) ) { Value * Ptr = isa < LoadInst > ( I ) ? I . getOperand ( <NUM_LIT> ) : I . getOperand ( <NUM_LIT> ) ; int64_t NextStride = getPtrStride ( PSE , Ptr , L ) ; if ( Stride == <NUM_LIT> && NextStride == <NUM_LIT> ) { Stride = NextStride ; continue ; } if ( Stride != NextStride ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } } }


<s> if ( ! ST -> hasMVEIntegerOps ( ) || DisableTailPredication ) return false ; return true ;


<s> int <ISA_LIT>TTIImpl :: getAddressComputationCost ( Type * Ty , ScalarEvolution * SE , const SCEV * Ptr ) { unsigned NumVectorInstToHideOverhead = <NUM_LIT> ; int MaxMergeDistance = <NUM_LIT> ; if ( ST -> hasNEON ( ) ) { if ( Ty -> isVectorTy ( ) && SE && ! BaseT :: isConstantStridedAccessLessThan ( SE , Ptr , MaxMergeDistance + <NUM_LIT> ) ) return NumVectorInstToHideOverhead ;


<s> { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , } ; if ( const auto * Entry = CostTableLookup ( CostTbl , ISDOpcode , LT . second ) ) return LT . first * Entry -> Cost ; int Cost = BaseT :: getArithmeticInstrCost ( Opcode , Ty , CostKind , Op1Info , Op2Info , Opd1PropInfo , Opd2PropInfo ) ; if ( LT . second == <STR_LIT> && Op2Info == TargetTransformInfo :: OK_UniformConstantValue ) Cost += <NUM_LIT> ;


<s> int ISD = TLI -> InstructionOpcodeToISD ( Opcode ) ; if ( ST -> hasNEON ( ) && ValTy -> isVectorTy ( ) && ISD == <STR_LIT> ) { static const TypeConversionCostTblEntry NEONVectorSelectTbl [ ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; EVT SelCondTy = TLI -> getValueType ( DL , CondTy ) ; EVT SelValTy = TLI -> getValueType ( DL , ValTy ) ; if ( SelCondTy . isSimple ( ) && SelValTy . isSimple ( ) ) { if ( const auto * Entry = ConvertCostTableLookup ( NEONVectorSelectTbl , ISD , SelCondTy . getSimpleVT ( ) , SelValTy . getSimpleVT ( ) ) ) return Entry -> Cost ; } std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ;


<s> assert ( DataTy -> isVectorTy ( ) && <STR_LIT> ) ; auto * VTy = cast < FixedVectorType > ( DataTy ) ; unsigned NumElems = VTy -> getNumElements ( ) ; unsigned EltSize = VTy -> getScalarSizeInBits ( ) ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , DataTy ) ; unsigned VectorCost = NumElems * LT . first ; unsigned ScalarCost = NumElems * LT . first + BaseT :: getScalarizationOverhead ( VTy , { } ) ; if ( Alignment < EltSize / <NUM_LIT> ) return ScalarCost ; unsigned ExtSize = EltSize ; if ( I != nullptr ) { if ( ( I -> getOpcode ( ) == Instruction :: Load || match ( I , m_Intrinsic < <STR_LIT> > ( ) ) ) && I -> hasOneUse ( ) ) { const User * Us = * I -> users ( ) . begin ( ) ; if ( isa < ZExtInst > ( Us ) || isa < SExtInst > ( Us ) ) { unsigned TypeSize = cast < Instruction > ( Us ) -> getType ( ) -> getScalarSizeInBits ( ) ; if ( ( ( TypeSize == <NUM_LIT> && ( EltSize == <NUM_LIT> || EltSize == <NUM_LIT> ) ) || ( TypeSize == <NUM_LIT> && EltSize == <NUM_LIT> ) ) && TypeSize * NumElems == <NUM_LIT> ) { ExtSize = TypeSize ; } } } TruncInst * T ; if ( ( I -> getOpcode ( ) == Instruction :: Store || match ( I , m_Intrinsic < <STR_LIT> > ( ) ) ) && ( T = dyn_cast < TruncInst > ( I -> getOperand ( <NUM_LIT> ) ) ) ) { unsigned TypeSize = T -> getOperand ( <NUM_LIT> ) -> getType ( ) -> getScalarSizeInBits ( ) ; if ( ( ( EltSize == <NUM_LIT> && TypeSize == <NUM_LIT> ) || ( EltSize == <NUM_LIT> && ( TypeSize == <NUM_LIT> || TypeSize == <NUM_LIT> ) ) ) && TypeSize * NumElems == <NUM_LIT> ) ExtSize = TypeSize ; } } if ( ExtSize * NumElems != <NUM_LIT> || NumElems < <NUM_LIT> ) return ScalarCost ; if ( ExtSize == <NUM_LIT> ) return VectorCost ; if ( ExtSize != <NUM_LIT> && ExtSize != <NUM_LIT> ) return ScalarCost ; if ( auto BC = dyn_cast < BitCastInst > ( Ptr ) ) Ptr = BC -> getOperand ( <NUM_LIT> ) ; if ( auto * GEP = dyn_cast < GetElementPtrInst > ( Ptr ) ) { if ( GEP -> getNumOperands ( ) != <NUM_LIT> ) return ScalarCost ; unsigned Scale = DL . getTypeAllocSize ( GEP -> getResultElementType ( ) ) ; if ( Scale != <NUM_LIT> && Scale * <NUM_LIT> != ExtSize ) return ScalarCost ; if ( auto ZExt = dyn_cast < ZExtInst > ( GEP -> getOperand ( <NUM_LIT> ) ) ) { if ( ZExt -> getOperand ( <NUM_LIT> ) -> getType ( ) -> getScalarSizeInBits ( ) <= ExtSize ) return VectorCost ; }


<s> auto * SubVecTy = FixedVectorType :: get ( VecTy -> getScalarType ( ) , NumElts / Factor ) ; int BaseCost = ST -> hasMVEIntegerOps ( ) ? ST -> getMVEVectorCostFactor ( ) : <NUM_LIT> ; if ( NumElts % Factor == <NUM_LIT> && TLI -> isLegalInterleavedAccessType ( Factor , SubVecTy , DL ) ) return Factor * BaseCost * TLI -> getNumInterleavedAccesses ( SubVecTy , DL ) ; if ( ST -> hasMVEIntegerOps ( ) && Factor == <NUM_LIT> && NumElts / Factor > <NUM_LIT> && VecTy -> isIntOrIntVectorTy ( ) && DL . getTypeSizeInBits ( SubVecTy ) <= <NUM_LIT> ) return <NUM_LIT> * BaseCost ; }

return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment , AddressSpace , CostKind , UseMaskForCond , UseMaskForGaps ) ;


<s> assert ( isa < VectorType > ( VecTy ) && <STR_LIT> ) ; bool EltIs64Bits = DL . getTypeSizeInBits ( VecTy -> getScalarType ( ) ) == <NUM_LIT> ; if ( Factor <= TLI -> getMaxSupportedInterleaveFactor ( ) && ! EltIs64Bits && ! UseMaskForCond && ! UseMaskForGaps ) { unsigned NumElts = cast < FixedVectorType > ( VecTy ) -> getNumElements ( ) ; auto * SubVecTy = FixedVectorType :: get ( VecTy -> getScalarType ( ) , NumElts / Factor ) ; int BaseCost = ST -> hasMVEIntegerOps ( ) ? ST -> getMVEVectorCostFactor ( ) : <NUM_LIT> ; if ( NumElts % Factor == <NUM_LIT> && TLI -> isLegalInterleavedAccessType ( Factor , SubVecTy , DL ) ) return Factor * BaseCost * TLI -> getNumInterleavedAccesses ( SubVecTy , DL ) ; if ( ST -> hasMVEIntegerOps ( ) && Factor == <NUM_LIT> && NumElts / Factor > <NUM_LIT> && VecTy -> isIntOrIntVectorTy ( ) && DL . getTypeSizeInBits ( SubVecTy ) <= <NUM_LIT> ) return <NUM_LIT> * BaseCost ; }


<s> int64_t SImmVal = Imm . getSExtValue ( ) ; uint64_t ZImmVal = Imm . getZExtValue ( ) ; if ( ! ST -> isThumb ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ; } if ( ST -> isThumb2 ( ) ) {


<s> int <ISA_LIT>TTIImpl :: getIntImmCost ( const APInt & Imm , Type * Ty , TTI :: TargetCostKind CostKind ) { assert ( Ty -> isIntegerTy ( ) ) ; unsigned Bits = Ty -> getPrimitiveSizeInBits ( ) ; if ( Bits == <NUM_LIT> || Imm . getActiveBits ( ) >= <NUM_LIT> ) return <NUM_LIT> ; int64_t SImmVal = Imm . getSExtValue ( ) ; uint64_t ZImmVal = Imm . getZExtValue ( ) ; if ( ! ST -> isThumb ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ; } if ( ST -> isThumb2 ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ; } if ( Bits == <NUM_LIT> || ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) ) return <NUM_LIT> ; if ( ( ~ SImmVal < <NUM_LIT> ) || <STR_LIT> ( ZImmVal ) ) return <NUM_LIT> ;


<s> if ( Opcode == Instruction :: And ) { if ( Imm == <NUM_LIT> || Imm == <NUM_LIT> ) return <NUM_LIT> ; return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( ~ Imm , Ty , CostKind ) ) ; } if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( - Imm , Ty , CostKind ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == <NUM_LIT> ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; if ( ST -> isThumb ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; } if ( Opcode == Instruction :: Xor && Imm . isAllOnesValue ( ) ) return <NUM_LIT> ; return getIntImmCost ( Imm , Ty , CostKind ) ;


<s> std :: vector < EVT > MemOps ; if ( getTLI ( ) -> findOptimalMemOpLowering ( MemOps , Limit , MemOp :: Copy ( Size , false , DstAlign , SrcAlign , true ) , MI -> getDestAddressSpace ( ) , MI -> getSourceAddressSpace ( ) , F -> getAttributes ( ) ) ) return MemOps . size ( ) * <NUM_LIT> ; return LibCallCost ;


<s> int <ISA_LIT>TTIImpl :: getMemcpyCost ( const Instruction * I ) { const MemCpyInst * MI = dyn_cast < MemCpyInst > ( I ) ; assert ( MI && <STR_LIT> ) ; ConstantInt * C = dyn_cast < ConstantInt > ( MI -> getLength ( ) ) ; const unsigned LibCallCost = <NUM_LIT> ;


<s> if ( ST -> hasNEON ( ) && Src -> isVectorTy ( ) && ( Alignment && * Alignment != Align ( <NUM_LIT> ) ) && cast < VectorType > ( Src ) -> getElementType ( ) -> isDoubleTy ( ) ) { return LT . first * <NUM_LIT> ; } int BaseCost = ST -> hasMVEIntegerOps ( ) && Src -> isVectorTy ( ) ? ST -> getMVEVectorCostFactor ( ) : <NUM_LIT> ; return BaseCost * LT . first ;


<s> if ( CostKind != TTI :: TCK_RecipThroughput ) return <NUM_LIT> ; if ( TLI -> getValueType ( DL , Src , true ) == <STR_LIT> ) return BaseT :: getMemoryOpCost ( Opcode , Src , Alignment , AddressSpace , CostKind ) ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Src ) ; if ( ST -> hasNEON ( ) && Src -> isVectorTy ( ) && ( Alignment && * Alignment != Align ( <NUM_LIT> ) ) && cast < VectorType > ( Src ) -> getElementType ( ) -> isDoubleTy ( ) ) { return LT . first * <NUM_LIT> ; }

int BaseCost = ST -> hasMVEIntegerOps ( ) && Src -> isVectorTy ( ) ? ST -> getMVEVectorCostFactor ( ) : <NUM_LIT> ;


<s> unsigned getRegisterBitWidth ( bool Vector ) const { if ( Vector ) { if ( ST -> hasNEON ( ) ) return <NUM_LIT> ; if ( ST -> hasMVEIntegerOps ( ) ) return <NUM_LIT> ; return <NUM_LIT> ; } return <NUM_LIT> ;


<s> if ( Kind == TTI :: SK_Broadcast ) { static const CostTblEntry NEONDupTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( NEONDupTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost ; } if ( Kind == TTI :: SK_Reverse ) { static const CostTblEntry NEONShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ;

if ( const auto * Entry = CostTableLookup ( NEONShuffleTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost ;


<s> { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( NEONSelShuffleTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost ; } } if ( ST -> hasMVEIntegerOps ( ) ) { if ( Kind == TTI :: SK_Broadcast ) { static const CostTblEntry MVEDupTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( MVEDupTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost * ST -> getMVEVectorCostFactor ( ) ; } } int BaseCost = ST -> hasMVEIntegerOps ( ) && Tp -> isVectorTy ( ) ? ST -> getMVEVectorCostFactor ( ) : <NUM_LIT> ;

return BaseCost * BaseT :: getShuffleCost ( Kind , Tp , Index , SubTp ) ;


<s> if ( ST -> hasMVEIntegerOps ( ) && ( Opcode == Instruction :: InsertElement || Opcode == Instruction :: ExtractElement ) ) { return std :: max ( BaseT :: getVectorInstrCost ( Opcode , ValTy , Index ) , ST -> getMVEVectorCostFactor ( ) ) * cast < FixedVectorType > ( ValTy ) -> getNumElements ( ) / <NUM_LIT> ; }

return BaseT :: getVectorInstrCost ( Opcode , ValTy , Index ) ;


<s> if ( ! SE . hasLoopInvariantBackedgeTakenCount ( L ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } const SCEV * BackedgeTakenCount = SE . getBackedgeTakenCount ( L ) ; if ( isa < SCEVCouldNotCompute > ( BackedgeTakenCount ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } const SCEV * TripCountSCEV = SE . getAddExpr ( BackedgeTakenCount , SE . getOne ( BackedgeTakenCount -> getType ( ) ) ) ; if ( SE . getUnsignedRangeMax ( TripCountSCEV ) . getBitWidth ( ) > <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto MaybeCall = [ this ] ( Instruction & I ) { const <ISA_LIT>TargetLowering * TLI = getTLI ( ) ; unsigned ISD = TLI -> InstructionOpcodeToISD ( I . getOpcode ( ) ) ; EVT VT = TLI -> getValueType ( DL , I . getType ( ) , true ) ; if ( TLI -> getOperationAction ( ISD , VT ) == TargetLowering :: LibCall ) return true ; if ( auto * Call = dyn_cast < CallInst > ( & I ) ) { if ( isa < IntrinsicInst > ( Call ) ) { if ( const Function * F = Call -> getCalledFunction ( ) ) return isLoweredToCall ( F ) ; } return true ; } switch ( I . getOpcode ( ) ) { default : break ; case Instruction :: FPToSI : case Instruction :: FPToUI : case Instruction :: SIToFP : case Instruction :: UIToFP : case Instruction :: FPTrunc : case Instruction :: FPExt : return ! ST -> hasFP<ISA_LIT>v8Base ( ) ; } if ( VT . isInteger ( ) && VT . getSizeInBits ( ) >= <NUM_LIT> ) { switch ( ISD ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; } } if ( ! VT . isFloatingPoint ( ) ) return false ; if ( TLI -> useSoftFloat ( ) ) { switch ( I . getOpcode ( ) ) { default : return true ; case Instruction :: Alloca : case Instruction :: Load : case Instruction :: Store : case Instruction :: Select : case Instruction :: PHI : return false ; } } if ( I . getType ( ) -> isDoubleTy ( ) && ! ST -> hasFP64 ( ) ) return true ; if ( I . getType ( ) -> isHalfTy ( ) && ! ST -> hasFullFP16 ( ) ) return true ; return false ; } ; auto IsHardwareLoopIntrinsic = [ ] ( Instruction & I ) { if ( auto * Call = dyn_cast < IntrinsicInst > ( & I ) ) { switch ( Call -> getIntrinsicID ( ) ) { default :


<s> if ( isa < VectorType > ( Ty ) ) return false ; unsigned EltWidth = Ty -> getScalarSizeInBits ( ) ;

return ( ( EltWidth == <NUM_LIT> && ( ! Alignment || * Alignment >= <NUM_LIT> ) ) || ( EltWidth == <NUM_LIT> && ( ! Alignment || * Alignment >= <NUM_LIT> ) ) || EltWidth == <NUM_LIT> ) ;


<s> bool <ISA_LIT>TTIImpl :: isLegalMaskedGather ( Type * Ty , MaybeAlign Alignment ) { if ( ! EnableMaskedGatherScatters || ! ST -> hasMVEIntegerOps ( ) ) return false ; if ( isa < VectorType > ( Ty ) ) return false ; unsigned EltWidth = Ty -> getScalarSizeInBits ( ) ;


<s> if ( VecTy -> getNumElements ( ) == <NUM_LIT> ) return false ; unsigned VecWidth = DataTy -> getPrimitiveSizeInBits ( ) ; if ( VecWidth != <NUM_LIT> && VecTy -> getElementType ( ) -> isFloatingPointTy ( ) ) return false ; } unsigned EltWidth = DataTy -> getScalarSizeInBits ( ) ; return ( EltWidth == <NUM_LIT> && ( ! Alignment || * Alignment >= <NUM_LIT> ) ) || ( EltWidth == <NUM_LIT> && ( ! Alignment || * Alignment >= <NUM_LIT> ) ) || ( EltWidth == <NUM_LIT> ) ;


<s> if ( VecTy -> getNumElements ( ) == <NUM_LIT> ) return false ; unsigned VecWidth = DataTy -> getPrimitiveSizeInBits ( ) ; if ( VecWidth != <NUM_LIT> && VecTy -> getElementType ( ) -> isFloatingPointTy ( ) ) return false ; }

unsigned EltWidth = DataTy -> getScalarSizeInBits ( ) ;


<s> bool isLegalMaskedStore ( Type * DataTy , MaybeAlign Alignment ) {

return isLegalMaskedLoad ( DataTy , Alignment ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( F -> getReturnType ( ) -> isDoubleTy ( ) && ! ST -> hasFP64 ( ) ) return true ; if ( F -> getReturnType ( ) -> isHalfTy ( ) && ! ST -> hasFullFP16 ( ) ) return true ; return ! ST -> hasFP<ISA_LIT>v8Base ( ) && ! ST -> hasVFP2Base ( ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return ! ST -> hasMVEIntegerOps ( ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> bool <ISA_LIT>TTIImpl :: isProfitableLSRChainElement ( Instruction * I ) { if ( IntrinsicInst * II = dyn_cast < IntrinsicInst > ( I ) ) { switch ( II -> getIntrinsicID ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; default : break ;


<s> if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; if ( L -> getNumBlocks ( ) > <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } assert ( L -> empty ( ) && <STR_LIT> ) ; HardwareLoopInfo HWLoopInfo ( L ) ; if ( ! HWLoopInfo . canAnalyze ( * LI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ;


<s> return false ; } assert ( L -> empty ( ) && <STR_LIT> ) ; HardwareLoopInfo HWLoopInfo ( L ) ; if ( ! HWLoopInfo . canAnalyze ( * LI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } if ( ! isHardwareLoopProfitable ( L , SE , AC , TLI , HWLoopInfo ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } if ( ! HWLoopInfo . isHardwareLoopCandidate ( SE , * LI , * DT ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } return canTailPredicateLoop ( L , LI , SE , DL , LAI ) ;


<s> if ( ST -> isROPI ( ) || ST -> isRWPI ( ) ) return ! C -> needsRelocation ( ) ; return true ;


<s> switch ( II -> getIntrinsicID ( ) ) { case <STR_LIT> : case <STR_LIT> : if ( ! II -> getFastMathFlags ( ) . allowReassoc ( ) ) return true ; return TLI -> useSoftFloat ( ) || ! TLI -> getSubtarget ( ) -> hasFPRegs ( ) ; case <STR_LIT> : case <STR_LIT> : return TLI -> useSoftFloat ( ) || ! TLI -> getSubtarget ( ) -> hasFPRegs ( ) || ! II -> getFastMathFlags ( ) . noNaNs ( ) ; default : return false ;


<s> return TLI -> useSoftFloat ( ) || ! TLI -> getSubtarget ( ) -> hasFPRegs ( ) || ! II -> getFastMathFlags ( ) . noNaNs ( ) ; default : return false ; }


<s> bool <ISA_LIT>TTIImpl :: shouldFavorBackedgeIndex ( const Loop * L ) const {

if ( L -> getHeader ( ) -> getParent ( ) -> hasOptSize ( ) ) return false ;


<s> bool <ISA_LIT>TTIImpl :: shouldFavorPostInc ( ) const { if ( ST -> hasMVEIntegerOps ( ) ) return true ; return false ;


<s> if ( ST -> hasMVEIntegerOps ( ) ) return true ; return false ;


<s> bool <ISA_LIT>TTIImpl :: useReductionIntrinsic ( unsigned Opcode , Type * Ty , TTI :: ReductionFlags Flags ) const { return ST -> hasMVEIntegerOps ( ) ;


<s> if ( ! ST -> isThumb2 ( ) ) return ; SmallVector < BasicBlock * , <NUM_LIT> > ExitingBlocks ; L -> getExitingBlocks ( ExitingBlocks ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << L -> getNumBlocks ( ) << <STR_LIT> << <STR_LIT> << ExitingBlocks . size ( ) << <STR_LIT> ) ; if ( ExitingBlocks . size ( ) > <NUM_LIT> ) return ; if ( ST -> hasBranchPredictor ( ) && L -> getNumBlocks ( ) > <NUM_LIT> ) return ; unsigned Cost = <NUM_LIT> ; for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { ImmutableCallSite CS ( & I ) ; if ( const Function * F = CS . getCalledFunction ( ) ) { if ( ! isLoweredToCall ( F ) ) continue ; } return ; } SmallVector < const Value * , <NUM_LIT> > Operands ( I . value_op_begin ( ) , I . value_op_end ( ) ) ; Cost += getUserCost ( & I , Operands ) ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << L -> getNumBlocks ( ) << <STR_LIT> << <STR_LIT> << ExitingBlocks . size ( ) << <STR_LIT> ) ; if ( ExitingBlocks . size ( ) > <NUM_LIT> ) return ; if ( ST -> hasBranchPredictor ( ) && L -> getNumBlocks ( ) > <NUM_LIT> ) return ; unsigned Cost = <NUM_LIT> ; for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { ImmutableCallSite CS ( & I ) ; if ( const Function * F = CS . getCalledFunction ( ) ) { if ( ! isLoweredToCall ( F ) ) continue ; } return ; } SmallVector < const Value * , <NUM_LIT> > Operands ( I . value_op_begin ( ) , I . value_op_end ( ) ) ; Cost += getUserCost ( & I , Operands ) ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ) ; UP . Partial = true ; UP . Runtime = true ;


<s> void * getAdjustedAnalysisPointer ( const void * ID ) override { if ( ID == & TargetTransformInfo :: ID ) return ( TargetTransformInfo * ) this ; return this ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { TargetTransformInfo :: getAnalysisUsage ( AU ) ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override {

TargetTransformInfo :: getAnalysisUsage ( AU ) ;


<s> std :: pair < unsigned , MVT > LT = TLI -> getTypeLegalizationCost ( Ty ) ; const unsigned FunctionCallDivCost = <NUM_LIT> ; const unsigned ReciprocalDivCost = <NUM_LIT> ; static const CostTblEntry < <STR_LIT> > CostTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , } ; int Idx = - <NUM_LIT> ;

if ( ST -> hasNEON ( ) ) Idx = CostTableLookup ( CostTbl , ISDOpcode , LT . second ) ;


<s> static const TypeConversionCostTblEntry < <STR_LIT> > NEONVectorSelectTbl [ ] = {


<s> unsigned <STR_LIT> ( unsigned Opcode , Type * ValTy , Type * CondTy ) const { int ISD = TLI -> InstructionOpcodeToISD ( Opcode ) ; if ( ST -> hasNEON ( ) && ValTy -> isVectorTy ( ) && ISD == <STR_LIT> ) { static const TypeConversionCostTblEntry < <STR_LIT> > NEONVectorSelectTbl [ ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; EVT SelCondTy = TLI -> getValueType ( CondTy ) ; EVT SelValTy = TLI -> getValueType ( ValTy ) ; if ( SelCondTy . isSimple ( ) && SelValTy . isSimple ( ) ) {

int Idx = ConvertCostTableLookup ( NEONVectorSelectTbl , ISD , SelCondTy . getSimpleVT ( ) , SelValTy . getSimpleVT ( ) ) ;


<s> if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ; } if ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) return <NUM_LIT> ;

if ( ( ~ ZImmVal < <NUM_LIT> ) || <STR_LIT> ( ZImmVal ) ) return <NUM_LIT> ;


<s> if ( ST -> isThumb2 ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ; }

if ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) return <NUM_LIT> ;


<s> unsigned getMaximumUnrollFactor ( ) const override { if ( ST -> isCortexA15 ( ) || ST -> isSwift ( ) ) return <NUM_LIT> ; return <NUM_LIT> ;


<s> unsigned getMaximumUnrollFactor ( ) const override { if ( ST -> isCortexA15 ( ) || ST -> isSwift ( ) ) return <NUM_LIT> ;


<s> if ( Src -> isVectorTy ( ) && Alignment != <NUM_LIT> && Src -> getVectorElementType ( ) -> isDoubleTy ( ) ) { return LT . first * <NUM_LIT> ;


<s> if ( ST -> isThumb1Only ( ) ) return <NUM_LIT> ; return <NUM_LIT> ;


<s> if ( ST -> hasNEON ( ) ) return <NUM_LIT> ; return <NUM_LIT> ; } return <NUM_LIT> ;


<s> if ( Kind != SK_Reverse ) return TargetTransformInfo :: getShuffleCost ( Kind , Tp , Index , SubTp ) ; static const CostTblEntry < <STR_LIT> > NEONShuffleTbl [ ] = {


<s> unsigned <STR_LIT> ( ShuffleKind Kind , Type * Tp , int Index , Type * SubTp ) const { if ( Kind != SK_Reverse ) return TargetTransformInfo :: getShuffleCost ( Kind , Tp , Index , SubTp ) ; static const CostTblEntry < <STR_LIT> > NEONShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < unsigned , MVT > LT = TLI -> getTypeLegalizationCost ( Tp ) ; int Idx = CostTableLookup ( NEONShuffleTbl , <STR_LIT> , LT . second ) ;

if ( Idx == - <NUM_LIT> ) return TargetTransformInfo :: getShuffleCost ( Kind , Tp , Index , SubTp ) ;


<s> void initializePass ( ) override { pushTTIStack ( this ) ;


<s> unsigned getNumberOfRegisters ( bool Vector ) { if ( Vector ) { if ( ST -> hasNEON ( ) ) return <NUM_LIT> ; if ( ST -> hasMVEIntegerOps ( ) ) return <NUM_LIT> ; return <NUM_LIT> ;


<s> bool isLegalMaskedStore ( Type * DataTy ) { return isLegalMaskedLoad ( DataTy ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getAddressComputationCost ( Type * Ty , bool IsComplex ) {

unsigned NumVectorInstToHideOverhead = <NUM_LIT> ;


<s> if ( ST -> hasNEON ( ) && ValTy -> isVectorTy ( ) && ISD == <STR_LIT> ) { static const TypeConversionCostTblEntry < <STR_LIT> > NEONVectorSelectTbl [ ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; EVT SelCondTy = TLI -> getValueType ( CondTy ) ; EVT SelValTy = TLI -> getValueType ( ValTy ) ; if ( SelCondTy . isSimple ( ) && SelValTy . isSimple ( ) ) {


<s> if ( ST -> isThumb2 ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ; } if ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) return <NUM_LIT> ;


<s> unsigned <ISA_LIT>TTIImpl :: getIntImmCost ( const APInt & Imm , Type * Ty ) { assert ( Ty -> isIntegerTy ( ) ) ; unsigned Bits = Ty -> getPrimitiveSizeInBits ( ) ; if ( Bits == <NUM_LIT> || Bits > <NUM_LIT> ) return <NUM_LIT> ; int32_t SImmVal = Imm . getSExtValue ( ) ; uint32_t ZImmVal = Imm . getZExtValue ( ) ; if ( ! ST -> isThumb ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ;

return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ;


<s> if ( Src -> isVectorTy ( ) && Alignment != <NUM_LIT> && Src -> getVectorElementType ( ) -> isDoubleTy ( ) ) {


<s> unsigned <ISA_LIT>TTIImpl :: getShuffleCost ( TTI :: ShuffleKind Kind , Type * Tp , int Index , Type * SubTp ) { if ( Kind != TTI :: SK_Reverse && Kind != TTI :: SK_Alternate ) return BaseT :: getShuffleCost ( Kind , Tp , Index , SubTp ) ; if ( Kind == TTI :: SK_Reverse ) { static const CostTblEntry < <STR_LIT> > NEONShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < unsigned , MVT > LT = TLI -> getTypeLegalizationCost ( Tp ) ; int Idx = CostTableLookup ( NEONShuffleTbl , <STR_LIT> , LT . second ) ; if ( Idx == - <NUM_LIT> ) return BaseT :: getShuffleCost ( Kind , Tp , Index , SubTp ) ; return LT . first * NEONShuffleTbl [ Idx ] . Cost ; } if ( Kind == TTI :: SK_Alternate ) { static const CostTblEntry < <STR_LIT> > NEONAltShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < unsigned , MVT > LT = TLI -> getTypeLegalizationCost ( Tp ) ; int Idx = CostTableLookup ( NEONAltShuffleTbl , <STR_LIT> , LT . second ) ;


<s> if ( ( Opcode == Instruction :: InsertElement || Opcode == Instruction :: ExtractElement ) && ValTy -> getVectorElementType ( ) -> isIntegerTy ( ) ) return <NUM_LIT> ;


<s> bool <ISA_LIT>TTIImpl :: emitGetActiveLaneMask ( Loop * L , LoopInfo * LI , ScalarEvolution & SE , bool TailFolded ) const { return false ;


<s> const unsigned ReciprocalDivCost = <NUM_LIT> ; static const CostTblEntry CostTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , } ; if ( const auto * Entry = CostTableLookup ( CostTbl , ISDOpcode , LT . second ) ) return LT . first * Entry -> Cost ; int Cost = BaseT :: getArithmeticInstrCost ( Opcode , Ty , CostKind , Op1Info , Op2Info , Opd1PropInfo , Opd2PropInfo ) ; if ( LT . second == <STR_LIT> && Op2Info == TargetTransformInfo :: OK_UniformConstantValue ) Cost += <NUM_LIT> ; return Cost ; } auto LooksLikeAFreeShift = [ & ] ( ) { if ( ST -> isThumb1Only ( ) || Ty -> isVectorTy ( ) ) return false ; if ( ! CxtI || ! CxtI -> hasOneUse ( ) || ! CxtI -> isShift ( ) ) return false ; if ( Op2Info != TargetTransformInfo :: OK_UniformConstantValue ) return false ; switch ( cast < Instruction > ( CxtI -> user_back ( ) ) -> getOpcode ( ) ) { case Instruction :: Add : case Instruction :: Sub : case Instruction :: And : case Instruction :: Xor :


<s> if ( ST -> hasNEON ( ) && Src -> isVectorTy ( ) && ( Alignment && * Alignment != Align ( <NUM_LIT> ) ) && cast < VectorType > ( Src ) -> getElementType ( ) -> isDoubleTy ( ) ) {


<s> assert ( isa < VectorType > ( Ty ) && <STR_LIT> ) ; unsigned ScalarBits = Ty -> getScalarSizeInBits ( ) ; if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; switch ( Opcode ) { case Instruction :: FAdd : case Instruction :: FMul : case Instruction :: And :


<s> case Instruction :: FCmp : return false ; case Instruction :: ICmp : case Instruction :: Add : return ScalarBits < <NUM_LIT> && ( ScalarBits * cast < FixedVectorType > ( Ty ) -> getNumElements ( ) ) % <NUM_LIT> == <NUM_LIT> ;


<s> const unsigned FunctionCallDivCost = <NUM_LIT> ; const unsigned ReciprocalDivCost = <NUM_LIT> ; static const CostTblEntry CostTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , } ;


<s> return LT . first * <NUM_LIT> ; } int BaseCost = ST -> hasMVEIntegerOps ( ) && Src -> isVectorTy ( ) ? ST -> getMVEVectorCostFactor ( ) : <NUM_LIT> ; return BaseCost * LT . first ;


<s> if ( ! isLoweredToCall ( F ) ) continue ; } return ; } if ( I . getType ( ) -> isVectorTy ( ) ) return ; SmallVector < const Value * , <NUM_LIT> > Operands ( I . value_op_begin ( ) , I . value_op_end ( ) ) ; Cost += getUserCost ( & I , Operands ) ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ) ; UP . Partial = true ; UP . Runtime = true ; UP . UpperBound = true ; UP . UnrollRemainder = true ; UP . DefaultUnrollRuntimeCount = <NUM_LIT> ; UP . UnrollAndJam = true ;

UP . UnrollAndJamInnerLoopThreshold = <NUM_LIT> ;


<s> bool <ISA_LIT>TTIImpl :: isLegalMaskedLoad ( Type * DataTy ) {

if ( ! EnableMaskedLoadStores || ! ST -> hasMVEIntegerOps ( ) ) return false ;


<s> if ( ! EnableMaskedLoadStores || ! ST -> hasMVEIntegerOps ( ) ) return false ; if ( DataTy -> isVectorTy ( ) ) { unsigned VecWidth = DataTy -> getPrimitiveSizeInBits ( ) ;

if ( VecWidth != <NUM_LIT> ) return false ;


<s> { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( NEONShuffleTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost ; } if ( Kind == TTI :: SK_Select ) { static const CostTblEntry NEONSelShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( NEONSelShuffleTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost ; } } if ( ST -> hasMVEIntegerOps ( ) ) { if ( Kind == TTI :: SK_Broadcast ) { static const CostTblEntry MVEDupTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( MVEDupTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost * ST -> getMVEVectorCostFactor ( TTI :: TCK_RecipThroughput ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; SmallVector < Instruction * , <NUM_LIT> > LiveOuts ; LiveOuts = llvm :: findDefsUsedOutsideOfLoop ( L ) ; bool ReductionsDisabled = EnableTailPredication == TailPredication :: EnabledNoReductions || EnableTailPredication == TailPredication :: ForceEnabledNoReductions ; for ( auto * I : LiveOuts ) { if ( ! I -> getType ( ) -> isIntegerTy ( ) && ! I -> getType ( ) -> isFloatTy ( ) && ! I -> getType ( ) -> isHalfTy ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } if ( ReductionsDisabled ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } } PredicatedScalarEvolution PSE = LAI -> getPSE ( ) ; SmallVector < Instruction * , <NUM_LIT> > LoadStores ; int ICmpCount = <NUM_LIT> ; for ( BasicBlock * BB : L -> blocks ( ) ) { for ( Instruction & I : BB -> instructionsWithoutDebug ( ) ) { if ( isa < PHINode > ( & I ) ) continue ; if ( ! canTailPredicateInstruction ( I , ICmpCount ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; I . dump ( ) ) ; return false ; } Type * T = I . getType ( ) ; if ( T -> isPointerTy ( ) ) T = T -> getPointerElementType ( ) ; if ( T -> getScalarSizeInBits ( ) > <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; T -> dump ( ) ) ; return false ; } if ( isa < StoreInst > ( I ) || isa < LoadInst > ( I ) ) { Value * Ptr = isa < LoadInst > ( I ) ? I . getOperand ( <NUM_LIT> ) : I . getOperand ( <NUM_LIT> ) ; int64_t NextStride = getPtrStride ( PSE , Ptr , L ) ; if ( NextStride == <NUM_LIT> ) { continue ; } else if ( NextStride == - <NUM_LIT> || ( NextStride == <NUM_LIT> && MVEMaxSupportedInterleaveFactor >= <NUM_LIT> ) || ( NextStride == <NUM_LIT> && MVEMaxSupportedInterleaveFactor >= <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ;

} else if ( EnableMaskedGatherScatters ) {


<s> if ( T -> isPointerTy ( ) ) T = T -> getPointerElementType ( ) ; if ( T -> getScalarSizeInBits ( ) > <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; T -> dump ( ) ) ; return false ; } if ( isa < StoreInst > ( I ) || isa < LoadInst > ( I ) ) { Value * Ptr = isa < LoadInst > ( I ) ? I . getOperand ( <NUM_LIT> ) : I . getOperand ( <NUM_LIT> ) ; int64_t NextStride = getPtrStride ( PSE , Ptr , L ) ; if ( NextStride == <NUM_LIT> ) { continue ; } else if ( NextStride == - <NUM_LIT> || ( NextStride == <NUM_LIT> && MVEMaxSupportedInterleaveFactor >= <NUM_LIT> ) || ( NextStride == <NUM_LIT> && MVEMaxSupportedInterleaveFactor >= <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } else if ( EnableMaskedGatherScatters ) { const SCEV * PtrScev = replaceSymbolicStrideSCEV ( PSE , llvm :: ValueToValueMap ( ) , Ptr ) ; if ( auto AR = dyn_cast < SCEVAddRecExpr > ( PtrScev ) ) { const SCEV * Step = AR -> getStepRecurrence ( * PSE . getSE ( ) ) ; if ( PSE . getSE ( ) -> isLoopInvariant ( Step , L ) ) continue ; } }


<s> std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ; static const CostTblEntry CostTblAdd [ ] { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , } ; if ( const auto * Entry = CostTableLookup ( CostTblAdd , ISD , LT . second ) ) return Entry -> Cost * ST -> getMVEVectorCostFactor ( ) * LT . first ;

return BaseT :: getArithmeticReductionCost ( Opcode , ValTy , IsPairwiseForm , CostKind ) ;


<s> if ( CostKind == TTI :: TCK_RecipThroughput && ( ST -> hasNEON ( ) || ST -> hasMVEIntegerOps ( ) ) ) {


<s> if ( CostKind == TTI :: TCK_RecipThroughput && ( ST -> hasNEON ( ) || ST -> hasMVEIntegerOps ( ) ) ) { return <NUM_LIT> ;


<s> return Cost ; } if ( ST -> hasNEON ( ) && ValTy -> isVectorTy ( ) && ISD == <STR_LIT> && CondTy ) { static const TypeConversionCostTblEntry NEONVectorSelectTbl [ ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; EVT SelCondTy = TLI -> getValueType ( DL , CondTy ) ; EVT SelValTy = TLI -> getValueType ( DL , ValTy ) ; if ( SelCondTy . isSimple ( ) && SelValTy . isSimple ( ) ) { if ( const auto * Entry = ConvertCostTableLookup ( NEONVectorSelectTbl , ISD , SelCondTy . getSimpleVT ( ) , SelValTy . getSimpleVT ( ) ) ) return Entry -> Cost ; } std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ; return LT . first ; }


<s> } if ( ST -> hasNEON ( ) && ValTy -> isVectorTy ( ) && ISD == <STR_LIT> && CondTy ) { static const TypeConversionCostTblEntry NEONVectorSelectTbl [ ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; EVT SelCondTy = TLI -> getValueType ( DL , CondTy ) ; EVT SelValTy = TLI -> getValueType ( DL , ValTy ) ; if ( SelCondTy . isSimple ( ) && SelValTy . isSimple ( ) ) { if ( const auto * Entry = ConvertCostTableLookup ( NEONVectorSelectTbl , ISD , SelCondTy . getSimpleVT ( ) , SelValTy . getSimpleVT ( ) ) ) return Entry -> Cost ; } std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ;


<s> if ( I != nullptr ) { if ( ( I -> getOpcode ( ) == Instruction :: Load || match ( I , m_Intrinsic < <STR_LIT> > ( ) ) ) && I -> hasOneUse ( ) ) { const User * Us = * I -> users ( ) . begin ( ) ; if ( isa < ZExtInst > ( Us ) || isa < SExtInst > ( Us ) ) { unsigned TypeSize = cast < Instruction > ( Us ) -> getType ( ) -> getScalarSizeInBits ( ) ; if ( ( ( TypeSize == <NUM_LIT> && ( EltSize == <NUM_LIT> || EltSize == <NUM_LIT> ) ) || ( TypeSize == <NUM_LIT> && EltSize == <NUM_LIT> ) ) && TypeSize * NumElems == <NUM_LIT> ) { ExtSize = TypeSize ; } } } TruncInst * T ; if ( ( I -> getOpcode ( ) == Instruction :: Store || match ( I , m_Intrinsic < <STR_LIT> > ( ) ) ) && ( T = dyn_cast < TruncInst > ( I -> getOperand ( <NUM_LIT> ) ) ) ) { unsigned TypeSize = T -> getOperand ( <NUM_LIT> ) -> getType ( ) -> getScalarSizeInBits ( ) ; if ( ( ( EltSize == <NUM_LIT> && TypeSize == <NUM_LIT> ) || ( EltSize == <NUM_LIT> && ( TypeSize == <NUM_LIT> || TypeSize == <NUM_LIT> ) ) ) && TypeSize * NumElems == <NUM_LIT> ) ExtSize = TypeSize ; } } if ( ExtSize * NumElems != <NUM_LIT> || NumElems < <NUM_LIT> ) return ScalarCost ; if ( ExtSize == <NUM_LIT> ) return VectorCost ; if ( ExtSize != <NUM_LIT> && ExtSize != <NUM_LIT> ) return ScalarCost ; if ( const auto * BC = dyn_cast < BitCastInst > ( Ptr ) ) Ptr = BC -> getOperand ( <NUM_LIT> ) ; if ( const auto * GEP = dyn_cast < GetElementPtrInst > ( Ptr ) ) { if ( GEP -> getNumOperands ( ) != <NUM_LIT> ) return ScalarCost ; unsigned Scale = DL . getTypeAllocSize ( GEP -> getResultElementType ( ) ) ; if ( Scale != <NUM_LIT> && Scale * <NUM_LIT> != ExtSize ) return ScalarCost ;

if ( const auto * ZExt = dyn_cast < ZExtInst > ( GEP -> getOperand ( <NUM_LIT> ) ) ) {


<s> assert ( isa < VectorType > ( VecTy ) && <STR_LIT> ) ; bool EltIs64Bits = DL . getTypeSizeInBits ( VecTy -> getScalarType ( ) ) == <NUM_LIT> ; if ( Factor <= TLI -> getMaxSupportedInterleaveFactor ( ) && ! EltIs64Bits && ! UseMaskForCond && ! UseMaskForGaps ) { unsigned NumElts = cast < FixedVectorType > ( VecTy ) -> getNumElements ( ) ; auto * SubVecTy = FixedVectorType :: get ( VecTy -> getScalarType ( ) , NumElts / Factor ) ; int BaseCost = ST -> hasMVEIntegerOps ( ) ? ST -> getMVEVectorCostFactor ( ) : <NUM_LIT> ; if ( NumElts % Factor == <NUM_LIT> && TLI -> isLegalInterleavedAccessType ( Factor , SubVecTy , DL ) ) return Factor * BaseCost * TLI -> getNumInterleavedAccesses ( SubVecTy , DL ) ; if ( ST -> hasMVEIntegerOps ( ) && Factor == <NUM_LIT> && NumElts / Factor > <NUM_LIT> && VecTy -> isIntOrIntVectorTy ( ) && DL . getTypeSizeInBits ( SubVecTy ) . getFixedSize ( ) <= <NUM_LIT> ) return <NUM_LIT> * BaseCost ; }

return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment , AddressSpace , CostKind , UseMaskForCond , UseMaskForGaps ) ;


<s> int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; if ( ST -> isThumb ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; } if ( Opcode == Instruction :: Xor && Imm . isAllOnesValue ( ) ) return <NUM_LIT> ;

if ( Inst && ( ( ST -> hasV6Ops ( ) && ! ST -> isThumb ( ) ) || ST -> isThumb2 ( ) ) && Ty -> getIntegerBitWidth ( ) <= <NUM_LIT> ) {


<s> } if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( - Imm , Ty , CostKind ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == <NUM_LIT> ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ;

if ( ST -> isThumb ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ;


<s> case <STR_LIT> : if ( ST -> hasMVEIntegerOps ( ) ) return <NUM_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( ! ST -> hasMVEIntegerOps ( ) ) break ; Type * VT = ICA . getReturnType ( ) ; if ( ! VT -> isVectorTy ( ) && ! ICA . getVectorFactor ( ) . isScalar ( ) ) VT = VectorType :: get ( VT , ICA . getVectorFactor ( ) ) ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , VT ) ; if ( LT . second == <STR_LIT> || LT . second == <STR_LIT> || LT . second == <STR_LIT> ) { unsigned Instrs = LT . second . getScalarSizeInBits ( ) == ICA . getReturnType ( ) -> getScalarSizeInBits ( ) ? <NUM_LIT> : <NUM_LIT> ; return LT . first * ST -> getMVEVectorCostFactor ( ) * Instrs ; } break ; }


<s> if ( ! isa < FixedVectorType > ( Src ) ) return BaseT :: getMaskedMemoryOpCost ( Opcode , Src , Alignment , AddressSpace , CostKind ) ; return cast < FixedVectorType > ( Src ) -> getNumElements ( ) * <NUM_LIT> ;


<s> unsigned <ISA_LIT>TTIImpl :: getMaskedMemoryOpCost ( unsigned Opcode , Type * Src , Align Alignment , unsigned AddressSpace , TTI :: TargetCostKind CostKind ) { if ( ST -> hasMVEIntegerOps ( ) ) { if ( Opcode == Instruction :: Load && isLegalMaskedLoad ( Src , Alignment ) ) return ST -> getMVEVectorCostFactor ( ) ; if ( Opcode == Instruction :: Store && isLegalMaskedStore ( Src , Alignment ) ) return ST -> getMVEVectorCostFactor ( ) ; } if ( ! isa < FixedVectorType > ( Src ) ) return BaseT :: getMaskedMemoryOpCost ( Opcode , Src , Alignment , AddressSpace , CostKind ) ;

return cast < FixedVectorType > ( Src ) -> getNumElements ( ) * <NUM_LIT> ;


<s> int NumOps = getNumMemOps ( cast < IntrinsicInst > ( I ) ) ; if ( NumOps == - <NUM_LIT> ) return <NUM_LIT> ; return NumOps ;


<s> return LT . first * <NUM_LIT> ; } if ( ST -> hasMVEFloatOps ( ) && isa < FixedVectorType > ( Src ) && I && ( ( Opcode == Instruction :: Load && I -> hasOneUse ( ) && isa < FPExtInst > ( * I -> user_begin ( ) ) ) || ( Opcode == Instruction :: Store && isa < FPTruncInst > ( I -> getOperand ( <NUM_LIT> ) ) ) ) ) { FixedVectorType * SrcVTy = cast < FixedVectorType > ( Src ) ;

Type * DstTy = Opcode == Instruction :: Load ? ( * I -> user_begin ( ) ) -> getType ( ) : cast < Instruction > ( I -> getOperand ( <NUM_LIT> ) ) -> getOperand ( <NUM_LIT> ) -> getType ( ) ;


<s> UP . OptSizeThreshold = <NUM_LIT> ; UP . PartialOptSizeThreshold = <NUM_LIT> ; if ( L -> getHeader ( ) -> getParent ( ) -> hasOptSize ( ) ) return ; if ( ! ST -> isThumb2 ( ) ) return ; SmallVector < BasicBlock * , <NUM_LIT> > ExitingBlocks ; L -> getExitingBlocks ( ExitingBlocks ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << L -> getNumBlocks ( ) << <STR_LIT> << <STR_LIT> << ExitingBlocks . size ( ) << <STR_LIT> ) ; if ( ExitingBlocks . size ( ) > <NUM_LIT> ) return ; if ( ST -> hasBranchPredictor ( ) && L -> getNumBlocks ( ) > <NUM_LIT> ) return ; if ( getBooleanLoopAttribute ( L , <STR_LIT> ) ) return ; unsigned Cost = <NUM_LIT> ; for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( I . getType ( ) -> isVectorTy ( ) ) return ; if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { if ( const Function * F = cast < CallBase > ( I ) . getCalledFunction ( ) ) { if ( ! isLoweredToCall ( F ) ) continue ; } return ; } SmallVector < const Value * , <NUM_LIT> > Operands ( I . operand_values ( ) ) ; Cost += getUserCost ( & I , Operands , TargetTransformInfo :: TCK_SizeAndLatency ) ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ) ; UP . Partial = true ; UP . Runtime = true ; UP . UpperBound = true ; UP . UnrollRemainder = true ; UP . DefaultUnrollRuntimeCount = <NUM_LIT> ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { Align MemAlign = getKnownAlignment ( II . getArgOperand ( <NUM_LIT> ) , IC . getDataLayout ( ) , & II , & IC . getAssumptionCache ( ) , & IC . getDominatorTree ( ) ) ; unsigned AlignArg = II . getNumArgOperands ( ) - <NUM_LIT> ; Value * AlignArgOp = II . getArgOperand ( AlignArg ) ; MaybeAlign Align = cast < ConstantInt > ( AlignArgOp ) -> getMaybeAlignValue ( ) ; if ( Align && * Align < MemAlign ) { return IC . replaceOperand ( II , AlignArg , ConstantInt :: get ( Type :: getInt32Ty ( II . getContext ( ) ) , MemAlign . value ( ) , false ) ) ; } break ; } case <STR_LIT> : { Value * Arg = II . getArgOperand ( <NUM_LIT> ) ; Value * ArgArg ; if ( match ( Arg , PatternMatch :: m_Intrinsic < <STR_LIT> > ( PatternMatch :: m_Value ( ArgArg ) ) ) && II . getType ( ) == ArgArg -> getType ( ) ) { return IC . replaceInstUsesWith ( II , ArgArg ) ; } Constant * XorMask ; if ( match ( Arg , m_Xor ( PatternMatch :: m_Intrinsic < <STR_LIT> > ( PatternMatch :: m_Value ( ArgArg ) ) , PatternMatch :: m_Constant ( XorMask ) ) ) && II . getType ( ) == ArgArg -> getType ( ) ) { if ( auto * CI = dyn_cast < ConstantInt > ( XorMask ) ) { if ( CI -> getValue ( ) . trunc ( <NUM_LIT> ) . isAllOnesValue ( ) ) { auto TrueVector = IC . Builder . CreateVectorSplat ( cast < FixedVectorType > ( II . getType ( ) ) -> getNumElements ( ) , IC . Builder . getTrue ( ) ) ; return BinaryOperator :: Create ( Instruction :: Xor , ArgArg , TrueVector ) ; } } } KnownBits ScalarKnown ( <NUM_LIT> ) ; if ( IC . SimplifyDemandedBits ( & II , <NUM_LIT> , APInt :: getLowBitsSet ( <NUM_LIT> , <NUM_LIT> ) , ScalarKnown , <NUM_LIT> ) ) { return & II ; } break ; } case <STR_LIT> : { Value * Arg = II . getArgOperand ( <NUM_LIT> ) ; Value * ArgArg ; if ( match ( Arg , PatternMatch :: m_Intrinsic < <STR_LIT> > ( PatternMatch :: m_Value ( ArgArg ) ) ) ) { return IC . replaceInstUsesWith ( II , ArgArg ) ; }

if ( ! II . getMetadata ( LLVMContext :: MD_range ) ) {


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto IsHardwareLoopIntrinsic = [ ] ( Instruction & I ) { if ( auto * Call = dyn_cast < IntrinsicInst > ( & I ) ) { switch ( Call -> getIntrinsicID ( ) ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; } } return false ; } ; bool IsTailPredLoop = false ; auto ScanLoop = [ & ] ( Loop * L ) { for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( maybeLoweredToCall ( I ) || IsHardwareLoopIntrinsic ( I ) || isa < InlineAsm > ( I ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << I << <STR_LIT> ) ; return false ; } if ( auto * II = dyn_cast < IntrinsicInst > ( & I ) ) IsTailPredLoop |= II -> getIntrinsicID ( ) == <STR_LIT> || II -> getIntrinsicID ( ) == <STR_LIT> || II -> getIntrinsicID ( ) == <STR_LIT> || II -> getIntrinsicID ( ) == <STR_LIT> || II -> getIntrinsicID ( ) == <STR_LIT> ; } } return true ; } ; for ( auto Inner : * L ) if ( ! ScanLoop ( Inner ) ) return false ; if ( ! ScanLoop ( L ) ) return false ; LLVMContext & C = L -> getHeader ( ) -> getContext ( ) ; HWLoopInfo . CounterInReg = true ; HWLoopInfo . IsNestingLegal = false ; HWLoopInfo . PerformEntryTest = AllowWLSLoops && ! IsTailPredLoop ; HWLoopInfo . CountType = Type :: getInt32Ty ( C ) ;

HWLoopInfo . LoopDecrement = ConstantInt :: get ( HWLoopInfo . CountType , <NUM_LIT> ) ;


<s> if ( MinSPF == SPF_SMIN && PatternMatch :: match ( MinRHS , PatternMatch :: m_ConstantInt ( MinC ) ) && MinC -> getValue ( ) == ( ( - Imm ) - <NUM_LIT> ) ) return true ; } return false ; } ; if ( isSSatMin ( Inst -> getOperand ( <NUM_LIT> ) ) || ( Inst -> hasNUses ( <NUM_LIT> ) && ( isSSatMin ( * Inst -> user_begin ( ) ) || isSSatMin ( * ( ++ Inst -> user_begin ( ) ) ) ) ) ) return true ; }


<s> } if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty ) , getIntImmCost ( - Imm , Ty ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == <NUM_LIT> ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; if ( ST -> isThumb ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; } if ( Opcode == Instruction :: Xor && Imm . isAllOnesValue ( ) ) return <NUM_LIT> ; return getIntImmCost ( Imm , Ty ) ;


<s> return true ; } } if ( ! VT . isFloatingPoint ( ) ) return false ; if ( TLI -> useSoftFloat ( ) ) { switch ( I . getOpcode ( ) ) { default : return true ; case Instruction :: Alloca : case Instruction :: Load : case Instruction :: Store : case Instruction :: Select : case Instruction :: PHI : return false ; } } if ( I . getType ( ) -> isDoubleTy ( ) && ! ST -> hasFP64 ( ) ) return true ; if ( I . getType ( ) -> isHalfTy ( ) && ! ST -> hasFullFP16 ( ) ) return true ; return false ; } ; auto IsHardwareLoopIntrinsic = [ ] ( Instruction & I ) { if ( auto * Call = dyn_cast < IntrinsicInst > ( & I ) ) { switch ( Call -> getIntrinsicID ( ) ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; } } return false ; } ; auto ScanLoop = [ & ] ( Loop * L ) { for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( MaybeCall ( I ) || IsHardwareLoopIntrinsic ( I ) ) return false ; } } return true ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( F -> getReturnType ( ) -> isDoubleTy ( ) && ! ST -> hasFP64 ( ) ) return true ; if ( F -> getReturnType ( ) -> isHalfTy ( ) && ! ST -> hasFullFP16 ( ) ) return true ; return ! ST -> hasFP<ISA_LIT>v8Base ( ) && ! ST -> hasVFP2Base ( ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return ! ST -> hasMVEIntegerOps ( ) ; case <STR_LIT> :


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( F -> getReturnType ( ) -> isDoubleTy ( ) && ! ST -> hasFP64 ( ) ) return true ; if ( F -> getReturnType ( ) -> isHalfTy ( ) && ! ST -> hasFullFP16 ( ) ) return true ; return ! ST -> hasFP<ISA_LIT>v8Base ( ) && ! ST -> hasVFP2Base ( ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> int <ISA_LIT>TTIImpl :: getInterleavedMemoryOpCost ( unsigned Opcode , Type * VecTy , unsigned Factor , ArrayRef < unsigned > Indices , unsigned Alignment , unsigned AddressSpace ) { assert ( Factor >= <NUM_LIT> && <STR_LIT> ) ; assert ( isa < VectorType > ( VecTy ) && <STR_LIT> ) ; bool EltIs64Bits = DL . getTypeSizeInBits ( VecTy -> getScalarType ( ) ) == <NUM_LIT> ; if ( Factor <= TLI -> getMaxSupportedInterleaveFactor ( ) && ! EltIs64Bits ) { unsigned NumElts = VecTy -> getVectorNumElements ( ) ; Type * SubVecTy = VectorType :: get ( VecTy -> getScalarType ( ) , NumElts / Factor ) ; unsigned SubVecSize = DL . getTypeSizeInBits ( SubVecTy ) ; if ( NumElts % Factor == <NUM_LIT> && ( SubVecSize == <NUM_LIT> || SubVecSize == <NUM_LIT> ) && ! VecTy -> getScalarType ( ) -> isHalfTy ( ) ) return Factor ;


<s> if ( ST -> hasBranchPredictor ( ) && L -> getNumBlocks ( ) > <NUM_LIT> ) return ; if ( getBooleanLoopAttribute ( L , <STR_LIT> ) ) return ; unsigned Cost = <NUM_LIT> ; for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( I . getType ( ) -> isVectorTy ( ) ) return ; if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { if ( const Function * F = cast < CallBase > ( I ) . getCalledFunction ( ) ) { if ( ! isLoweredToCall ( F ) ) continue ; } return ;


<s> for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( I . getType ( ) -> isVectorTy ( ) ) return ; if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { if ( const Function * F = cast < CallBase > ( I ) . getCalledFunction ( ) ) { if ( ! isLoweredToCall ( F ) ) continue ; } return ; } SmallVector < const Value * , <NUM_LIT> > Operands ( I . value_op_begin ( ) , I . value_op_end ( ) ) ; Cost += getUserCost ( & I , Operands , TargetTransformInfo :: TCK_SizeAndLatency ) ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ) ; UP . Partial = true ; UP . Runtime = true ; UP . UpperBound = true ; UP . UnrollRemainder = true ; UP . DefaultUnrollRuntimeCount = <NUM_LIT> ;


<s> bool isLegalMaskedGather ( Type * Ty , MaybeAlign Alignment ) {


<s> <ISA_LIT>TTI ( ) : ImmutablePass ( ID ) , TM ( nullptr ) , ST ( nullptr ) , TLI ( nullptr ) { llvm_unreachable ( <STR_LIT> ) ;


<s> if ( ! ST -> hasMVEFloatOps ( ) ) break ; Type * VT = ICA . getReturnType ( ) ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , VT ) ; if ( LT . second == <STR_LIT> || LT . second == <STR_LIT> ) return LT . first * ST -> getMVEVectorCostFactor ( CostKind ) ; break ; } case <STR_LIT> : case <STR_LIT> : { if ( ICA . getArgTypes ( ) . empty ( ) ) break ; bool IsSigned = ICA . getID ( ) == <STR_LIT> ; auto LT = TLI -> getTypeLegalizationCost ( DL , ICA . getArgTypes ( ) [ <NUM_LIT> ] ) ; EVT MTy = TLI -> getValueType ( DL , ICA . getReturnType ( ) ) ; if ( ( ST -> hasVFP2Base ( ) && LT . second == <STR_LIT> && MTy == <STR_LIT> ) || ( ST -> hasFP64 ( ) && LT . second == <STR_LIT> && MTy == <STR_LIT> ) || ( ST -> hasFullFP16 ( ) && LT . second == <STR_LIT> && MTy == <STR_LIT> ) ) return LT . first ; if ( ST -> hasMVEFloatOps ( ) && ( LT . second == <STR_LIT> || LT . second == <STR_LIT> ) && LT . second . getScalarSizeInBits ( ) == MTy . getScalarSizeInBits ( ) ) return LT . first * ST -> getMVEVectorCostFactor ( CostKind ) ; if ( ( ( ST -> hasVFP2Base ( ) && LT . second == <STR_LIT> ) || ( ST -> hasFP64 ( ) && LT . second == <STR_LIT> ) || ( ST -> hasFullFP16 ( ) && LT . second == <STR_LIT> ) || ( ST -> hasMVEFloatOps ( ) && ( LT . second == <STR_LIT> || LT . second == <STR_LIT> ) ) ) && LT . second . getScalarSizeInBits ( ) >= MTy . getScalarSizeInBits ( ) ) { Type * LegalTy = Type :: getIntNTy ( ICA . getReturnType ( ) -> getContext ( ) , LT . second . getScalarSizeInBits ( ) ) ; InstructionCost Cost = LT . second . isVector ( ) ? ST -> getMVEVectorCostFactor ( CostKind ) : <NUM_LIT> ; IntrinsicCostAttributes Attrs1 ( IsSigned ? <STR_LIT> : <STR_LIT> , LegalTy , { LegalTy , LegalTy } ) ; Cost += getIntrinsicInstrCost ( Attrs1 , CostKind ) ; IntrinsicCostAttributes Attrs2 ( IsSigned ? <STR_LIT> : <STR_LIT> , LegalTy , { LegalTy , LegalTy } ) ; Cost += getIntrinsicInstrCost ( Attrs2 , CostKind ) ; return LT . first * Cost ; } break ;


<s> std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( NEONDupTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost ; } if ( Kind == TTI :: SK_Reverse ) { static const CostTblEntry NEONShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( NEONShuffleTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost ; } if ( Kind == TTI :: SK_Select ) { static const CostTblEntry NEONSelShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( NEONSelShuffleTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost ; } } if ( ST -> hasMVEIntegerOps ( ) ) { if ( Kind == TTI :: SK_Broadcast ) { static const CostTblEntry MVEDupTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( MVEDupTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost * ST -> getMVEVectorCostFactor ( TTI :: TCK_RecipThroughput ) ;


<s> if ( ST -> hasMVEIntegerOps ( ) && ICA . getID ( ) == <STR_LIT> ) return <NUM_LIT> ; return BaseT :: getIntrinsicInstrCost ( ICA , CostKind ) ;


<s> UP . PartialOptSizeThreshold = <NUM_LIT> ; if ( L -> getHeader ( ) -> getParent ( ) -> hasOptSize ( ) ) return ; if ( ! ST -> isThumb2 ( ) ) return ; SmallVector < BasicBlock * , <NUM_LIT> > ExitingBlocks ; L -> getExitingBlocks ( ExitingBlocks ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << L -> getNumBlocks ( ) << <STR_LIT> << <STR_LIT> << ExitingBlocks . size ( ) << <STR_LIT> ) ; if ( ExitingBlocks . size ( ) > <NUM_LIT> ) return ; if ( ST -> hasBranchPredictor ( ) && L -> getNumBlocks ( ) > <NUM_LIT> ) return ; unsigned Cost = <NUM_LIT> ; for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( I . getType ( ) -> isVectorTy ( ) ) return ; if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { if ( const Function * F = cast < CallBase > ( I ) . getCalledFunction ( ) ) { if ( ! isLoweredToCall ( F ) ) continue ; } return ;


<s> break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; } } return false ; } ; auto ScanLoop = [ & ] ( Loop * L ) { for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( maybeLoweredToCall ( I ) || IsHardwareLoopIntrinsic ( I ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << I << <STR_LIT> ) ; return false ; } } } return true ; } ; for ( auto Inner : * L ) if ( ! ScanLoop ( Inner ) ) return false ; if ( ! ScanLoop ( L ) ) return false ; LLVMContext & C = L -> getHeader ( ) -> getContext ( ) ; HWLoopInfo . CounterInReg = true ; HWLoopInfo . IsNestingLegal = false ; HWLoopInfo . PerformEntryTest = true ;

HWLoopInfo . CountType = Type :: getInt32Ty ( C ) ;


<s> if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; unsigned ScalarBits = Ty -> getScalarSizeInBits ( ) ; switch ( Opcode ) { case Instruction :: Add : return ScalarBits <= <NUM_LIT> ; default : return false ;


<s> } PredicatedScalarEvolution PSE = LAI -> getPSE ( ) ; SmallVector < Instruction * , <NUM_LIT> > LoadStores ; int ICmpCount = <NUM_LIT> ; int Stride = <NUM_LIT> ; for ( BasicBlock * BB : L -> blocks ( ) ) { for ( Instruction & I : BB -> instructionsWithoutDebug ( ) ) { if ( isa < PHINode > ( & I ) ) continue ; if ( ! canTailPredicateInstruction ( I , ICmpCount ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; I . dump ( ) ) ; return false ; } Type * T = I . getType ( ) ; if ( T -> isPointerTy ( ) ) T = T -> getPointerElementType ( ) ; if ( T -> getScalarSizeInBits ( ) > <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; T -> dump ( ) ) ; return false ;


<s> if ( ( ( EltSize == <NUM_LIT> && TypeSize == <NUM_LIT> ) || ( EltSize == <NUM_LIT> && ( TypeSize == <NUM_LIT> || TypeSize == <NUM_LIT> ) ) ) && TypeSize * NumElems == <NUM_LIT> ) ExtSize = TypeSize ; } } if ( ExtSize * NumElems != <NUM_LIT> || NumElems < <NUM_LIT> ) return ScalarCost ; if ( ExtSize == <NUM_LIT> ) return VectorCost ; if ( ExtSize != <NUM_LIT> && ExtSize != <NUM_LIT> ) return ScalarCost ; if ( const auto * BC = dyn_cast < BitCastInst > ( Ptr ) ) Ptr = BC -> getOperand ( <NUM_LIT> ) ; if ( const auto * GEP = dyn_cast < GetElementPtrInst > ( Ptr ) ) { if ( GEP -> getNumOperands ( ) != <NUM_LIT> ) return ScalarCost ; unsigned Scale = DL . getTypeAllocSize ( GEP -> getResultElementType ( ) ) ; if ( Scale != <NUM_LIT> && Scale * <NUM_LIT> != ExtSize ) return ScalarCost ; if ( const auto * ZExt = dyn_cast < ZExtInst > ( GEP -> getOperand ( <NUM_LIT> ) ) ) { if ( ZExt -> getOperand ( <NUM_LIT> ) -> getType ( ) -> getScalarSizeInBits ( ) <= ExtSize ) return VectorCost ; } return ScalarCost ;


<s> if ( NumElts % Factor == <NUM_LIT> && TLI -> isLegalInterleavedAccessType ( Factor , SubVecTy , DL ) ) return Factor * BaseCost * TLI -> getNumInterleavedAccesses ( SubVecTy , DL ) ; if ( ST -> hasMVEIntegerOps ( ) && Factor == <NUM_LIT> && NumElts / Factor > <NUM_LIT> && VecTy -> isIntOrIntVectorTy ( ) && DL . getTypeSizeInBits ( SubVecTy ) <= <NUM_LIT> ) return <NUM_LIT> * BaseCost ; } return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment , AddressSpace , CostKind , UseMaskForCond , UseMaskForGaps ) ;


<s> int BaseCost = ST -> hasMVEIntegerOps ( ) ? ST -> getMVEVectorCostFactor ( ) : <NUM_LIT> ; if ( NumElts % Factor == <NUM_LIT> && TLI -> isLegalInterleavedAccessType ( Factor , SubVecTy , DL ) ) return Factor * BaseCost * TLI -> getNumInterleavedAccesses ( SubVecTy , DL ) ; if ( ST -> hasMVEIntegerOps ( ) && Factor == <NUM_LIT> && NumElts / Factor > <NUM_LIT> && VecTy -> isIntOrIntVectorTy ( ) && DL . getTypeSizeInBits ( SubVecTy ) <= <NUM_LIT> ) return <NUM_LIT> * BaseCost ;


<s> <STR_LIT> IID = II . getIntrinsicID ( ) ; switch ( IID ) { default : break ; case <STR_LIT> : { Align MemAlign = getKnownAlignment ( II . getArgOperand ( <NUM_LIT> ) , IC . getDataLayout ( ) , & II , & IC . getAssumptionCache ( ) , & IC . getDominatorTree ( ) ) ; if ( Value * V = simplifyNeonVld1 ( II , MemAlign . value ( ) , IC . Builder ) ) { return IC . replaceInstUsesWith ( II , V ) ; } break ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { Align MemAlign = getKnownAlignment ( II . getArgOperand ( <NUM_LIT> ) , IC . getDataLayout ( ) , & II , & IC . getAssumptionCache ( ) , & IC . getDominatorTree ( ) ) ; unsigned AlignArg = II . getNumArgOperands ( ) - <NUM_LIT> ; Value * AlignArgOp = II . getArgOperand ( AlignArg ) ; MaybeAlign Align = cast < ConstantInt > ( AlignArgOp ) -> getMaybeAlignValue ( ) ; if ( Align && * Align < MemAlign ) { return IC . replaceOperand ( II , AlignArg , ConstantInt :: get ( Type :: getInt32Ty ( II . getContext ( ) ) , MemAlign . value ( ) , false ) ) ; }


<s> if ( ! EnableTailPredication ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; if ( L -> getNumBlocks ( ) > <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } assert ( L -> empty ( ) && <STR_LIT> ) ; HardwareLoopInfo HWLoopInfo ( L ) ; if ( ! HWLoopInfo . canAnalyze ( * LI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } if ( ! isHardwareLoopProfitable ( L , SE , AC , TLI , HWLoopInfo ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ;


<s> if ( ( LT . second == <STR_LIT> && ResVT . getSizeInBits ( ) <= <NUM_LIT> ) || ( LT . second == <STR_LIT> && ResVT . getSizeInBits ( ) <= ( IsMLA ? <NUM_LIT> : <NUM_LIT> ) ) || ( LT . second == <STR_LIT> && ResVT . getSizeInBits ( ) <= <NUM_LIT> ) ) return ST -> getMVEVectorCostFactor ( CostKind ) * LT . first ; } return BaseT :: getExtendedAddReductionCost ( IsMLA , IsUnsigned , ResTy , ValTy , CostKind ) ;


<s> EVT ResVT = TLI -> getValueType ( DL , ResTy ) ; if ( ST -> hasMVEIntegerOps ( ) && ValVT . isSimple ( ) && ResVT . isSimple ( ) ) { std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ;

if ( ( LT . second == <STR_LIT> && ResVT . getSizeInBits ( ) <= <NUM_LIT> ) || ( LT . second == <STR_LIT> && ResVT . getSizeInBits ( ) <= ( IsMLA ? <NUM_LIT> : <NUM_LIT> ) ) || ( LT . second == <STR_LIT> && ResVT . getSizeInBits ( ) <= <NUM_LIT> ) ) return ST -> getMVEVectorCostFactor ( CostKind ) * LT . first ;


<s> if ( Imm == <NUM_LIT> || Imm == <NUM_LIT> ) return <NUM_LIT> ; return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( ~ Imm , Ty , CostKind ) ) ; } if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( - Imm , Ty , CostKind ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == <NUM_LIT> ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ;


<s> if ( ST -> isThumb2 ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; if ( ST -> isThumb ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; } if ( Opcode == Instruction :: Xor && Imm . isAllOnesValue ( ) ) return <NUM_LIT> ; if ( Inst && ( ( ST -> hasV6Ops ( ) && ! ST -> isThumb ( ) ) || ST -> isThumb2 ( ) ) && Ty -> getIntegerBitWidth ( ) <= <NUM_LIT> ) { if ( isSSATMinMaxPattern ( Inst , Imm ) || ( isa < ICmpInst > ( Inst ) && Inst -> hasOneUse ( ) && isSSATMinMaxPattern ( cast < Instruction > ( * Inst -> user_begin ( ) ) , Imm ) ) ) return <NUM_LIT> ; }

return getIntImmCost ( Imm , Ty , CostKind ) ;


<s> unsigned getMaxInterleaveFactor ( unsigned VF ) { if ( ST -> isCortexA15 ( ) || ST -> isSwift ( ) ) return <NUM_LIT> ; return <NUM_LIT> ;


<s> unsigned SrcAddrSpace = ~ <NUM_LIT> ; const Function * F = I -> getParent ( ) -> getParent ( ) ; if ( const auto * MC = dyn_cast < MemTransferInst > ( I ) ) { ConstantInt * C = dyn_cast < ConstantInt > ( MC -> getLength ( ) ) ; if ( ! C ) return - <NUM_LIT> ; const unsigned Size = C -> getValue ( ) . getZExtValue ( ) ; const Align DstAlign = * MC -> getDestAlign ( ) ; const Align SrcAlign = * MC -> getSourceAlign ( ) ; MOp = MemOp :: Copy ( Size , false , DstAlign , SrcAlign , false , false ) ; DstAddrSpace = MC -> getDestAddressSpace ( ) ; SrcAddrSpace = MC -> getSourceAddressSpace ( ) ; } else if ( const auto * MS = dyn_cast < MemSetInst > ( I ) ) { ConstantInt * C = dyn_cast < ConstantInt > ( MS -> getLength ( ) ) ; if ( ! C ) return - <NUM_LIT> ; const unsigned Size = C -> getValue ( ) . getZExtValue ( ) ; const Align DstAlign = * MS -> getDestAlign ( ) ; MOp = MemOp :: Set ( Size , false , DstAlign , false , false ) ; DstAddrSpace = MS -> getDestAddressSpace ( ) ; }


<s> unsigned getRegisterBitWidthUpperBound ( bool Vector ) { return getRegisterBitWidth ( Vector ) ;


<s> SmallVector < const Value * , <NUM_LIT> > Operands ( I . operand_values ( ) ) ; Cost += getUserCost ( & I , Operands , TargetTransformInfo :: TCK_SizeAndLatency ) ; } } unsigned UnrollCount = <NUM_LIT> ; if ( ST -> isThumb1Only ( ) ) { unsigned ExitingValues = <NUM_LIT> ; SmallVector < BasicBlock * , <NUM_LIT> > ExitBlocks ; L -> getExitBlocks ( ExitBlocks ) ; for ( auto * Exit : ExitBlocks ) { unsigned LiveOuts = count_if ( Exit -> phis ( ) , [ ] ( auto & PH ) { return PH . getNumOperands ( ) != <NUM_LIT> || ! isa < GetElementPtrInst > ( PH . getOperand ( <NUM_LIT> ) ) ; } ) ; ExitingValues = ExitingValues < LiveOuts ? LiveOuts : ExitingValues ; } if ( ExitingValues ) UnrollCount /= ExitingValues ; if ( UnrollCount <= <NUM_LIT> ) return ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << UnrollCount << <STR_LIT> ) ; UP . Partial = true ; UP . Runtime = true ; UP . UnrollRemainder = true ; UP . DefaultUnrollRuntimeCount = UnrollCount ; UP . UnrollAndJam = true ; UP . UnrollAndJamInnerLoopThreshold = <NUM_LIT> ;

if ( Cost < <NUM_LIT> ) UP . Force = true ;


<s> UP . UpperBound = true ; if ( ! ST -> isMClass ( ) ) return BasicTTIImplBase :: getUnrollingPreferences ( L , SE , UP ) ; UP . OptSizeThreshold = <NUM_LIT> ; UP . PartialOptSizeThreshold = <NUM_LIT> ; if ( L -> getHeader ( ) -> getParent ( ) -> hasOptSize ( ) ) return ; SmallVector < BasicBlock * , <NUM_LIT> > ExitingBlocks ; L -> getExitingBlocks ( ExitingBlocks ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << L -> getNumBlocks ( ) << <STR_LIT> << <STR_LIT> << ExitingBlocks . size ( ) << <STR_LIT> ) ; if ( ExitingBlocks . size ( ) > <NUM_LIT> ) return ; if ( ST -> hasBranchPredictor ( ) && L -> getNumBlocks ( ) > <NUM_LIT> ) return ; if ( getBooleanLoopAttribute ( L , <STR_LIT> ) ) return ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { Align MemAlign = getKnownAlignment ( II . getArgOperand ( <NUM_LIT> ) , IC . getDataLayout ( ) , & II , & IC . getAssumptionCache ( ) , & IC . getDominatorTree ( ) ) ; unsigned AlignArg = II . getNumArgOperands ( ) - <NUM_LIT> ; Value * AlignArgOp = II . getArgOperand ( AlignArg ) ; MaybeAlign Align = cast < ConstantInt > ( AlignArgOp ) -> getMaybeAlignValue ( ) ; if ( Align && * Align < MemAlign ) { return IC . replaceOperand ( II , AlignArg , ConstantInt :: get ( Type :: getInt32Ty ( II . getContext ( ) ) , MemAlign . value ( ) , false ) ) ; } break ; } case <STR_LIT> : { Value * Arg = II . getArgOperand ( <NUM_LIT> ) ; Value * ArgArg ; if ( match ( Arg , PatternMatch :: m_Intrinsic < <STR_LIT> > ( PatternMatch :: m_Value ( ArgArg ) ) ) && II . getType ( ) == ArgArg -> getType ( ) ) { return IC . replaceInstUsesWith ( II , ArgArg ) ; } Constant * XorMask ; if ( match ( Arg , m_Xor ( PatternMatch :: m_Intrinsic < <STR_LIT> > ( PatternMatch :: m_Value ( ArgArg ) ) , PatternMatch :: m_Constant ( XorMask ) ) ) && II . getType ( ) == ArgArg -> getType ( ) ) { if ( auto * CI = dyn_cast < ConstantInt > ( XorMask ) ) { if ( CI -> getValue ( ) . trunc ( <NUM_LIT> ) . isAllOnesValue ( ) ) { auto TrueVector = IC . Builder . CreateVectorSplat ( cast < FixedVectorType > ( II . getType ( ) ) -> getNumElements ( ) , IC . Builder . getTrue ( ) ) ; return BinaryOperator :: Create ( Instruction :: Xor , ArgArg , TrueVector ) ; } } } KnownBits ScalarKnown ( <NUM_LIT> ) ; if ( IC . SimplifyDemandedBits ( & II , <NUM_LIT> , APInt :: getLowBitsSet ( <NUM_LIT> , <NUM_LIT> ) , ScalarKnown , <NUM_LIT> ) ) { return & II ; } break ; } case <STR_LIT> : { Value * Arg = II . getArgOperand ( <NUM_LIT> ) ; Value * ArgArg ; if ( match ( Arg , PatternMatch :: m_Intrinsic < <STR_LIT> > ( PatternMatch :: m_Value ( ArgArg ) ) ) ) { return IC . replaceInstUsesWith ( II , ArgArg ) ; }


<s> explicit <ISA_LIT>TTIImpl ( const <ISA_LIT>BaseTargetMachine * TM , Function & F ) : BaseT ( TM , F . getParent ( ) -> getDataLayout ( ) ) , ST ( TM -> getSubtargetImpl ( F ) ) , TLI ( ST -> getTargetLowering ( ) ) {


<s> int <ISA_LIT>TTIImpl :: getCmpSelInstrCost ( unsigned Opcode , Type * ValTy , Type * CondTy , TTI :: TargetCostKind CostKind , const Instruction * I ) { int ISD = TLI -> InstructionOpcodeToISD ( Opcode ) ; if ( CostKind == TTI :: TCK_CodeSize && ISD == <STR_LIT> && ST -> isThumb ( ) && ! ValTy -> isVectorTy ( ) ) { if ( TLI -> getValueType ( DL , ValTy , true ) == <STR_LIT> ) return TTI :: TCC_Expensive ; int Cost = TLI -> getTypeLegalizationCost ( DL , ValTy ) . first ; ++ Cost ; if ( ValTy -> isIntegerTy ( <NUM_LIT> ) ) ++ Cost ;


<s> if ( ValTy -> isIntegerTy ( <NUM_LIT> ) ) ++ Cost ; return Cost ; } if ( CostKind != TTI :: TCK_RecipThroughput ) return BaseT :: getCmpSelInstrCost ( Opcode , ValTy , CondTy , CostKind , I ) ; if ( ST -> hasNEON ( ) && ValTy -> isVectorTy ( ) && ISD == <STR_LIT> ) { static const TypeConversionCostTblEntry NEONVectorSelectTbl [ ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; EVT SelCondTy = TLI -> getValueType ( DL , CondTy ) ; EVT SelValTy = TLI -> getValueType ( DL , ValTy ) ; if ( SelCondTy . isSimple ( ) && SelValTy . isSimple ( ) ) { if ( const auto * Entry = ConvertCostTableLookup ( NEONVectorSelectTbl , ISD , SelCondTy . getSimpleVT ( ) , SelValTy . getSimpleVT ( ) ) ) return Entry -> Cost ; } std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ; return LT . first ;


<s> ExtSize = TypeSize ; } } } TruncInst * T ; if ( ( I -> getOpcode ( ) == Instruction :: Store || match ( I , m_Intrinsic < <STR_LIT> > ( ) ) ) && ( T = dyn_cast < TruncInst > ( I -> getOperand ( <NUM_LIT> ) ) ) ) { unsigned TypeSize = T -> getOperand ( <NUM_LIT> ) -> getType ( ) -> getScalarSizeInBits ( ) ; if ( ( ( EltSize == <NUM_LIT> && TypeSize == <NUM_LIT> ) || ( EltSize == <NUM_LIT> && ( TypeSize == <NUM_LIT> || TypeSize == <NUM_LIT> ) ) ) && TypeSize * NumElems == <NUM_LIT> ) ExtSize = TypeSize ; } } if ( ExtSize * NumElems != <NUM_LIT> || NumElems < <NUM_LIT> ) return ScalarCost ; if ( ExtSize == <NUM_LIT> ) return VectorCost ; if ( ExtSize != <NUM_LIT> && ExtSize != <NUM_LIT> ) return ScalarCost ; if ( const auto * BC = dyn_cast < BitCastInst > ( Ptr ) ) Ptr = BC -> getOperand ( <NUM_LIT> ) ; if ( const auto * GEP = dyn_cast < GetElementPtrInst > ( Ptr ) ) { if ( GEP -> getNumOperands ( ) != <NUM_LIT> ) return ScalarCost ;

unsigned Scale = DL . getTypeAllocSize ( GEP -> getResultElementType ( ) ) ;


<s> } return true ; } switch ( I . getOpcode ( ) ) { default : break ; case Instruction :: FPToSI : case Instruction :: FPToUI : case Instruction :: SIToFP : case Instruction :: UIToFP : case Instruction :: FPTrunc : case Instruction :: FPExt : return ! ST -> hasFP<ISA_LIT>v8Base ( ) ; } if ( VT . isInteger ( ) && VT . getSizeInBits ( ) >= <NUM_LIT> ) { switch ( ISD ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; } } if ( ! VT . isFloatingPoint ( ) ) return false ; if ( TLI -> useSoftFloat ( ) ) { switch ( I . getOpcode ( ) ) { default : return true ; case Instruction :: Alloca : case Instruction :: Load : case Instruction :: Store : case Instruction :: Select : case Instruction :: PHI : return false ; } } if ( I . getType ( ) -> isDoubleTy ( ) && ! ST -> hasFP64 ( ) ) return true ; if ( I . getType ( ) -> isHalfTy ( ) && ! ST -> hasFullFP16 ( ) ) return true ;


<s> if ( TLI -> getOperationAction ( ISD , VT ) == TargetLowering :: LibCall ) return true ; if ( auto * Call = dyn_cast < CallInst > ( & I ) ) { if ( isa < IntrinsicInst > ( Call ) ) { if ( const Function * F = Call -> getCalledFunction ( ) ) return isLoweredToCall ( F ) ; } return true ; } switch ( I . getOpcode ( ) ) { default : break ; case Instruction :: FPToSI : case Instruction :: FPToUI : case Instruction :: SIToFP : case Instruction :: UIToFP : case Instruction :: FPTrunc : case Instruction :: FPExt : return ! ST -> hasFP<ISA_LIT>v8Base ( ) ; } if ( VT . isInteger ( ) && VT . getSizeInBits ( ) >= <NUM_LIT> ) { switch ( ISD ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; }


<s> { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , } ; if ( const auto * Entry = CostTableLookup ( CostTbl , ISDOpcode , LT . second ) ) return LT . first * Entry -> Cost ; int Cost = BaseT :: getArithmeticInstrCost ( Opcode , Ty , Op1Info , Op2Info , Opd1PropInfo , Opd2PropInfo ) ; if ( LT . second == <STR_LIT> && Op2Info == TargetTransformInfo :: OK_UniformConstantValue ) Cost += <NUM_LIT> ; return Cost ; }


<s> if ( NumElts % Factor == <NUM_LIT> && TLI -> isLegalInterleavedAccessType ( Factor , SubVecTy , DL ) ) return Factor * BaseCost * TLI -> getNumInterleavedAccesses ( SubVecTy , DL ) ; if ( ST -> hasMVEIntegerOps ( ) && Factor == <NUM_LIT> && NumElts / Factor > <NUM_LIT> && VecTy -> isIntOrIntVectorTy ( ) && DL . getTypeSizeInBits ( SubVecTy ) <= <NUM_LIT> ) return <NUM_LIT> * BaseCost ; }

return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment , AddressSpace , UseMaskForCond , UseMaskForGaps ) ;


<s> unsigned NumElts = cast < VectorType > ( VecTy ) -> getNumElements ( ) ; auto * SubVecTy = VectorType :: get ( VecTy -> getScalarType ( ) , NumElts / Factor ) ; int BaseCost = ST -> hasMVEIntegerOps ( ) ? ST -> getMVEVectorCostFactor ( ) : <NUM_LIT> ;

if ( NumElts % Factor == <NUM_LIT> && TLI -> isLegalInterleavedAccessType ( Factor , SubVecTy , DL ) ) return Factor * BaseCost * TLI -> getNumInterleavedAccesses ( SubVecTy , DL ) ;


<s> std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Src ) ; if ( ST -> hasNEON ( ) && Src -> isVectorTy ( ) && ( Alignment && * Alignment != Align ( <NUM_LIT> ) ) && cast < VectorType > ( Src ) -> getElementType ( ) -> isDoubleTy ( ) ) {


<s> if ( ST -> hasNEON ( ) && ( Opcode == Instruction :: InsertElement || Opcode == Instruction :: ExtractElement ) ) { if ( cast < VectorType > ( ValTy ) -> getElementType ( ) -> isIntegerTy ( ) ) return <NUM_LIT> ; if ( ValTy -> isVectorTy ( ) && ValTy -> getScalarSizeInBits ( ) <= <NUM_LIT> ) return std :: max ( BaseT :: getVectorInstrCost ( Opcode , ValTy , Index ) , <NUM_LIT> ) ; } if ( ST -> hasMVEIntegerOps ( ) && ( Opcode == Instruction :: InsertElement || Opcode == Instruction :: ExtractElement ) ) { return std :: max ( BaseT :: getVectorInstrCost ( Opcode , ValTy , Index ) , ST -> getMVEVectorCostFactor ( ) ) * cast < VectorType > ( ValTy ) -> getNumElements ( ) / <NUM_LIT> ;


<s> case Instruction :: Xor : case Instruction :: Mul : case Instruction :: FCmp : return false ; case Instruction :: ICmp : case Instruction :: Add : return ScalarBits < <NUM_LIT> && ( ScalarBits * cast < VectorType > ( Ty ) -> getNumElements ( ) ) % <NUM_LIT> == <NUM_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ; } return false ;


<s> case Instruction :: FCmp : return false ; case Instruction :: ICmp : case Instruction :: Add : return ScalarBits < <NUM_LIT> && ( ScalarBits * cast < VectorType > ( Ty ) -> getNumElements ( ) ) % <NUM_LIT> == <NUM_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ; }


<s> assert ( Ty -> isIntegerTy ( ) ) ; unsigned Bits = Ty -> getPrimitiveSizeInBits ( ) ; if ( Bits == <NUM_LIT> || Bits > <NUM_LIT> ) return <NUM_LIT> ; int64_t SImmVal = Imm . getSExtValue ( ) ; uint64_t ZImmVal = Imm . getZExtValue ( ) ; if ( ! ST -> isThumb ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ;

return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ;


<s> if ( ! ST -> isThumb ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ; } if ( ST -> isThumb2 ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ;


<s> <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; EVT SelCondTy = TLI -> getValueType ( DL , CondTy ) ; EVT SelValTy = TLI -> getValueType ( DL , ValTy ) ; if ( SelCondTy . isSimple ( ) && SelValTy . isSimple ( ) ) { if ( const auto * Entry = ConvertCostTableLookup ( NEONVectorSelectTbl , ISD , SelCondTy . getSimpleVT ( ) , SelValTy . getSimpleVT ( ) ) ) return Entry -> Cost ; } std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ; return LT . first ; } return BaseT :: getCmpSelInstrCost ( Opcode , ValTy , CondTy ) ;


<s> if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { ImmutableCallSite CS ( & I ) ; if ( const Function * F = CS . getCalledFunction ( ) ) { if ( ! isLoweredToCall ( F ) ) continue ; } return ; } SmallVector < const Value * , <NUM_LIT> > Operands ( I . value_op_begin ( ) , I . value_op_end ( ) ) ; Cost += getUserCost ( & I , Operands ) ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ) ; UP . Partial = true ;