<s> unsigned getNumberOfRegisters ( unsigned ClassID ) const { bool Vector = ( ClassID == 1 ) ; if ( Vector ) { if ( ST -> hasNEON ( ) ) return 16 ; if ( ST -> hasMVEIntegerOps ( ) ) return 8 ; return 0 ; } if ( ST -> isThumb1Only ( ) ) return 8 ; return 13 ; } </s>


<s> const ARMTargetLowering * getTLI ( ) const { return TLI ; } </s>

Getter for generic TargetLowering class .


<s>
void
ARMTTIImpl
::
getUnrollingPreferences
(
Loop
*
L
,
ScalarEvolution
&
SE
,
TTI
::
UnrollingPreferences
&
UP
)
{
if
(
!
ST
->
isMClass
(
)
)
return
BasicTTIImplBase
::
getUnrollingPreferences
(
L
,
SE
,
UP
)
;
UP
.
OptSizeThreshold
=
0
;
UP
.
PartialOptSizeThreshold
=
0
;
if
(
L
->
getHeader
(
)
->
getParent
(
)
->
optForSize
(
)
)
return
;
if
(
!
ST
->
isThumb2
(
)
)
return
;
SmallVector
<
BasicBlock
*
,
4
>
ExitingBlocks
;
L
->
getExitingBlocks
(
ExitingBlocks
)
;
LLVM_DEBUG
(
dbgs
(
)
<<
"Loop has:\n"
<<
"Blocks: "
<<
L
->
getNumBlocks
(
)
<<
"\n"
<<
"Exit blocks: "
<<
ExitingBlocks
.
size
(
)
<<
"\n"
)
;
if
(
ExitingBlocks
.
size
(
)
>
2
)
return
;
if
(
ST
->
hasBranchPredictor
(
)
&&
L
->
getNumBlocks
(
)
>
4
)
return
;
unsigned
Cost
=
0
;
for
(
auto
*
BB
:
L
->
getBlocks
(
)
)
{
for
(
auto
&
I
:
*
BB
)
{
if
(
isa
<
CallInst
>
(
I
)
||
isa
<
InvokeInst
>
(
I
)
)
{
ImmutableCallSite
CS
(
&
I
)
;
if
(
const
Function
*
F
=
CS
.
getCalledFunction
(
)
)
{
if
(
!
isLoweredToCall
(
F
)
)
continue
;
}
return
;
}
SmallVector
<
const
Value
*
,
4
>
Operands
(
I
.
value_op_begin
(
)
,
I
.
value_op_end
(
)
)
;
Cost
+=
getUserCost
(
&
I
,
Operands
)
;
}
}
LLVM_DEBUG
(
dbgs
(
)
<<
"Cost of loop: "
<<
Cost
<<
"\n"
)
;
UP
.
Partial
=
true
;
UP
.
Runtime
=
true
;
UP
.
UnrollRemainder
=
true
;
UP
.
DefaultUnrollRuntimeCount
=
4
;
UP
.
UnrollAndJam
=
true
;
UP
.
UnrollAndJamInnerLoopThreshold
=
60
;
if
(
Cost
<
12
)
UP
.
Force
=
true
;
}
</s>

Get target-customized preferences for the generic loop unrolling transformation .


<s> bool isFPVectorizationPotentiallyUnsafe ( ) { return ! ST -> isTargetDarwin ( ) && ! ST -> hasMVEFloatOps ( ) ; } </s>

Indicate that it is potentially unsafe to automatically vectorize floating-point operations because the semantics of vector and scalar floating-point semantics may differ .


<s> bool isLegalMaskedScatter ( Type * Ty , Align Alignment ) { return isLegalMaskedGather ( Ty , Alignment ) ; } </s>

Return true if the target supports masked scatter .


<s> bool isLegalMaskedStore ( Type * DataTy , Align Alignment ) { return isLegalMaskedLoad ( DataTy , Alignment ) ; } </s>

Return true if the target supports masked store .


<s> bool shouldBuildLookupTablesForConstant ( Constant * C ) const { if ( ST -> isROPI ( ) || ST -> isRWPI ( ) ) return ! C -> needsDynamicRelocation ( ) ; return true ; } </s>

Return true if switches should be turned into lookup tables containing this constant value for the target .


<s> int ARMTTIImpl :: getFPOpCost ( Type * Ty ) { if ( ST -> hasVFP2 ( ) && ! ST -> isThumb1Only ( ) ) { if ( Ty -> isFloatTy ( ) ) { return TargetTransformInfo :: TCC_Basic ; } if ( Ty -> isDoubleTy ( ) ) { return ST -> isFPOnlySP ( ) ? TargetTransformInfo :: TCC_Expensive : TargetTransformInfo :: TCC_Basic ; } } return TargetTransformInfo :: TCC_Expensive ; } </s>

Return the expected cost of supporting the floating point operation of the specified type .


<s> int ARMTTIImpl :: getIntImmCost ( const APInt & Imm , Type * Ty ) { assert ( Ty -> isIntegerTy ( ) ) ; unsigned Bits = Ty -> getPrimitiveSizeInBits ( ) ; if ( Bits == 0 || Bits > 32 ) return 4 ; int32_t SImmVal = Imm . getSExtValue ( ) ; uint32_t ZImmVal = Imm . getZExtValue ( ) ; if ( ! ST -> isThumb ( ) ) { if ( ( SImmVal >= 0 && SImmVal < 65536 ) || ( ARM_AM :: getSOImmVal ( ZImmVal ) != - 1 ) || ( ARM_AM :: getSOImmVal ( ~ ZImmVal ) != - 1 ) ) return 1 ; return ST -> hasV6T2Ops ( ) ? 2 : 3 ; } if ( ST -> isThumb2 ( ) ) { if ( ( SImmVal >= 0 && SImmVal < 65536 ) || ( ARM_AM :: getT2SOImmVal ( ZImmVal ) != - 1 ) || ( ARM_AM :: getT2SOImmVal ( ~ ZImmVal ) != - 1 ) ) return 1 ; return ST -> hasV6T2Ops ( ) ? 2 : 3 ; } if ( SImmVal >= 0 && SImmVal < 256 ) return 1 ; if ( ( ~ ZImmVal < 256 ) || ARM_AM :: isThumbImmShiftedVal ( ZImmVal ) ) return 2 ; return 3 ; } </s>

Calculate the cost of materializing a 64-bit value .


<s>
unsigned
ARMTTIImpl
::
getGatherScatterOpCost
(
unsigned
Opcode
,
Type
*
DataTy
,
Value
*
Ptr
,
bool
VariableMask
,
unsigned
Alignment
)
{
if
(
!
ST
->
hasMVEIntegerOps
(
)
||
!
EnableMaskedGatherScatters
)
return
BaseT
::
getGatherScatterOpCost
(
Opcode
,
DataTy
,
Ptr
,
VariableMask
,
Alignment
)
;
assert
(
DataTy
->
isVectorTy
(
)
&&
"Can't do gather/scatters on scalar!"
)
;
VectorType
*
VTy
=
cast
<
VectorType
>
(
DataTy
)
;
unsigned
NumElems
=
VTy
->
getNumElements
(
)
;
unsigned
EltSize
=
VTy
->
getScalarSizeInBits
(
)
;
std
::
pair
<
int
,
MVT
>
LT
=
TLI
->
getTypeLegalizationCost
(
DL
,
DataTy
)
;
unsigned
VectorCost
=
NumElems
*
LT
.
first
;
unsigned
ScalarCost
=
NumElems
*
LT
.
first
+
BaseT
::
getScalarizationOverhead
(
DataTy
,
{
}
)
;
if
(
EltSize
*
NumElems
!=
128
||
NumElems
<
4
)
return
ScalarCost
;
if
(
Alignment
<
EltSize
/
8
)
return
ScalarCost
;
if
(
EltSize
==
32
)
return
VectorCost
;
if
(
EltSize
!=
8
&&
EltSize
!=
16
)
return
ScalarCost
;
if
(
auto
BC
=
dyn_cast
<
BitCastInst
>
(
Ptr
)
)
Ptr
=
BC
->
getOperand
(
0
)
;
if
(
auto
*
GEP
=
dyn_cast
<
GetElementPtrInst
>
(
Ptr
)
)
{
if
(
GEP
->
getNumOperands
(
)
!=
2
)
return
ScalarCost
;
unsigned
Scale
=
DL
.
getTypeAllocSize
(
GEP
->
getResultElementType
(
)
)
;
if
(
Scale
!=
1
&&
Scale
*
8
!=
EltSize
)
return
ScalarCost
;
if
(
auto
ZExt
=
dyn_cast
<
ZExtInst
>
(
GEP
->
getOperand
(
1
)
)
)
if
(
ZExt
->
getOperand
(
0
)
->
getType
(
)
->
getScalarSizeInBits
(
)
<=
EltSize
)
return
VectorCost
;
return
ScalarCost
;
}
return
ScalarCost
;
}
</s>

Calculate the cost of Gather / Scatter operation .


<s> int ARMTTIImpl :: getIntImmCostInst ( unsigned Opcode , unsigned Idx , const APInt & Imm , Type * Ty ) { if ( ( Opcode == Instruction :: SDiv || Opcode == Instruction :: UDiv || Opcode == Instruction :: SRem || Opcode == Instruction :: URem ) && Idx == 1 ) return 0 ; if ( Opcode == Instruction :: And ) { if ( Imm == 255 || Imm == 65535 ) return 0 ; return std :: min ( getIntImmCost ( Imm , Ty ) , getIntImmCost ( ~ Imm , Ty ) ) ; } if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty ) , getIntImmCost ( - Imm , Ty ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == 32 ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < 1 << 12 ) return 0 ; if ( ST -> isThumb ( ) && NegImm < 1 << 8 ) return 0 ; } if ( Opcode == Instruction :: Xor && Imm . isAllOnesValue ( ) ) return 0 ; return getIntImmCost ( Imm , Ty ) ; } </s>

Return the expected cost of materialization for the given integer immediate of the specified type for a given instruction .


<s>
void
ARMTTIImpl
::
getUnrollingPreferences
(
Loop
*
L
,
ScalarEvolution
&
SE
,
TTI
::
UnrollingPreferences
&
UP
)
{
if
(
!
ST
->
isMClass
(
)
)
return
BasicTTIImplBase
::
getUnrollingPreferences
(
L
,
SE
,
UP
)
;
UP
.
OptSizeThreshold
=
0
;
UP
.
PartialOptSizeThreshold
=
0
;
if
(
L
->
getHeader
(
)
->
getParent
(
)
->
hasOptSize
(
)
)
return
;
if
(
!
ST
->
isThumb2
(
)
)
return
;
SmallVector
<
BasicBlock
*
,
4
>
ExitingBlocks
;
L
->
getExitingBlocks
(
ExitingBlocks
)
;
LLVM_DEBUG
(
dbgs
(
)
<<
"Loop has:\n"
<<
"Blocks: "
<<
L
->
getNumBlocks
(
)
<<
"\n"
<<
"Exit blocks: "
<<
ExitingBlocks
.
size
(
)
<<
"\n"
)
;
if
(
ExitingBlocks
.
size
(
)
>
2
)
return
;
if
(
ST
->
hasBranchPredictor
(
)
&&
L
->
getNumBlocks
(
)
>
4
)
return
;
unsigned
Cost
=
0
;
for
(
auto
*
BB
:
L
->
getBlocks
(
)
)
{
for
(
auto
&
I
:
*
BB
)
{
if
(
I
.
getType
(
)
->
isVectorTy
(
)
)
return
;
if
(
isa
<
CallInst
>
(
I
)
||
isa
<
InvokeInst
>
(
I
)
)
{
ImmutableCallSite
CS
(
&
I
)
;
if
(
const
Function
*
F
=
CS
.
getCalledFunction
(
)
)
{
if
(
!
isLoweredToCall
(
F
)
)
continue
;
}
return
;
}
SmallVector
<
const
Value
*
,
4
>
Operands
(
I
.
value_op_begin
(
)
,
I
.
value_op_end
(
)
)
;
Cost
+=
getUserCost
(
&
I
,
Operands
)
;
}
}
LLVM_DEBUG
(
dbgs
(
)
<<
"Cost of loop: "
<<
Cost
<<
"\n"
)
;
UP
.
Partial
=
true
;
UP
.
Runtime
=
true
;
UP
.
UpperBound
=
true
;
UP
.
UnrollRemainder
=
true
;
UP
.
DefaultUnrollRuntimeCount
=
4
;
UP
.
UnrollAndJam
=
true
;
UP
.
UnrollAndJamInnerLoopThreshold
=
60
;
if
(
Cost
<
12
)
UP
.
Force
=
true
;
}
</s>


<s> bool ARMTTIImpl :: isLegalMaskedGather ( Type * Ty , MaybeAlign Alignment ) { if ( ! EnableMaskedGatherScatters || ! ST -> hasMVEIntegerOps ( ) ) return false ; if ( isa < VectorType > ( Ty ) ) return false ; unsigned EltWidth = Ty -> getScalarSizeInBits ( ) ; return ( ( EltWidth == 32 && ( ! Alignment || Alignment >= 4 ) ) || ( EltWidth == 16 && ( ! Alignment || Alignment >= 2 ) ) || EltWidth == 8 ) ; } </s>

Return true if the target supports masked gather .


<s> bool ARMTTIImpl :: isLegalMaskedLoad ( Type * DataTy , MaybeAlign Alignment ) { if ( ! EnableMaskedLoadStores || ! ST -> hasMVEIntegerOps ( ) ) return false ; if ( auto * VecTy = dyn_cast < VectorType > ( DataTy ) ) { if ( VecTy -> getNumElements ( ) == 2 ) return false ; unsigned VecWidth = DataTy -> getPrimitiveSizeInBits ( ) ; if ( VecWidth != 128 && VecTy -> getElementType ( ) -> isFloatingPointTy ( ) ) return false ; } unsigned EltWidth = DataTy -> getScalarSizeInBits ( ) ; return ( EltWidth == 32 && ( ! Alignment || Alignment >= 4 ) ) || ( EltWidth == 16 && ( ! Alignment || Alignment >= 2 ) ) || ( EltWidth == 8 ) ; } </s>

Return true if the target supports masked load .


<s> int ARMTTIImpl :: getIntImmCost ( unsigned Opcode , unsigned Idx , const APInt & Imm , Type * Ty ) { if ( ( Opcode == Instruction :: SDiv || Opcode == Instruction :: UDiv || Opcode == Instruction :: SRem || Opcode == Instruction :: URem ) && Idx == 1 ) return 0 ; if ( Opcode == Instruction :: And ) return std :: min ( getIntImmCost ( Imm , Ty ) , getIntImmCost ( ~ Imm , Ty ) ) ; if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty ) , getIntImmCost ( - Imm , Ty ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == 32 ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < 1 << 12 ) return 0 ; if ( ST -> isThumb ( ) && NegImm < 1 << 8 ) return 0 ; } return getIntImmCost ( Imm , Ty ) ; } </s>


<s> bool isLegalMaskedScatter ( Type * Ty , MaybeAlign Alignment ) { return false ; } </s>


<s> InstructionCost ARMTTIImpl :: getArithmeticReductionCost ( unsigned Opcode , VectorType * ValTy , Optional < FastMathFlags > FMF , TTI :: TargetCostKind CostKind ) { if ( TTI :: requiresOrderedReduction ( FMF ) ) return BaseT :: getArithmeticReductionCost ( Opcode , ValTy , FMF , CostKind ) ; EVT ValVT = TLI -> getValueType ( DL , ValTy ) ; int ISD = TLI -> InstructionOpcodeToISD ( Opcode ) ; if ( ! ST -> hasMVEIntegerOps ( ) || ! ValVT . isSimple ( ) || ISD != ISD :: ADD ) return BaseT :: getArithmeticReductionCost ( Opcode , ValTy , FMF , CostKind ) ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ; static const CostTblEntry CostTblAdd [ ] { { ISD :: ADD , MVT :: v16i8 , 1 } , { ISD :: ADD , MVT :: v8i16 , 1 } , { ISD :: ADD , MVT :: v4i32 , 1 } , } ; if ( const auto * Entry = CostTableLookup ( CostTblAdd , ISD , LT . second ) ) return Entry -> Cost * ST -> getMVEVectorCostFactor ( CostKind ) * LT . first ; return BaseT :: getArithmeticReductionCost ( Opcode , ValTy , FMF , CostKind ) ; } </s>

Calculate the cost of vector reduction intrinsics .


<s> InstructionCost ARMTTIImpl :: getIntImmCodeSizeCost ( unsigned Opcode , unsigned Idx , const APInt & Imm , Type * Ty ) { if ( Imm . isNonNegative ( ) && Imm . getLimitedValue ( ) < 256 ) return 0 ; return 1 ; } </s>

Return the expected cost for the given integer when optimising for size .


<s> InstructionCost ARMTTIImpl :: getIntImmCost ( const APInt & Imm , Type * Ty , TTI :: TargetCostKind CostKind ) { assert ( Ty -> isIntegerTy ( ) ) ; unsigned Bits = Ty -> getPrimitiveSizeInBits ( ) ; if ( Bits == 0 || Imm . getActiveBits ( ) >= 64 ) return 4 ; int64_t SImmVal = Imm . getSExtValue ( ) ; uint64_t ZImmVal = Imm . getZExtValue ( ) ; if ( ! ST -> isThumb ( ) ) { if ( ( SImmVal >= 0 && SImmVal < 65536 ) || ( ARM_AM :: getSOImmVal ( ZImmVal ) != - 1 ) || ( ARM_AM :: getSOImmVal ( ~ ZImmVal ) != - 1 ) ) return 1 ; return ST -> hasV6T2Ops ( ) ? 2 : 3 ; } if ( ST -> isThumb2 ( ) ) { if ( ( SImmVal >= 0 && SImmVal < 65536 ) || ( ARM_AM :: getT2SOImmVal ( ZImmVal ) != - 1 ) || ( ARM_AM :: getT2SOImmVal ( ~ ZImmVal ) != - 1 ) ) return 1 ; return ST -> hasV6T2Ops ( ) ? 2 : 3 ; } if ( Bits == 8 || ( SImmVal >= 0 && SImmVal < 256 ) ) return 1 ; if ( ( ~ SImmVal < 256 ) || ARM_AM :: isThumbImmShiftedVal ( ZImmVal ) ) return 2 ; return 3 ; } </s>


<s> InstructionCost ARMTTIImpl :: getIntImmCostInst ( unsigned Opcode , unsigned Idx , const APInt & Imm , Type * Ty , TTI :: TargetCostKind CostKind , Instruction * Inst ) { if ( ( Opcode == Instruction :: SDiv || Opcode == Instruction :: UDiv || Opcode == Instruction :: SRem || Opcode == Instruction :: URem ) && Idx == 1 ) return 0 ; if ( Opcode == Instruction :: GetElementPtr && Idx != 0 ) return 0 ; if ( Opcode == Instruction :: And ) { if ( Imm == 255 || Imm == 65535 ) return 0 ; return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( ~ Imm , Ty , CostKind ) ) ; } if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( - Imm , Ty , CostKind ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == 32 ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < 1 << 12 ) return 0 ; if ( ST -> isThumb ( ) && NegImm < 1 << 8 ) return 0 ; } if ( Opcode == Instruction :: Xor && Imm . isAllOnes ( ) ) return 0 ; if ( Inst && ( ( ST -> hasV6Ops ( ) && ! ST -> isThumb ( ) ) || ST -> isThumb2 ( ) ) && Ty -> getIntegerBitWidth ( ) <= 32 ) { if ( isSSATMinMaxPattern ( Inst , Imm ) || ( isa < ICmpInst > ( Inst ) && Inst -> hasOneUse ( ) && isSSATMinMaxPattern ( cast < Instruction > ( * Inst -> user_begin ( ) ) , Imm ) ) ) return 0 ; } if ( Inst && Opcode == Instruction :: ICmp && Idx == 1 && Imm . isAllOnesValue ( ) ) { ICmpInst :: Predicate Pred = cast < ICmpInst > ( Inst ) -> getPredicate ( ) ; if ( Pred == ICmpInst :: ICMP_SGT || Pred == ICmpInst :: ICMP_SLE ) return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( Imm + 1 , Ty , CostKind ) ) ; } return getIntImmCost ( Imm , Ty , CostKind ) ; } </s>


<s> InstructionCost ARMTTIImpl :: getIntrinsicInstrCost ( const IntrinsicCostAttributes & ICA , TTI :: TargetCostKind CostKind ) { switch ( ICA . getID ( ) ) { case Intrinsic :: get_active_lane_mask : if ( ST -> hasMVEIntegerOps ( ) ) return 0 ; break ; case Intrinsic :: sadd_sat : case Intrinsic :: ssub_sat : case Intrinsic :: uadd_sat : case Intrinsic :: usub_sat : { if ( ! ST -> hasMVEIntegerOps ( ) ) break ; Type * VT = ICA . getReturnType ( ) ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , VT ) ; if ( LT . second == MVT :: v4i32 || LT . second == MVT :: v8i16 || LT . second == MVT :: v16i8 ) { unsigned Instrs = LT . second . getScalarSizeInBits ( ) == VT -> getScalarSizeInBits ( ) ? 1 : 4 ; return LT . first * ST -> getMVEVectorCostFactor ( CostKind ) * Instrs ; } break ; } case Intrinsic :: abs : case Intrinsic :: smin : case Intrinsic :: smax : case Intrinsic :: umin : case Intrinsic :: umax : { if ( ! ST -> hasMVEIntegerOps ( ) ) break ; Type * VT = ICA . getReturnType ( ) ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , VT ) ; if ( LT . second == MVT :: v4i32 || LT . second == MVT :: v8i16 || LT . second == MVT :: v16i8 ) return LT . first * ST -> getMVEVectorCostFactor ( CostKind ) ; break ; } case Intrinsic :: minnum : case Intrinsic :: maxnum : { if ( ! ST -> hasMVEFloatOps ( ) ) break ; Type * VT = ICA . getReturnType ( ) ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , VT ) ; if ( LT . second == MVT :: v4f32 || LT . second == MVT :: v8f16 ) return LT . first * ST -> getMVEVectorCostFactor ( CostKind ) ; break ; } } return BaseT :: getIntrinsicInstrCost ( ICA , CostKind ) ; } </s>


<s>
int
ARMTTIImpl
::
getNumMemOps
(
const
IntrinsicInst
*
I
)
const
{
MemOp
MOp
;
unsigned
DstAddrSpace
=
~
0u
;
unsigned
SrcAddrSpace
=
~
0u
;
const
Function
*
F
=
I
->
getParent
(
)
->
getParent
(
)
;
if
(
const
auto
*
MC
=
dyn_cast
<
MemTransferInst
>
(
I
)
)
{
ConstantInt
*
C
=
dyn_cast
<
ConstantInt
>
(
MC
->
getLength
(
)
)
;
if
(
!
C
)
return
1
;
const
unsigned
Size
=
C
->
getValue
(
)
.
getZExtValue
(
)
;
const
Align
DstAlign
=
*
MC
->
getDestAlign
(
)
;
const
Align
SrcAlign
=
*
MC
->
getSourceAlign
(
)
;
MOp
=
MemOp
::
Copy
(
Size
,
false
,
DstAlign
,
SrcAlign
,
false
)
;
DstAddrSpace
=
MC
->
getDestAddressSpace
(
)
;
SrcAddrSpace
=
MC
->
getSourceAddressSpace
(
)
;
}
else
if
(
const
auto
*
MS
=
dyn_cast
<
MemSetInst
>
(
I
)
)
{
ConstantInt
*
C
=
dyn_cast
<
ConstantInt
>
(
MS
->
getLength
(
)
)
;
if
(
!
C
)
return
1
;
const
unsigned
Size
=
C
->
getValue
(
)
.
getZExtValue
(
)
;
const
Align
DstAlign
=
*
MS
->
getDestAlign
(
)
;
MOp
=
MemOp
::
Set
(
Size
,
false
,
DstAlign
,
false
,
false
)
;
DstAddrSpace
=
MS
->
getDestAddressSpace
(
)
;
}
else
llvm_unreachable
(
"Expected a memcpy/move or memset!"
)
;
unsigned
Limit
,
Factor
=
2
;
switch
(
I
->
getIntrinsicID
(
)
)
{
case
Intrinsic
::
memcpy
:
Limit
=
TLI
->
getMaxStoresPerMemcpy
(
F
->
hasMinSize
(
)
)
;
break
;
case
Intrinsic
::
memmove
:
Limit
=
TLI
->
getMaxStoresPerMemmove
(
F
->
hasMinSize
(
)
)
;
break
;
case
Intrinsic
::
memset
:
Limit
=
TLI
->
getMaxStoresPerMemset
(
F
->
hasMinSize
(
)
)
;
Factor
=
1
;
break
;
default
:
llvm_unreachable
(
"Expected a memcpy/move or memset!"
)
;
}
std
::
vector
<
EVT
>
MemOps
;
if
(
getTLI
(
)
->
findOptimalMemOpLowering
(
MemOps
,
Limit
,
MOp
,
DstAddrSpace
,
SrcAddrSpace
,
F
->
getAttributes
(
)
)
)
return
MemOps
.
size
(
)
*
Factor
;
return
1
;
}
</s>

Given a memcpy/memset/memmove instruction , return the number of memory operations performed , via querying findOptimalMemOpLowering .


<s> void ARMTTIImpl :: getPeelingPreferences ( Loop * L , ScalarEvolution & SE , TTI :: PeelingPreferences & PP ) { BaseT :: getPeelingPreferences ( L , SE , PP ) ; } </s>

Get target-customized preferences for the generic loop peeling transformation .


<s> TTI :: AddressingModeKind ARMTTIImpl :: getPreferredAddressingMode ( const Loop * L , ScalarEvolution * SE ) const { if ( ST -> hasMVEIntegerOps ( ) ) return TTI :: AMK_PostIndexed ; if ( L -> getHeader ( ) -> getParent ( ) -> hasOptSize ( ) ) return TTI :: AMK_None ; if ( ST -> isMClass ( ) && ST -> isThumb2 ( ) && L -> getNumBlocks ( ) == 1 ) return TTI :: AMK_PreIndexed ; return TTI :: AMK_None ; } </s>

Return the preferred addressing mode LSR should make efforts to generate .


<s>
void
ARMTTIImpl
::
getUnrollingPreferences
(
Loop
*
L
,
ScalarEvolution
&
SE
,
TTI
::
UnrollingPreferences
&
UP
,
OptimizationRemarkEmitter
*
ORE
)
{
UP
.
UpperBound
=
true
;
if
(
!
ST
->
isMClass
(
)
)
return
BasicTTIImplBase
::
getUnrollingPreferences
(
L
,
SE
,
UP
,
ORE
)
;
UP
.
OptSizeThreshold
=
0
;
UP
.
PartialOptSizeThreshold
=
0
;
if
(
L
->
getHeader
(
)
->
getParent
(
)
->
hasOptSize
(
)
)
return
;
SmallVector
<
BasicBlock
*
,
4
>
ExitingBlocks
;
L
->
getExitingBlocks
(
ExitingBlocks
)
;
LLVM_DEBUG
(
dbgs
(
)
<<
"Loop has:\n"
<<
"Blocks: "
<<
L
->
getNumBlocks
(
)
<<
"\n"
<<
"Exit blocks: "
<<
ExitingBlocks
.
size
(
)
<<
"\n"
)
;
if
(
ExitingBlocks
.
size
(
)
>
2
)
return
;
if
(
ST
->
hasBranchPredictor
(
)
&&
L
->
getNumBlocks
(
)
>
4
)
return
;
if
(
getBooleanLoopAttribute
(
L
,
"llvm.loop.isvectorized"
)
)
return
;
InstructionCost
Cost
=
0
;
for
(
auto
*
BB
:
L
->
getBlocks
(
)
)
{
for
(
auto
&
I
:
*
BB
)
{
if
(
I
.
getType
(
)
->
isVectorTy
(
)
)
return
;
if
(
isa
<
CallInst
>
(
I
)
||
isa
<
InvokeInst
>
(
I
)
)
{
if
(
const
Function
*
F
=
cast
<
CallBase
>
(
I
)
.
getCalledFunction
(
)
)
{
if
(
!
isLoweredToCall
(
F
)
)
continue
;
}
return
;
}
SmallVector
<
const
Value
*
,
4
>
Operands
(
I
.
operand_values
(
)
)
;
Cost
+=
getUserCost
(
&
I
,
Operands
,
TargetTransformInfo
::
TCK_SizeAndLatency
)
;
}
}
unsigned
UnrollCount
=
4
;
if
(
ST
->
isThumb1Only
(
)
)
{
unsigned
ExitingValues
=
0
;
SmallVector
<
BasicBlock
*
,
4
>
ExitBlocks
;
L
->
getExitBlocks
(
ExitBlocks
)
;
for
(
auto
*
Exit
:
ExitBlocks
)
{
unsigned
LiveOuts
=
count_if
(
Exit
->
phis
(
)
,
[
]
(
auto
&
PH
)
{
return
PH
.
getNumOperands
(
)
!=
1
||
!
isa
<
GetElementPtrInst
>
(
PH
.
getOperand
(
0
)
)
;
}
)
;
ExitingValues
=
ExitingValues
<
LiveOuts
LiveOuts
:
ExitingValues
;
}
if
(
ExitingValues
)
UnrollCount
/=
ExitingValues
;
if
(
UnrollCount
<=
1
)
return
;
}
LLVM_DEBUG
(
dbgs
(
)
<<
"Cost of loop: "
<<
Cost
<<
"\n"
)
;
LLVM_DEBUG
(
dbgs
(
)
<<
"Default Runtime Unroll Count: "
<<
UnrollCount
<<
"\n"
)
;
UP
.
Partial
=
true
;
UP
.
Runtime
=
true
;
UP
.
UnrollRemainder
=
true
;
UP
.
DefaultUnrollRuntimeCount
=
UnrollCount
;
UP
.
UnrollAndJam
=
true
;
UP
.
UnrollAndJamInnerLoopThreshold
=
60
;
if
(
Cost
<
12
)
UP
.
Force
=
true
;
}
</s>


"ARMHWLoops: Trip count does not fit into 32bits\n"

Query the target whether it would be profitable to convert the given loop into a hardware loop .


<s> bool ARMTTIImpl :: isLegalMaskedGather ( Type * Ty , Align Alignment ) { if ( ! EnableMaskedGatherScatters || ! ST -> hasMVEIntegerOps ( ) ) return false ; if ( isa < VectorType > ( Ty ) ) return false ; unsigned EltWidth = Ty -> getScalarSizeInBits ( ) ; return ( ( EltWidth == 32 && Alignment >= 4 ) || ( EltWidth == 16 && Alignment >= 2 ) || EltWidth == 8 ) ; } </s>


<s> bool ARMTTIImpl :: isLegalMaskedLoad ( Type * DataTy , Align Alignment ) { if ( ! EnableMaskedLoadStores || ! ST -> hasMVEIntegerOps ( ) ) return false ; if ( auto * VecTy = dyn_cast < FixedVectorType > ( DataTy ) ) { if ( VecTy -> getNumElements ( ) == 2 ) return false ; unsigned VecWidth = DataTy -> getPrimitiveSizeInBits ( ) ; if ( VecWidth != 128 && VecTy -> getElementType ( ) -> isFloatingPointTy ( ) ) return false ; } unsigned EltWidth = DataTy -> getScalarSizeInBits ( ) ; return ( EltWidth == 32 && Alignment >= 4 ) || ( EltWidth == 16 && Alignment >= 2 ) || ( EltWidth == 8 ) ; } </s>


"preferPredicateOverEpilogue: not a single block "

"preferPredicateOverEpilogue: hardware-loop is not "

Query the target whether it would be prefered to create a predicated vector loop , which can avoid the need to emit a scalar epilogue loop .


<s> Optional < Value * > ARMTTIImpl :: simplifyDemandedVectorEltsIntrinsic ( InstCombiner & IC , IntrinsicInst & II , APInt OrigDemandedElts , APInt & UndefElts , APInt & UndefElts2 , APInt & UndefElts3 , std :: function < void ( Instruction * , unsigned , APInt , APInt & ) > SimplifyAndSetOp ) const { auto SimplifyNarrowInstrTopBottom = [ & ] ( unsigned TopOpc ) { unsigned NumElts = cast < FixedVectorType > ( II . getType ( ) ) -> getNumElements ( ) ; unsigned IsTop = cast < ConstantInt > ( II . getOperand ( TopOpc ) ) -> getZExtValue ( ) ; APInt DemandedElts = APInt :: getSplat ( NumElts , IsTop ? APInt :: getLowBitsSet ( 2 , 1 ) : APInt :: getHighBitsSet ( 2 , 1 ) ) ; SimplifyAndSetOp ( & II , 0 , OrigDemandedElts & DemandedElts , UndefElts ) ; UndefElts &= APInt :: getSplat ( NumElts , ! IsTop ? APInt :: getLowBitsSet ( 2 , 1 ) : APInt :: getHighBitsSet ( 2 , 1 ) ) ; return None ; } ; switch ( II . getIntrinsicID ( ) ) { default : break ; case Intrinsic :: arm_mve_vcvt_narrow : SimplifyNarrowInstrTopBottom ( 2 ) ; break ; case Intrinsic :: arm_mve_vqmovn : SimplifyNarrowInstrTopBottom ( 4 ) ; break ; case Intrinsic :: arm_mve_vshrn : SimplifyNarrowInstrTopBottom ( 7 ) ; break ; } return None ; } </s>

Can be used to implement target-specific instruction combining .


<s>
void
ARMTTIImpl
::
getUnrollingPreferences
(
Loop
*
L
,
ScalarEvolution
&
SE
,
TTI
::
UnrollingPreferences
&
UP
)
{
if
(
!
ST
->
isMClass
(
)
)
return
BasicTTIImplBase
::
getUnrollingPreferences
(
L
,
SE
,
UP
)
;
UP
.
OptSizeThreshold
=
0
;
UP
.
PartialOptSizeThreshold
=
0
;
if
(
L
->
getHeader
(
)
->
getParent
(
)
->
optForSize
(
)
)
return
;
if
(
!
ST
->
isThumb2
(
)
)
return
;
SmallVector
<
BasicBlock
*
,
4
>
ExitingBlocks
;
L
->
getExitingBlocks
(
ExitingBlocks
)
;
DEBUG
(
dbgs
(
)
<<
"Loop has:\n"
<<
"Blocks: "
<<
L
->
getNumBlocks
(
)
<<
"\n"
<<
"Exit blocks: "
<<
ExitingBlocks
.
size
(
)
<<
"\n"
)
;
if
(
ExitingBlocks
.
size
(
)
>
2
)
return
;
if
(
ST
->
hasBranchPredictor
(
)
&&
L
->
getNumBlocks
(
)
>
4
)
return
;
unsigned
Cost
=
0
;
for
(
auto
*
BB
:
L
->
getBlocks
(
)
)
{
for
(
auto
&
I
:
*
BB
)
{
if
(
isa
<
CallInst
>
(
I
)
||
isa
<
InvokeInst
>
(
I
)
)
{
ImmutableCallSite
CS
(
&
I
)
;
if
(
const
Function
*
F
=
CS
.
getCalledFunction
(
)
)
{
if
(
!
isLoweredToCall
(
F
)
)
continue
;
}
return
;
}
SmallVector
<
const
Value
*
,
4
>
Operands
(
I
.
value_op_begin
(
)
,
I
.
value_op_end
(
)
)
;
Cost
+=
getUserCost
(
&
I
,
Operands
)
;
}
}
DEBUG
(
dbgs
(
)
<<
"Cost of loop: "
<<
Cost
<<
"\n"
)
;
UP
.
Partial
=
true
;
UP
.
Runtime
=
true
;
UP
.
UnrollRemainder
=
true
;
UP
.
DefaultUnrollRuntimeCount
=
4
;
if
(
Cost
<
12
)
UP
.
Force
=
true
;
}
</s>

Get target-customized preferences for the generic loop unrolling transformation .


<s> int ARMTTIImpl :: getIntImmCost ( unsigned Opcode , unsigned Idx , const APInt & Imm , Type * Ty ) { if ( ( Opcode == Instruction :: SDiv || Opcode == Instruction :: UDiv || Opcode == Instruction :: SRem || Opcode == Instruction :: URem ) && Idx == 1 ) return 0 ; return getIntImmCost ( Imm , Ty ) ; } </s>


<s> InstructionCost ARMTTIImpl :: getIntImmCostInst ( unsigned Opcode , unsigned Idx , const APInt & Imm , Type * Ty , TTI :: TargetCostKind CostKind , Instruction * Inst ) { if ( ( Opcode == Instruction :: SDiv || Opcode == Instruction :: UDiv || Opcode == Instruction :: SRem || Opcode == Instruction :: URem ) && Idx == 1 ) return 0 ; if ( Opcode == Instruction :: GetElementPtr && Idx != 0 ) return 0 ; if ( Opcode == Instruction :: And ) { if ( Imm == 255 || Imm == 65535 ) return 0 ; return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( ~ Imm , Ty , CostKind ) ) ; } if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( - Imm , Ty , CostKind ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == 32 ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < 1 << 12 ) return 0 ; if ( ST -> isThumb ( ) && NegImm < 1 << 8 ) return 0 ; } if ( Opcode == Instruction :: Xor && Imm . isAllOnes ( ) ) return 0 ; if ( Inst && ( ( ST -> hasV6Ops ( ) && ! ST -> isThumb ( ) ) || ST -> isThumb2 ( ) ) && Ty -> getIntegerBitWidth ( ) <= 32 ) { if ( isSSATMinMaxPattern ( Inst , Imm ) || ( isa < ICmpInst > ( Inst ) && Inst -> hasOneUse ( ) && isSSATMinMaxPattern ( cast < Instruction > ( * Inst -> user_begin ( ) ) , Imm ) ) ) return 0 ; } if ( Inst && ST -> hasVFP2Base ( ) && isFPSatMinMaxPattern ( Inst , Imm ) ) return 0 ; if ( Inst && Opcode == Instruction :: ICmp && Idx == 1 && Imm . isAllOnesValue ( ) ) { ICmpInst :: Predicate Pred = cast < ICmpInst > ( Inst ) -> getPredicate ( ) ; if ( Pred == ICmpInst :: ICMP_SGT || Pred == ICmpInst :: ICMP_SLE ) return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( Imm + 1 , Ty , CostKind ) ) ; } return getIntImmCost ( Imm , Ty , CostKind ) ; } </s>


<s> bool ARMTTIImpl :: isLegalMaskedGather ( Type * Ty , Align Alignment ) { if ( ! EnableMaskedGatherScatters || ! ST -> hasMVEIntegerOps ( ) ) return false ; unsigned EltWidth = Ty -> getScalarSizeInBits ( ) ; return ( ( EltWidth == 32 && Alignment >= 4 ) || ( EltWidth == 16 && Alignment >= 2 ) || EltWidth == 8 ) ; } </s>


<s> void ARMTTIImpl :: getUnrollingPreferences ( Loop * L , ScalarEvolution & SE , TTI :: UnrollingPreferences & UP ) { if ( ! ST -> isMClass ( ) ) return BasicTTIImplBase :: getUnrollingPreferences ( L , SE , UP ) ; if ( ! ST -> isThumb2 ( ) || L -> getNumBlocks ( ) != 1 ) return ; UP . OptSizeThreshold = 0 ; UP . PartialOptSizeThreshold = 0 ; BasicBlock * BB = L -> getLoopLatch ( ) ; if ( BB -> getParent ( ) -> optForSize ( ) ) return ; unsigned Cost = 0 ; for ( auto & I : * BB ) { if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { ImmutableCallSite CS ( & I ) ; if ( const Function * F = CS . getCalledFunction ( ) ) { if ( ! isLoweredToCall ( F ) ) continue ; } return ; } SmallVector < const Value * , 4 > Operands ( I . value_op_begin ( ) , I . value_op_end ( ) ) ; Cost += getUserCost ( & I , Operands ) ; } UP . Partial = true ; UP . Runtime = true ; UP . UnrollRemainder = true ; UP . DefaultUnrollRuntimeCount = 4 ; if ( Cost < 12 ) UP . Force = true ; } </s>


<s> bool ARMTTIImpl :: areInlineCompatible ( const Function * Caller , const Function * Callee ) const { const TargetMachine & TM = getTLI ( ) -> getTargetMachine ( ) ; const FeatureBitset & CallerBits = TM . getSubtargetImpl ( * Caller ) -> getFeatureBits ( ) ; const FeatureBitset & CalleeBits = TM . getSubtargetImpl ( * Callee ) -> getFeatureBits ( ) ; bool MatchExact = ( CallerBits & ~ InlineFeaturesAllowed ) == ( CalleeBits & ~ InlineFeaturesAllowed ) ; bool MatchSubset = ( ( CallerBits & CalleeBits ) & InlineFeaturesAllowed ) == ( CalleeBits & InlineFeaturesAllowed ) ; return MatchExact && MatchSubset ; } </s>

Determine whether a callee with the given TLI can be inlined into caller with this TLI , based on 'nobuiltin ' attributes .


<s> int ARMTTIImpl :: getIntImmCost ( const APInt & Imm , Type * Ty , TTI :: TargetCostKind CostKind ) { assert ( Ty -> isIntegerTy ( ) ) ; unsigned Bits = Ty -> getPrimitiveSizeInBits ( ) ; if ( Bits == 0 || Imm . getActiveBits ( ) >= 64 ) return 4 ; int64_t SImmVal = Imm . getSExtValue ( ) ; uint64_t ZImmVal = Imm . getZExtValue ( ) ; if ( ! ST -> isThumb ( ) ) { if ( ( SImmVal >= 0 && SImmVal < 65536 ) || ( ARM_AM :: getSOImmVal ( ZImmVal ) != - 1 ) || ( ARM_AM :: getSOImmVal ( ~ ZImmVal ) != - 1 ) ) return 1 ; return ST -> hasV6T2Ops ( ) ? 2 : 3 ; } if ( ST -> isThumb2 ( ) ) { if ( ( SImmVal >= 0 && SImmVal < 65536 ) || ( ARM_AM :: getT2SOImmVal ( ZImmVal ) != - 1 ) || ( ARM_AM :: getT2SOImmVal ( ~ ZImmVal ) != - 1 ) ) return 1 ; return ST -> hasV6T2Ops ( ) ? 2 : 3 ; } if ( Bits == 8 || ( SImmVal >= 0 && SImmVal < 256 ) ) return 1 ; if ( ( ~ SImmVal < 256 ) || ARM_AM :: isThumbImmShiftedVal ( ZImmVal ) ) return 2 ; return 3 ; } </s>


<s> int ARMTTIImpl :: getIntImmCostInst ( unsigned Opcode , unsigned Idx , const APInt & Imm , Type * Ty , TTI :: TargetCostKind CostKind ) { if ( ( Opcode == Instruction :: SDiv || Opcode == Instruction :: UDiv || Opcode == Instruction :: SRem || Opcode == Instruction :: URem ) && Idx == 1 ) return 0 ; if ( Opcode == Instruction :: And ) { if ( Imm == 255 || Imm == 65535 ) return 0 ; return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( ~ Imm , Ty , CostKind ) ) ; } if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( - Imm , Ty , CostKind ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == 32 ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < 1 << 12 ) return 0 ; if ( ST -> isThumb ( ) && NegImm < 1 << 8 ) return 0 ; } if ( Opcode == Instruction :: Xor && Imm . isAllOnesValue ( ) ) return 0 ; return getIntImmCost ( Imm , Ty , CostKind ) ; } </s>


<s>
void
ARMTTIImpl
::
getUnrollingPreferences
(
Loop
*
L
,
ScalarEvolution
&
SE
,
TTI
::
UnrollingPreferences
&
UP
)
{
if
(
!
ST
->
isMClass
(
)
)
return
BasicTTIImplBase
::
getUnrollingPreferences
(
L
,
SE
,
UP
)
;
UP
.
OptSizeThreshold
=
0
;
UP
.
PartialOptSizeThreshold
=
0
;
if
(
L
->
getHeader
(
)
->
getParent
(
)
->
hasOptSize
(
)
)
return
;
if
(
!
ST
->
isThumb2
(
)
)
return
;
SmallVector
<
BasicBlock
*
,
4
>
ExitingBlocks
;
L
->
getExitingBlocks
(
ExitingBlocks
)
;
LLVM_DEBUG
(
dbgs
(
)
<<
"Loop has:\n"
<<
"Blocks: "
<<
L
->
getNumBlocks
(
)
<<
"\n"
<<
"Exit blocks: "
<<
ExitingBlocks
.
size
(
)
<<
"\n"
)
;
if
(
ExitingBlocks
.
size
(
)
>
2
)
return
;
if
(
ST
->
hasBranchPredictor
(
)
&&
L
->
getNumBlocks
(
)
>
4
)
return
;
unsigned
Cost
=
0
;
for
(
auto
*
BB
:
L
->
getBlocks
(
)
)
{
for
(
auto
&
I
:
*
BB
)
{
if
(
I
.
getType
(
)
->
isVectorTy
(
)
)
return
;
if
(
isa
<
CallInst
>
(
I
)
||
isa
<
InvokeInst
>
(
I
)
)
{
if
(
const
Function
*
F
=
cast
<
CallBase
>
(
I
)
.
getCalledFunction
(
)
)
{
if
(
!
isLoweredToCall
(
F
)
)
continue
;
}
return
;
}
SmallVector
<
const
Value
*
,
4
>
Operands
(
I
.
value_op_begin
(
)
,
I
.
value_op_end
(
)
)
;
Cost
+=
getUserCost
(
&
I
,
Operands
,
TargetTransformInfo
::
TCK_CodeSize
)
;
}
}
LLVM_DEBUG
(
dbgs
(
)
<<
"Cost of loop: "
<<
Cost
<<
"\n"
)
;
UP
.
Partial
=
true
;
UP
.
Runtime
=
true
;
UP
.
UpperBound
=
true
;
UP
.
UnrollRemainder
=
true
;
UP
.
DefaultUnrollRuntimeCount
=
4
;
UP
.
UnrollAndJam
=
true
;
UP
.
UnrollAndJamInnerLoopThreshold
=
60
;
if
(
Cost
<
12
)
UP
.
Force
=
true
;
}
</s>


<s> bool ARMTTIImpl :: isLegalMaskedGather ( Type * Ty , MaybeAlign Alignment ) { if ( ! EnableMaskedGatherScatters || ! ST -> hasMVEIntegerOps ( ) ) return false ; if ( isa < VectorType > ( Ty ) ) return false ; unsigned EltWidth = Ty -> getScalarSizeInBits ( ) ; return ( ( EltWidth == 32 && ( ! Alignment || * Alignment >= 4 ) ) || ( EltWidth == 16 && ( ! Alignment || * Alignment >= 2 ) ) || EltWidth == 8 ) ; } </s>


<s> bool ARMTTIImpl :: isLegalMaskedLoad ( Type * DataTy , MaybeAlign Alignment ) { if ( ! EnableMaskedLoadStores || ! ST -> hasMVEIntegerOps ( ) ) return false ; if ( auto * VecTy = dyn_cast < FixedVectorType > ( DataTy ) ) { if ( VecTy -> getNumElements ( ) == 2 ) return false ; unsigned VecWidth = DataTy -> getPrimitiveSizeInBits ( ) ; if ( VecWidth != 128 && VecTy -> getElementType ( ) -> isFloatingPointTy ( ) ) return false ; } unsigned EltWidth = DataTy -> getScalarSizeInBits ( ) ; return ( EltWidth == 32 && ( ! Alignment || * Alignment >= 4 ) ) || ( EltWidth == 16 && ( ! Alignment || * Alignment >= 2 ) ) || ( EltWidth == 8 ) ; } </s>


<s> bool isLegalMaskedScatter ( Type * Ty , MaybeAlign Alignment ) { return isLegalMaskedGather ( Ty , Alignment ) ; } </s>

Return true if the target supports masked scatter .


<s> bool isLegalMaskedStore ( Type * DataTy , MaybeAlign Alignment ) { return isLegalMaskedLoad ( DataTy , Alignment ) ; } </s>

Return true if the target supports masked store .


<s> bool ARMTTIImpl :: isLoweredToCall ( const Function * F ) { if ( ! F -> isIntrinsic ( ) ) BaseT :: isLoweredToCall ( F ) ; if ( F -> getName ( ) . startswith ( "llvm.arm" ) ) return false ; switch ( F -> getIntrinsicID ( ) ) { default : break ; case Intrinsic :: powi : case Intrinsic :: sin : case Intrinsic :: cos : case Intrinsic :: pow : case Intrinsic :: log : case Intrinsic :: log10 : case Intrinsic :: log2 : case Intrinsic :: exp : case Intrinsic :: exp2 : return true ; case Intrinsic :: sqrt : case Intrinsic :: fabs : case Intrinsic :: copysign : case Intrinsic :: floor : case Intrinsic :: ceil : case Intrinsic :: trunc : case Intrinsic :: rint : case Intrinsic :: nearbyint : case Intrinsic :: round : case Intrinsic :: canonicalize : case Intrinsic :: lround : case Intrinsic :: llround : case Intrinsic :: lrint : case Intrinsic :: llrint : if ( F -> getReturnType ( ) -> isDoubleTy ( ) && ! ST -> hasFP64 ( ) ) return true ; if ( F -> getReturnType ( ) -> isHalfTy ( ) && ! ST -> hasFullFP16 ( ) ) return true ; return ! ST -> hasFPARMv8Base ( ) && ! ST -> hasVFP2Base ( ) ; case Intrinsic :: masked_store : case Intrinsic :: masked_load : case Intrinsic :: masked_gather : case Intrinsic :: masked_scatter : return ! ST -> hasMVEIntegerOps ( ) ; case Intrinsic :: sadd_with_overflow : case Intrinsic :: uadd_with_overflow : case Intrinsic :: ssub_with_overflow : case Intrinsic :: usub_with_overflow : case Intrinsic :: sadd_sat : case Intrinsic :: uadd_sat : case Intrinsic :: ssub_sat : case Intrinsic :: usub_sat : return false ; } return BaseT :: isLoweredToCall ( F ) ; } </s>

Test whether calls to a function lower to actual program function calls .


"preferPredicateOverEpilogue: not a single block "

"preferPredicateOverEpilogue: hardware-loop is not "


<s> bool shouldBuildLookupTablesForConstant ( Constant * C ) const { if ( ST -> isROPI ( ) || ST -> isRWPI ( ) ) return ! C -> needsRelocation ( ) ; return true ; } </s>

Return true if switches should be turned into lookup tables containing this constant value for the target .


<s>
void
ARMTTIImpl
::
getUnrollingPreferences
(
Loop
*
L
,
ScalarEvolution
&
SE
,
TTI
::
UnrollingPreferences
&
UP
)
{
if
(
!
ST
->
isMClass
(
)
)
return
BasicTTIImplBase
::
getUnrollingPreferences
(
L
,
SE
,
UP
)
;
UP
.
OptSizeThreshold
=
0
;
UP
.
PartialOptSizeThreshold
=
0
;
if
(
L
->
getHeader
(
)
->
getParent
(
)
->
optForSize
(
)
)
return
;
if
(
!
ST
->
isThumb2
(
)
)
return
;
SmallVector
<
BasicBlock
*
,
4
>
ExitingBlocks
;
L
->
getExitingBlocks
(
ExitingBlocks
)
;
LLVM_DEBUG
(
dbgs
(
)
<<
"Loop has:\n"
<<
"Blocks: "
<<
L
->
getNumBlocks
(
)
<<
"\n"
<<
"Exit blocks: "
<<
ExitingBlocks
.
size
(
)
<<
"\n"
)
;
if
(
ExitingBlocks
.
size
(
)
>
2
)
return
;
if
(
ST
->
hasBranchPredictor
(
)
&&
L
->
getNumBlocks
(
)
>
4
)
return
;
unsigned
Cost
=
0
;
for
(
auto
*
BB
:
L
->
getBlocks
(
)
)
{
for
(
auto
&
I
:
*
BB
)
{
if
(
isa
<
CallInst
>
(
I
)
||
isa
<
InvokeInst
>
(
I
)
)
{
ImmutableCallSite
CS
(
&
I
)
;
if
(
const
Function
*
F
=
CS
.
getCalledFunction
(
)
)
{
if
(
!
isLoweredToCall
(
F
)
)
continue
;
}
return
;
}
SmallVector
<
const
Value
*
,
4
>
Operands
(
I
.
value_op_begin
(
)
,
I
.
value_op_end
(
)
)
;
Cost
+=
getUserCost
(
&
I
,
Operands
)
;
}
}
LLVM_DEBUG
(
dbgs
(
)
<<
"Cost of loop: "
<<
Cost
<<
"\n"
)
;
UP
.
Partial
=
true
;
UP
.
Runtime
=
true
;
UP
.
UnrollRemainder
=
true
;
UP
.
DefaultUnrollRuntimeCount
=
4
;
if
(
Cost
<
12
)
UP
.
Force
=
true
;
}
</s>


<s> void * getAdjustedAnalysisPointer ( const void * ID ) override { if ( ID == & TargetTransformInfo :: ID ) return ( TargetTransformInfo * ) this ; return this ; } </s>

getAdjustedAnalysisPointer - This method is used when a pass implements an analysis interface through multiple inheritance .


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { TargetTransformInfo :: getAnalysisUsage ( AU ) ; } </s>

getAnalysisUsage - Subclasses that override getAnalysisUsage must call this .


<s> unsigned ARMTTI :: getIntImmCost ( const APInt & Imm , Type * Ty ) const { assert ( Ty -> isIntegerTy ( ) ) ; unsigned Bits = Ty -> getPrimitiveSizeInBits ( ) ; if ( Bits == 0 || Bits > 32 ) return 4 ; int32_t SImmVal = Imm . getSExtValue ( ) ; uint32_t ZImmVal = Imm . getZExtValue ( ) ; if ( ! ST -> isThumb ( ) ) { if ( ( SImmVal >= 0 && SImmVal < 65536 ) || ( ARM_AM :: getSOImmVal ( ZImmVal ) != - 1 ) || ( ARM_AM :: getSOImmVal ( ~ ZImmVal ) != - 1 ) ) return 1 ; return ST -> hasV6T2Ops ( ) ? 2 : 3 ; } if ( ST -> isThumb2 ( ) ) { if ( ( SImmVal >= 0 && SImmVal < 65536 ) || ( ARM_AM :: getT2SOImmVal ( ZImmVal ) != - 1 ) || ( ARM_AM :: getT2SOImmVal ( ~ ZImmVal ) != - 1 ) ) return 1 ; return ST -> hasV6T2Ops ( ) ? 2 : 3 ; } if ( SImmVal >= 0 && SImmVal < 256 ) return 1 ; if ( ( ~ ZImmVal < 256 ) || ARM_AM :: isThumbImmShiftedVal ( ZImmVal ) ) return 2 ; return 3 ; } </s>

Calculate the cost of materializing a 64-bit value .


<s> unsigned getNumberOfRegisters ( bool Vector ) const override { if ( Vector ) { if ( ST -> hasNEON ( ) ) return 16 ; return 0 ; } if ( ST -> isThumb1Only ( ) ) return 8 ; return 13 ; } </s>


<s> void initializePass ( ) override { pushTTIStack ( this ) ; } </s>

initializePass - This method may be overriden by immutable passes to allow them to perform various initialization actions they require .


<s> unsigned getNumberOfRegisters ( bool Vector ) { if ( Vector ) { if ( ST -> hasNEON ( ) ) return 16 ; if ( ST -> hasMVEIntegerOps ( ) ) return 8 ; return 0 ; } if ( ST -> isThumb1Only ( ) ) return 8 ; return 13 ; } </s>


<s> bool isLegalMaskedStore ( Type * DataTy ) { return isLegalMaskedLoad ( DataTy ) ; } </s>


<s> unsigned ARMTTIImpl :: getIntImmCost ( const APInt & Imm , Type * Ty ) { assert ( Ty -> isIntegerTy ( ) ) ; unsigned Bits = Ty -> getPrimitiveSizeInBits ( ) ; if ( Bits == 0 || Bits > 32 ) return 4 ; int32_t SImmVal = Imm . getSExtValue ( ) ; uint32_t ZImmVal = Imm . getZExtValue ( ) ; if ( ! ST -> isThumb ( ) ) { if ( ( SImmVal >= 0 && SImmVal < 65536 ) || ( ARM_AM :: getSOImmVal ( ZImmVal ) != - 1 ) || ( ARM_AM :: getSOImmVal ( ~ ZImmVal ) != - 1 ) ) return 1 ; return ST -> hasV6T2Ops ( ) ? 2 : 3 ; } if ( ST -> isThumb2 ( ) ) { if ( ( SImmVal >= 0 && SImmVal < 65536 ) || ( ARM_AM :: getT2SOImmVal ( ZImmVal ) != - 1 ) || ( ARM_AM :: getT2SOImmVal ( ~ ZImmVal ) != - 1 ) ) return 1 ; return ST -> hasV6T2Ops ( ) ? 2 : 3 ; } if ( SImmVal >= 0 && SImmVal < 256 ) return 1 ; if ( ( ~ ZImmVal < 256 ) || ARM_AM :: isThumbImmShiftedVal ( ZImmVal ) ) return 2 ; return 3 ; } </s>


<s>
void
ARMTTIImpl
::
getUnrollingPreferences
(
Loop
*
L
,
ScalarEvolution
&
SE
,
TTI
::
UnrollingPreferences
&
UP
)
{
if
(
!
ST
->
isMClass
(
)
)
return
BasicTTIImplBase
::
getUnrollingPreferences
(
L
,
SE
,
UP
)
;
UP
.
OptSizeThreshold
=
0
;
UP
.
PartialOptSizeThreshold
=
0
;
if
(
L
->
getHeader
(
)
->
getParent
(
)
->
hasOptSize
(
)
)
return
;
if
(
!
ST
->
isThumb2
(
)
)
return
;
SmallVector
<
BasicBlock
*
,
4
>
ExitingBlocks
;
L
->
getExitingBlocks
(
ExitingBlocks
)
;
LLVM_DEBUG
(
dbgs
(
)
<<
"Loop has:\n"
<<
"Blocks: "
<<
L
->
getNumBlocks
(
)
<<
"\n"
<<
"Exit blocks: "
<<
ExitingBlocks
.
size
(
)
<<
"\n"
)
;
if
(
ExitingBlocks
.
size
(
)
>
2
)
return
;
if
(
ST
->
hasBranchPredictor
(
)
&&
L
->
getNumBlocks
(
)
>
4
)
return
;
unsigned
Cost
=
0
;
for
(
auto
*
BB
:
L
->
getBlocks
(
)
)
{
for
(
auto
&
I
:
*
BB
)
{
if
(
isa
<
CallInst
>
(
I
)
||
isa
<
InvokeInst
>
(
I
)
)
{
ImmutableCallSite
CS
(
&
I
)
;
if
(
const
Function
*
F
=
CS
.
getCalledFunction
(
)
)
{
if
(
!
isLoweredToCall
(
F
)
)
continue
;
}
return
;
}
if
(
I
.
getType
(
)
->
isVectorTy
(
)
)
return
;
SmallVector
<
const
Value
*
,
4
>
Operands
(
I
.
value_op_begin
(
)
,
I
.
value_op_end
(
)
)
;
Cost
+=
getUserCost
(
&
I
,
Operands
)
;
}
}
LLVM_DEBUG
(
dbgs
(
)
<<
"Cost of loop: "
<<
Cost
<<
"\n"
)
;
UP
.
Partial
=
true
;
UP
.
Runtime
=
true
;
UP
.
UpperBound
=
true
;
UP
.
UnrollRemainder
=
true
;
UP
.
DefaultUnrollRuntimeCount
=
4
;
UP
.
UnrollAndJam
=
true
;
UP
.
UnrollAndJamInnerLoopThreshold
=
60
;
if
(
Cost
<
12
)
UP
.
Force
=
true
;
}
</s>


<s> bool ARMTTIImpl :: isLegalMaskedLoad ( Type * DataTy ) { if ( ! EnableMaskedLoadStores || ! ST -> hasMVEIntegerOps ( ) ) return false ; if ( DataTy -> isVectorTy ( ) ) { unsigned VecWidth = DataTy -> getPrimitiveSizeInBits ( ) ; if ( VecWidth != 128 ) return false ; } unsigned EltWidth = DataTy -> getScalarSizeInBits ( ) ; return EltWidth == 32 || EltWidth == 16 || EltWidth == 8 ; } </s>

Return true if the target supports masked load .


<s> int ARMTTIImpl :: getArithmeticReductionCost ( unsigned Opcode , VectorType * ValTy , bool IsPairwiseForm , TTI :: TargetCostKind CostKind ) { EVT ValVT = TLI -> getValueType ( DL , ValTy ) ; int ISD = TLI -> InstructionOpcodeToISD ( Opcode ) ; if ( ! ST -> hasMVEIntegerOps ( ) || ! ValVT . isSimple ( ) || ISD != ISD :: ADD ) return BaseT :: getArithmeticReductionCost ( Opcode , ValTy , IsPairwiseForm , CostKind ) ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ; static const CostTblEntry CostTblAdd [ ] { { ISD :: ADD , MVT :: v16i8 , 1 } , { ISD :: ADD , MVT :: v8i16 , 1 } , { ISD :: ADD , MVT :: v4i32 , 1 } , } ; if ( const auto * Entry = CostTableLookup ( CostTblAdd , ISD , LT . second ) ) return Entry -> Cost * ST -> getMVEVectorCostFactor ( ) * LT . first ; return BaseT :: getArithmeticReductionCost ( Opcode , ValTy , IsPairwiseForm , CostKind ) ; } </s>

Calculate the cost of vector reduction intrinsics .


<s> int ARMTTIImpl :: getIntImmCostInst ( unsigned Opcode , unsigned Idx , const APInt & Imm , Type * Ty , TTI :: TargetCostKind CostKind , Instruction * Inst ) { if ( ( Opcode == Instruction :: SDiv || Opcode == Instruction :: UDiv || Opcode == Instruction :: SRem || Opcode == Instruction :: URem ) && Idx == 1 ) return 0 ; if ( Opcode == Instruction :: And ) { if ( Imm == 255 || Imm == 65535 ) return 0 ; return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( ~ Imm , Ty , CostKind ) ) ; } if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( - Imm , Ty , CostKind ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == 32 ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < 1 << 12 ) return 0 ; if ( ST -> isThumb ( ) && NegImm < 1 << 8 ) return 0 ; } if ( Opcode == Instruction :: Xor && Imm . isAllOnesValue ( ) ) return 0 ; if ( Inst && ( ( ST -> hasV6Ops ( ) && ! ST -> isThumb ( ) ) || ST -> isThumb2 ( ) ) && Ty -> getIntegerBitWidth ( ) <= 32 ) { if ( isSSATMinMaxPattern ( Inst , Imm ) || ( isa < ICmpInst > ( Inst ) && Inst -> hasOneUse ( ) && isSSATMinMaxPattern ( cast < Instruction > ( * Inst -> user_begin ( ) ) , Imm ) ) ) return 0 ; } return getIntImmCost ( Imm , Ty , CostKind ) ; } </s>

Return the expected cost of materialization for the given integer immediate of the specified type for a given instruction .


<s> int ARMTTIImpl :: getIntrinsicInstrCost ( const IntrinsicCostAttributes & ICA , TTI :: TargetCostKind CostKind ) { switch ( ICA . getID ( ) ) { case Intrinsic :: get_active_lane_mask : if ( ST -> hasMVEIntegerOps ( ) ) return 0 ; break ; case Intrinsic :: sadd_sat : case Intrinsic :: ssub_sat : case Intrinsic :: uadd_sat : case Intrinsic :: usub_sat : { if ( ! ST -> hasMVEIntegerOps ( ) ) break ; Type * VT = ICA . getReturnType ( ) ; if ( ! VT -> isVectorTy ( ) && ! ICA . getVectorFactor ( ) . isScalar ( ) ) VT = VectorType :: get ( VT , ICA . getVectorFactor ( ) ) ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , VT ) ; if ( LT . second == MVT :: v4i32 || LT . second == MVT :: v8i16 || LT . second == MVT :: v16i8 ) { unsigned Instrs = LT . second . getScalarSizeInBits ( ) == ICA . getReturnType ( ) -> getScalarSizeInBits ( ) ? 1 : 4 ; return LT . first * ST -> getMVEVectorCostFactor ( ) * Instrs ; } break ; } } return BaseT :: getIntrinsicInstrCost ( ICA , CostKind ) ; } </s>


<s>
void
ARMTTIImpl
::
getUnrollingPreferences
(
Loop
*
L
,
ScalarEvolution
&
SE
,
TTI
::
UnrollingPreferences
&
UP
)
{
if
(
!
ST
->
isMClass
(
)
)
return
BasicTTIImplBase
::
getUnrollingPreferences
(
L
,
SE
,
UP
)
;
UP
.
OptSizeThreshold
=
0
;
UP
.
PartialOptSizeThreshold
=
0
;
if
(
L
->
getHeader
(
)
->
getParent
(
)
->
hasOptSize
(
)
)
return
;
if
(
!
ST
->
isThumb2
(
)
)
return
;
SmallVector
<
BasicBlock
*
,
4
>
ExitingBlocks
;
L
->
getExitingBlocks
(
ExitingBlocks
)
;
LLVM_DEBUG
(
dbgs
(
)
<<
"Loop has:\n"
<<
"Blocks: "
<<
L
->
getNumBlocks
(
)
<<
"\n"
<<
"Exit blocks: "
<<
ExitingBlocks
.
size
(
)
<<
"\n"
)
;
if
(
ExitingBlocks
.
size
(
)
>
2
)
return
;
if
(
ST
->
hasBranchPredictor
(
)
&&
L
->
getNumBlocks
(
)
>
4
)
return
;
if
(
getBooleanLoopAttribute
(
L
,
"llvm.loop.isvectorized"
)
)
return
;
unsigned
Cost
=
0
;
for
(
auto
*
BB
:
L
->
getBlocks
(
)
)
{
for
(
auto
&
I
:
*
BB
)
{
if
(
I
.
getType
(
)
->
isVectorTy
(
)
)
return
;
if
(
isa
<
CallInst
>
(
I
)
||
isa
<
InvokeInst
>
(
I
)
)
{
if
(
const
Function
*
F
=
cast
<
CallBase
>
(
I
)
.
getCalledFunction
(
)
)
{
if
(
!
isLoweredToCall
(
F
)
)
continue
;
}
return
;
}
SmallVector
<
const
Value
*
,
4
>
Operands
(
I
.
operand_values
(
)
)
;
Cost
+=
getUserCost
(
&
I
,
Operands
,
TargetTransformInfo
::
TCK_SizeAndLatency
)
;
}
}
LLVM_DEBUG
(
dbgs
(
)
<<
"Cost of loop: "
<<
Cost
<<
"\n"
)
;
UP
.
Partial
=
true
;
UP
.
Runtime
=
true
;
UP
.
UpperBound
=
true
;
UP
.
UnrollRemainder
=
true
;
UP
.
DefaultUnrollRuntimeCount
=
4
;
UP
.
UnrollAndJam
=
true
;
UP
.
UnrollAndJamInnerLoopThreshold
=
60
;
if
(
Cost
<
12
)
UP
.
Force
=
true
;
}
</s>


"ARMHWLoops: Trip count does not fit into 32bits\n"


<s> int ARMTTIImpl :: getIntImmCost ( unsigned Opcode , unsigned Idx , const APInt & Imm , Type * Ty ) { if ( ( Opcode == Instruction :: SDiv || Opcode == Instruction :: UDiv || Opcode == Instruction :: SRem || Opcode == Instruction :: URem ) && Idx == 1 ) return 0 ; if ( Opcode == Instruction :: And ) { if ( Imm == 255 || Imm == 65535 ) return 0 ; return std :: min ( getIntImmCost ( Imm , Ty ) , getIntImmCost ( ~ Imm , Ty ) ) ; } if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty ) , getIntImmCost ( - Imm , Ty ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == 32 ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < 1 << 12 ) return 0 ; if ( ST -> isThumb ( ) && NegImm < 1 << 8 ) return 0 ; } if ( Opcode == Instruction :: Xor && Imm . isAllOnesValue ( ) ) return 0 ; return getIntImmCost ( Imm , Ty ) ; } </s>


<s>
void
ARMTTIImpl
::
getUnrollingPreferences
(
Loop
*
L
,
ScalarEvolution
&
SE
,
TTI
::
UnrollingPreferences
&
UP
)
{
if
(
!
ST
->
isMClass
(
)
)
return
BasicTTIImplBase
::
getUnrollingPreferences
(
L
,
SE
,
UP
)
;
UP
.
OptSizeThreshold
=
0
;
UP
.
PartialOptSizeThreshold
=
0
;
if
(
L
->
getHeader
(
)
->
getParent
(
)
->
hasOptSize
(
)
)
return
;
if
(
!
ST
->
isThumb2
(
)
)
return
;
SmallVector
<
BasicBlock
*
,
4
>
ExitingBlocks
;
L
->
getExitingBlocks
(
ExitingBlocks
)
;
LLVM_DEBUG
(
dbgs
(
)
<<
"Loop has:\n"
<<
"Blocks: "
<<
L
->
getNumBlocks
(
)
<<
"\n"
<<
"Exit blocks: "
<<
ExitingBlocks
.
size
(
)
<<
"\n"
)
;
if
(
ExitingBlocks
.
size
(
)
>
2
)
return
;
if
(
ST
->
hasBranchPredictor
(
)
&&
L
->
getNumBlocks
(
)
>
4
)
return
;
unsigned
Cost
=
0
;
for
(
auto
*
BB
:
L
->
getBlocks
(
)
)
{
for
(
auto
&
I
:
*
BB
)
{
if
(
isa
<
CallInst
>
(
I
)
||
isa
<
InvokeInst
>
(
I
)
)
{
ImmutableCallSite
CS
(
&
I
)
;
if
(
const
Function
*
F
=
CS
.
getCalledFunction
(
)
)
{
if
(
!
isLoweredToCall
(
F
)
)
continue
;
}
return
;
}
SmallVector
<
const
Value
*
,
4
>
Operands
(
I
.
value_op_begin
(
)
,
I
.
value_op_end
(
)
)
;
Cost
+=
getUserCost
(
&
I
,
Operands
)
;
}
}
LLVM_DEBUG
(
dbgs
(
)
<<
"Cost of loop: "
<<
Cost
<<
"\n"
)
;
UP
.
Partial
=
true
;
UP
.
Runtime
=
true
;
UP
.
UpperBound
=
true
;
UP
.
UnrollRemainder
=
true
;
UP
.
DefaultUnrollRuntimeCount
=
4
;
UP
.
UnrollAndJam
=
true
;
UP
.
UnrollAndJamInnerLoopThreshold
=
60
;
if
(
Cost
<
12
)
UP
.
Force
=
true
;
}
</s>


<s> bool ARMTTIImpl :: isLoweredToCall ( const Function * F ) { if ( ! F -> isIntrinsic ( ) ) return BaseT :: isLoweredToCall ( F ) ; if ( F -> getName ( ) . startswith ( "llvm.arm" ) ) return false ; switch ( F -> getIntrinsicID ( ) ) { default : break ; case Intrinsic :: powi : case Intrinsic :: sin : case Intrinsic :: cos : case Intrinsic :: pow : case Intrinsic :: log : case Intrinsic :: log10 : case Intrinsic :: log2 : case Intrinsic :: exp : case Intrinsic :: exp2 : return true ; case Intrinsic :: sqrt : case Intrinsic :: fabs : case Intrinsic :: copysign : case Intrinsic :: floor : case Intrinsic :: ceil : case Intrinsic :: trunc : case Intrinsic :: rint : case Intrinsic :: nearbyint : case Intrinsic :: round : case Intrinsic :: canonicalize : case Intrinsic :: lround : case Intrinsic :: llround : case Intrinsic :: lrint : case Intrinsic :: llrint : if ( F -> getReturnType ( ) -> isDoubleTy ( ) && ! ST -> hasFP64 ( ) ) return true ; if ( F -> getReturnType ( ) -> isHalfTy ( ) && ! ST -> hasFullFP16 ( ) ) return true ; return ! ST -> hasFPARMv8Base ( ) && ! ST -> hasVFP2Base ( ) ; case Intrinsic :: masked_store : case Intrinsic :: masked_load : case Intrinsic :: masked_gather : case Intrinsic :: masked_scatter : return ! ST -> hasMVEIntegerOps ( ) ; case Intrinsic :: sadd_with_overflow : case Intrinsic :: uadd_with_overflow : case Intrinsic :: ssub_with_overflow : case Intrinsic :: usub_with_overflow : case Intrinsic :: sadd_sat : case Intrinsic :: uadd_sat : case Intrinsic :: ssub_sat : case Intrinsic :: usub_sat : return false ; } return BaseT :: isLoweredToCall ( F ) ; } </s>


<s>
void
ARMTTIImpl
::
getUnrollingPreferences
(
Loop
*
L
,
ScalarEvolution
&
SE
,
TTI
::
UnrollingPreferences
&
UP
)
{
if
(
!
ST
->
isMClass
(
)
)
return
BasicTTIImplBase
::
getUnrollingPreferences
(
L
,
SE
,
UP
)
;
UP
.
OptSizeThreshold
=
0
;
UP
.
PartialOptSizeThreshold
=
0
;
if
(
L
->
getHeader
(
)
->
getParent
(
)
->
hasOptSize
(
)
)
return
;
if
(
!
ST
->
isThumb2
(
)
)
return
;
SmallVector
<
BasicBlock
*
,
4
>
ExitingBlocks
;
L
->
getExitingBlocks
(
ExitingBlocks
)
;
LLVM_DEBUG
(
dbgs
(
)
<<
"Loop has:\n"
<<
"Blocks: "
<<
L
->
getNumBlocks
(
)
<<
"\n"
<<
"Exit blocks: "
<<
ExitingBlocks
.
size
(
)
<<
"\n"
)
;
if
(
ExitingBlocks
.
size
(
)
>
2
)
return
;
if
(
ST
->
hasBranchPredictor
(
)
&&
L
->
getNumBlocks
(
)
>
4
)
return
;
if
(
getBooleanLoopAttribute
(
L
,
"llvm.loop.isvectorized"
)
)
return
;
unsigned
Cost
=
0
;
for
(
auto
*
BB
:
L
->
getBlocks
(
)
)
{
for
(
auto
&
I
:
*
BB
)
{
if
(
I
.
getType
(
)
->
isVectorTy
(
)
)
return
;
if
(
isa
<
CallInst
>
(
I
)
||
isa
<
InvokeInst
>
(
I
)
)
{
if
(
const
Function
*
F
=
cast
<
CallBase
>
(
I
)
.
getCalledFunction
(
)
)
{
if
(
!
isLoweredToCall
(
F
)
)
continue
;
}
return
;
}
SmallVector
<
const
Value
*
,
4
>
Operands
(
I
.
value_op_begin
(
)
,
I
.
value_op_end
(
)
)
;
Cost
+=
getUserCost
(
&
I
,
Operands
,
TargetTransformInfo
::
TCK_SizeAndLatency
)
;
}
}
LLVM_DEBUG
(
dbgs
(
)
<<
"Cost of loop: "
<<
Cost
<<
"\n"
)
;
UP
.
Partial
=
true
;
UP
.
Runtime
=
true
;
UP
.
UpperBound
=
true
;
UP
.
UnrollRemainder
=
true
;
UP
.
DefaultUnrollRuntimeCount
=
4
;
UP
.
UnrollAndJam
=
true
;
UP
.
UnrollAndJamInnerLoopThreshold
=
60
;
if
(
Cost
<
12
)
UP
.
Force
=
true
;
}
</s>


<s> bool isLegalMaskedGather ( Type * Ty , MaybeAlign Alignment ) { return false ; } </s>

Return true if the target supports masked gather .


<s> int ARMTTIImpl :: getIntrinsicInstrCost ( const IntrinsicCostAttributes & ICA , TTI :: TargetCostKind CostKind ) { if ( ST -> hasMVEIntegerOps ( ) && ICA . getID ( ) == Intrinsic :: get_active_lane_mask ) return 0 ; return BaseT :: getIntrinsicInstrCost ( ICA , CostKind ) ; } </s>


<s>
void
ARMTTIImpl
::
getUnrollingPreferences
(
Loop
*
L
,
ScalarEvolution
&
SE
,
TTI
::
UnrollingPreferences
&
UP
)
{
if
(
!
ST
->
isMClass
(
)
)
return
BasicTTIImplBase
::
getUnrollingPreferences
(
L
,
SE
,
UP
)
;
UP
.
OptSizeThreshold
=
0
;
UP
.
PartialOptSizeThreshold
=
0
;
if
(
L
->
getHeader
(
)
->
getParent
(
)
->
hasOptSize
(
)
)
return
;
if
(
!
ST
->
isThumb2
(
)
)
return
;
SmallVector
<
BasicBlock
*
,
4
>
ExitingBlocks
;
L
->
getExitingBlocks
(
ExitingBlocks
)
;
LLVM_DEBUG
(
dbgs
(
)
<<
"Loop has:\n"
<<
"Blocks: "
<<
L
->
getNumBlocks
(
)
<<
"\n"
<<
"Exit blocks: "
<<
ExitingBlocks
.
size
(
)
<<
"\n"
)
;
if
(
ExitingBlocks
.
size
(
)
>
2
)
return
;
if
(
ST
->
hasBranchPredictor
(
)
&&
L
->
getNumBlocks
(
)
>
4
)
return
;
unsigned
Cost
=
0
;
for
(
auto
*
BB
:
L
->
getBlocks
(
)
)
{
for
(
auto
&
I
:
*
BB
)
{
if
(
I
.
getType
(
)
->
isVectorTy
(
)
)
return
;
if
(
isa
<
CallInst
>
(
I
)
||
isa
<
InvokeInst
>
(
I
)
)
{
if
(
const
Function
*
F
=
cast
<
CallBase
>
(
I
)
.
getCalledFunction
(
)
)
{
if
(
!
isLoweredToCall
(
F
)
)
continue
;
}
return
;
}
SmallVector
<
const
Value
*
,
4
>
Operands
(
I
.
value_op_begin
(
)
,
I
.
value_op_end
(
)
)
;
Cost
+=
getUserCost
(
&
I
,
Operands
,
TargetTransformInfo
::
TCK_SizeAndLatency
)
;
}
}
LLVM_DEBUG
(
dbgs
(
)
<<
"Cost of loop: "
<<
Cost
<<
"\n"
)
;
UP
.
Partial
=
true
;
UP
.
Runtime
=
true
;
UP
.
UpperBound
=
true
;
UP
.
UnrollRemainder
=
true
;
UP
.
DefaultUnrollRuntimeCount
=
4
;
UP
.
UnrollAndJam
=
true
;
UP
.
UnrollAndJamInnerLoopThreshold
=
60
;
if
(
Cost
<
12
)
UP
.
Force
=
true
;
}
</s>


Query the target whether it would be profitable to convert the given loop into a hardware loop .


"preferPredicateOverEpilogue: not a single block "

"preferPredicateOverEpilogue: hardware-loop is not "

Query the target whether it would be prefered to create a predicated vector loop , which can avoid the need to emit a scalar epilogue loop .


<s> InstructionCost ARMTTIImpl :: getIntImmCostInst ( unsigned Opcode , unsigned Idx , const APInt & Imm , Type * Ty , TTI :: TargetCostKind CostKind , Instruction * Inst ) { if ( ( Opcode == Instruction :: SDiv || Opcode == Instruction :: UDiv || Opcode == Instruction :: SRem || Opcode == Instruction :: URem ) && Idx == 1 ) return 0 ; if ( Opcode == Instruction :: GetElementPtr && Idx != 0 ) return 0 ; if ( Opcode == Instruction :: And ) { if ( Imm == 255 || Imm == 65535 ) return 0 ; return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( ~ Imm , Ty , CostKind ) ) ; } if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( - Imm , Ty , CostKind ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == 32 ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < 1 << 12 ) return 0 ; if ( ST -> isThumb ( ) && NegImm < 1 << 8 ) return 0 ; } if ( Opcode == Instruction :: Xor && Imm . isAllOnesValue ( ) ) return 0 ; if ( Inst && ( ( ST -> hasV6Ops ( ) && ! ST -> isThumb ( ) ) || ST -> isThumb2 ( ) ) && Ty -> getIntegerBitWidth ( ) <= 32 ) { if ( isSSATMinMaxPattern ( Inst , Imm ) || ( isa < ICmpInst > ( Inst ) && Inst -> hasOneUse ( ) && isSSATMinMaxPattern ( cast < Instruction > ( * Inst -> user_begin ( ) ) , Imm ) ) ) return 0 ; } return getIntImmCost ( Imm , Ty , CostKind ) ; } </s>


<s>
int
ARMTTIImpl
::
getNumMemOps
(
const
IntrinsicInst
*
I
)
const
{
MemOp
MOp
;
unsigned
DstAddrSpace
=
~
0u
;
unsigned
SrcAddrSpace
=
~
0u
;
const
Function
*
F
=
I
->
getParent
(
)
->
getParent
(
)
;
if
(
const
auto
*
MC
=
dyn_cast
<
MemTransferInst
>
(
I
)
)
{
ConstantInt
*
C
=
dyn_cast
<
ConstantInt
>
(
MC
->
getLength
(
)
)
;
if
(
!
C
)
return
1
;
const
unsigned
Size
=
C
->
getValue
(
)
.
getZExtValue
(
)
;
const
Align
DstAlign
=
*
MC
->
getDestAlign
(
)
;
const
Align
SrcAlign
=
*
MC
->
getSourceAlign
(
)
;
MOp
=
MemOp
::
Copy
(
Size
,
false
,
DstAlign
,
SrcAlign
,
false
,
false
)
;
DstAddrSpace
=
MC
->
getDestAddressSpace
(
)
;
SrcAddrSpace
=
MC
->
getSourceAddressSpace
(
)
;
}
else
if
(
const
auto
*
MS
=
dyn_cast
<
MemSetInst
>
(
I
)
)
{
ConstantInt
*
C
=
dyn_cast
<
ConstantInt
>
(
MS
->
getLength
(
)
)
;
if
(
!
C
)
return
1
;
const
unsigned
Size
=
C
->
getValue
(
)
.
getZExtValue
(
)
;
const
Align
DstAlign
=
*
MS
->
getDestAlign
(
)
;
MOp
=
MemOp
::
Set
(
Size
,
false
,
DstAlign
,
false
,
false
)
;
DstAddrSpace
=
MS
->
getDestAddressSpace
(
)
;
}
else
llvm_unreachable
(
"Expected a memcpy/move or memset!"
)
;
unsigned
Limit
,
Factor
=
2
;
switch
(
I
->
getIntrinsicID
(
)
)
{
case
Intrinsic
::
memcpy
:
Limit
=
TLI
->
getMaxStoresPerMemcpy
(
F
->
hasMinSize
(
)
)
;
break
;
case
Intrinsic
::
memmove
:
Limit
=
TLI
->
getMaxStoresPerMemmove
(
F
->
hasMinSize
(
)
)
;
break
;
case
Intrinsic
::
memset
:
Limit
=
TLI
->
getMaxStoresPerMemset
(
F
->
hasMinSize
(
)
)
;
Factor
=
1
;
break
;
default
:
llvm_unreachable
(
"Expected a memcpy/move or memset!"
)
;
}
std
::
vector
<
EVT
>
MemOps
;
if
(
getTLI
(
)
->
findOptimalMemOpLowering
(
MemOps
,
Limit
,
MOp
,
DstAddrSpace
,
SrcAddrSpace
,
F
->
getAttributes
(
)
)
)
return
MemOps
.
size
(
)
*
Factor
;
return
1
;
}
</s>

Given a memcpy/memset/memmove instruction , return the number of memory operations performed , via querying findOptimalMemOpLowering .


<s>
void
ARMTTIImpl
::
getUnrollingPreferences
(
Loop
*
L
,
ScalarEvolution
&
SE
,
TTI
::
UnrollingPreferences
&
UP
)
{
UP
.
UpperBound
=
true
;
if
(
!
ST
->
isMClass
(
)
)
return
BasicTTIImplBase
::
getUnrollingPreferences
(
L
,
SE
,
UP
)
;
UP
.
OptSizeThreshold
=
0
;
UP
.
PartialOptSizeThreshold
=
0
;
if
(
L
->
getHeader
(
)
->
getParent
(
)
->
hasOptSize
(
)
)
return
;
SmallVector
<
BasicBlock
*
,
4
>
ExitingBlocks
;
L
->
getExitingBlocks
(
ExitingBlocks
)
;
LLVM_DEBUG
(
dbgs
(
)
<<
"Loop has:\n"
<<
"Blocks: "
<<
L
->
getNumBlocks
(
)
<<
"\n"
<<
"Exit blocks: "
<<
ExitingBlocks
.
size
(
)
<<
"\n"
)
;
if
(
ExitingBlocks
.
size
(
)
>
2
)
return
;
if
(
ST
->
hasBranchPredictor
(
)
&&
L
->
getNumBlocks
(
)
>
4
)
return
;
if
(
getBooleanLoopAttribute
(
L
,
"llvm.loop.isvectorized"
)
)
return
;
InstructionCost
Cost
=
0
;
for
(
auto
*
BB
:
L
->
getBlocks
(
)
)
{
for
(
auto
&
I
:
*
BB
)
{
if
(
I
.
getType
(
)
->
isVectorTy
(
)
)
return
;
if
(
isa
<
CallInst
>
(
I
)
||
isa
<
InvokeInst
>
(
I
)
)
{
if
(
const
Function
*
F
=
cast
<
CallBase
>
(
I
)
.
getCalledFunction
(
)
)
{
if
(
!
isLoweredToCall
(
F
)
)
continue
;
}
return
;
}
SmallVector
<
const
Value
*
,
4
>
Operands
(
I
.
operand_values
(
)
)
;
Cost
+=
getUserCost
(
&
I
,
Operands
,
TargetTransformInfo
::
TCK_SizeAndLatency
)
;
}
}
unsigned
UnrollCount
=
4
;
if
(
ST
->
isThumb1Only
(
)
)
{
unsigned
ExitingValues
=
0
;
SmallVector
<
BasicBlock
*
,
4
>
ExitBlocks
;
L
->
getExitBlocks
(
ExitBlocks
)
;
for
(
auto
*
Exit
:
ExitBlocks
)
{
unsigned
LiveOuts
=
count_if
(
Exit
->
phis
(
)
,
[
]
(
auto
&
PH
)
{
return
PH
.
getNumOperands
(
)
!=
1
||
!
isa
<
GetElementPtrInst
>
(
PH
.
getOperand
(
0
)
)
;
}
)
;
ExitingValues
=
ExitingValues
<
LiveOuts
LiveOuts
:
ExitingValues
;
}
if
(
ExitingValues
)
UnrollCount
/=
ExitingValues
;
if
(
UnrollCount
<=
1
)
return
;
}
LLVM_DEBUG
(
dbgs
(
)
<<
"Cost of loop: "
<<
Cost
<<
"\n"
)
;
LLVM_DEBUG
(
dbgs
(
)
<<
"Default Runtime Unroll Count: "
<<
UnrollCount
<<
"\n"
)
;
UP
.
Partial
=
true
;
UP
.
Runtime
=
true
;
UP
.
UnrollRemainder
=
true
;
UP
.
DefaultUnrollRuntimeCount
=
UnrollCount
;
UP
.
UnrollAndJam
=
true
;
UP
.
UnrollAndJamInnerLoopThreshold
=
60
;
if
(
Cost
<
12
)
UP
.
Force
=
true
;
}
</s>

Get target-customized preferences for the generic loop unrolling transformation .


<s> int ARMTTIImpl :: getIntImmCost ( const APInt & Imm , Type * Ty ) { assert ( Ty -> isIntegerTy ( ) ) ; unsigned Bits = Ty -> getPrimitiveSizeInBits ( ) ; if ( Bits == 0 || Bits > 64 ) return 4 ; int64_t SImmVal = Imm . getSExtValue ( ) ; uint64_t ZImmVal = Imm . getZExtValue ( ) ; if ( ! ST -> isThumb ( ) ) { if ( ( SImmVal >= 0 && SImmVal < 65536 ) || ( ARM_AM :: getSOImmVal ( ZImmVal ) != - 1 ) || ( ARM_AM :: getSOImmVal ( ~ ZImmVal ) != - 1 ) ) return 1 ; return ST -> hasV6T2Ops ( ) ? 2 : 3 ; } if ( ST -> isThumb2 ( ) ) { if ( ( SImmVal >= 0 && SImmVal < 65536 ) || ( ARM_AM :: getT2SOImmVal ( ZImmVal ) != - 1 ) || ( ARM_AM :: getT2SOImmVal ( ~ ZImmVal ) != - 1 ) ) return 1 ; return ST -> hasV6T2Ops ( ) ? 2 : 3 ; } if ( SImmVal >= 0 && SImmVal < 256 ) return 1 ; if ( ( ~ ZImmVal < 256 ) || ARM_AM :: isThumbImmShiftedVal ( ZImmVal ) ) return 2 ; return 3 ; } </s>


<s>
void
ARMTTIImpl
::
getUnrollingPreferences
(
Loop
*
L
,
ScalarEvolution
&
SE
,
TTI
::
UnrollingPreferences
&
UP
)
{
if
(
!
ST
->
isMClass
(
)
)
return
BasicTTIImplBase
::
getUnrollingPreferences
(
L
,
SE
,
UP
)
;
UP
.
OptSizeThreshold
=
0
;
UP
.
PartialOptSizeThreshold
=
0
;
if
(
L
->
getHeader
(
)
->
getParent
(
)
->
hasOptSize
(
)
)
return
;
if
(
!
ST
->
isThumb2
(
)
)
return
;
SmallVector
<
BasicBlock
*
,
4
>
ExitingBlocks
;
L
->
getExitingBlocks
(
ExitingBlocks
)
;
LLVM_DEBUG
(
dbgs
(
)
<<
"Loop has:\n"
<<
"Blocks: "
<<
L
->
getNumBlocks
(
)
<<
"\n"
<<
"Exit blocks: "
<<
ExitingBlocks
.
size
(
)
<<
"\n"
)
;
if
(
ExitingBlocks
.
size
(
)
>
2
)
return
;
if
(
ST
->
hasBranchPredictor
(
)
&&
L
->
getNumBlocks
(
)
>
4
)
return
;
unsigned
Cost
=
0
;
for
(
auto
*
BB
:
L
->
getBlocks
(
)
)
{
for
(
auto
&
I
:
*
BB
)
{
if
(
isa
<
CallInst
>
(
I
)
||
isa
<
InvokeInst
>
(
I
)
)
{
ImmutableCallSite
CS
(
&
I
)
;
if
(
const
Function
*
F
=
CS
.
getCalledFunction
(
)
)
{
if
(
!
isLoweredToCall
(
F
)
)
continue
;
}
return
;
}
SmallVector
<
const
Value
*
,
4
>
Operands
(
I
.
value_op_begin
(
)
,
I
.
value_op_end
(
)
)
;
Cost
+=
getUserCost
(
&
I
,
Operands
)
;
}
}
LLVM_DEBUG
(
dbgs
(
)
<<
"Cost of loop: "
<<
Cost
<<
"\n"
)
;
UP
.
Partial
=
true
;
UP
.
Runtime
=
true
;
UP
.
UnrollRemainder
=
true
;
UP
.
DefaultUnrollRuntimeCount
=
4
;
UP
.
UnrollAndJam
=
true
;
UP
.
UnrollAndJamInnerLoopThreshold
=
60
;
if
(
Cost
<
12
)
UP
.
Force
=
true
;
}
</s>


<s> InstructionCost ARMTTIImpl :: getArithmeticReductionCost ( unsigned Opcode , VectorType * ValTy , bool IsPairwiseForm , TTI :: TargetCostKind CostKind ) { EVT ValVT = TLI -> getValueType ( DL , ValTy ) ; int ISD = TLI -> InstructionOpcodeToISD ( Opcode ) ; if ( ! ST -> hasMVEIntegerOps ( ) || ! ValVT . isSimple ( ) || ISD != ISD :: ADD ) return BaseT :: getArithmeticReductionCost ( Opcode , ValTy , IsPairwiseForm , CostKind ) ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ; static const CostTblEntry CostTblAdd [ ] { { ISD :: ADD , MVT :: v16i8 , 1 } , { ISD :: ADD , MVT :: v8i16 , 1 } , { ISD :: ADD , MVT :: v4i32 , 1 } , } ; if ( const auto * Entry = CostTableLookup ( CostTblAdd , ISD , LT . second ) ) return Entry -> Cost * ST -> getMVEVectorCostFactor ( CostKind ) * LT . first ; return BaseT :: getArithmeticReductionCost ( Opcode , ValTy , IsPairwiseForm , CostKind ) ; } </s>


<s> InstructionCost ARMTTIImpl :: getIntImmCostInst ( unsigned Opcode , unsigned Idx , const APInt & Imm , Type * Ty , TTI :: TargetCostKind CostKind , Instruction * Inst ) { if ( ( Opcode == Instruction :: SDiv || Opcode == Instruction :: UDiv || Opcode == Instruction :: SRem || Opcode == Instruction :: URem ) && Idx == 1 ) return 0 ; if ( Opcode == Instruction :: And ) { if ( Imm == 255 || Imm == 65535 ) return 0 ; return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( ~ Imm , Ty , CostKind ) ) ; } if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( - Imm , Ty , CostKind ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == 32 ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < 1 << 12 ) return 0 ; if ( ST -> isThumb ( ) && NegImm < 1 << 8 ) return 0 ; } if ( Opcode == Instruction :: Xor && Imm . isAllOnesValue ( ) ) return 0 ; if ( Inst && ( ( ST -> hasV6Ops ( ) && ! ST -> isThumb ( ) ) || ST -> isThumb2 ( ) ) && Ty -> getIntegerBitWidth ( ) <= 32 ) { if ( isSSATMinMaxPattern ( Inst , Imm ) || ( isa < ICmpInst > ( Inst ) && Inst -> hasOneUse ( ) && isSSATMinMaxPattern ( cast < Instruction > ( * Inst -> user_begin ( ) ) , Imm ) ) ) return 0 ; } return getIntImmCost ( Imm , Ty , CostKind ) ; } </s>


<s>
void
ARMTTIImpl
::
getUnrollingPreferences
(
Loop
*
L
,
ScalarEvolution
&
SE
,
TTI
::
UnrollingPreferences
&
UP
)
{
UP
.
UpperBound
=
true
;
if
(
!
ST
->
isMClass
(
)
)
return
BasicTTIImplBase
::
getUnrollingPreferences
(
L
,
SE
,
UP
)
;
UP
.
OptSizeThreshold
=
0
;
UP
.
PartialOptSizeThreshold
=
0
;
if
(
L
->
getHeader
(
)
->
getParent
(
)
->
hasOptSize
(
)
)
return
;
SmallVector
<
BasicBlock
*
,
4
>
ExitingBlocks
;
L
->
getExitingBlocks
(
ExitingBlocks
)
;
LLVM_DEBUG
(
dbgs
(
)
<<
"Loop has:\n"
<<
"Blocks: "
<<
L
->
getNumBlocks
(
)
<<
"\n"
<<
"Exit blocks: "
<<
ExitingBlocks
.
size
(
)
<<
"\n"
)
;
if
(
ExitingBlocks
.
size
(
)
>
2
)
return
;
if
(
ST
->
hasBranchPredictor
(
)
&&
L
->
getNumBlocks
(
)
>
4
)
return
;
if
(
getBooleanLoopAttribute
(
L
,
"llvm.loop.isvectorized"
)
)
return
;
InstructionCost
Cost
=
0
;
for
(
auto
*
BB
:
L
->
getBlocks
(
)
)
{
for
(
auto
&
I
:
*
BB
)
{
if
(
I
.
getType
(
)
->
isVectorTy
(
)
)
return
;
if
(
isa
<
CallInst
>
(
I
)
||
isa
<
InvokeInst
>
(
I
)
)
{
if
(
const
Function
*
F
=
cast
<
CallBase
>
(
I
)
.
getCalledFunction
(
)
)
{
if
(
!
isLoweredToCall
(
F
)
)
continue
;
}
return
;
}
SmallVector
<
const
Value
*
,
4
>
Operands
(
I
.
operand_values
(
)
)
;
Cost
+=
getUserCost
(
&
I
,
Operands
,
TargetTransformInfo
::
TCK_SizeAndLatency
)
;
}
}
LLVM_DEBUG
(
dbgs
(
)
<<
"Cost of loop: "
<<
Cost
<<
"\n"
)
;
UP
.
Partial
=
true
;
UP
.
Runtime
=
true
;
UP
.
UnrollRemainder
=
true
;
UP
.
DefaultUnrollRuntimeCount
=
4
;
UP
.
UnrollAndJam
=
true
;
UP
.
UnrollAndJamInnerLoopThreshold
=
60
;
if
(
Cost
<
12
)
UP
.
Force
=
true
;
}
</s>


<s> unsigned ARMTTIImpl :: getFPOpCost ( Type * Ty ) { if ( ST -> hasVFP2 ( ) && ! ST -> isThumb1Only ( ) ) { if ( Ty -> isFloatTy ( ) ) { return TargetTransformInfo :: TCC_Basic ; } if ( Ty -> isDoubleTy ( ) ) { return ST -> isFPOnlySP ( ) ? TargetTransformInfo :: TCC_Expensive : TargetTransformInfo :: TCC_Basic ; } } return TargetTransformInfo :: TCC_Expensive ; } </s>

Return the expected cost of supporting the floating point operation of the specified type .


<s> virtual void * getAdjustedAnalysisPointer ( const void * ID ) { if ( ID == & TargetTransformInfo :: ID ) return ( TargetTransformInfo * ) this ; return this ; } </s>


<s> virtual void getAnalysisUsage ( AnalysisUsage & AU ) const { TargetTransformInfo :: getAnalysisUsage ( AU ) ; } </s>


<s> unsigned ARMTTI :: getIntImmCost ( const APInt & Imm , Type * Ty ) const { assert ( Ty -> isIntegerTy ( ) ) ; unsigned Bits = Ty -> getPrimitiveSizeInBits ( ) ; if ( Bits == 0 || Bits > 32 ) return 4 ; int32_t SImmVal = Imm . getSExtValue ( ) ; uint32_t ZImmVal = Imm . getZExtValue ( ) ; if ( ! ST -> isThumb ( ) ) { if ( ( SImmVal >= 0 && SImmVal < 65536 ) || ( ARM_AM :: getSOImmVal ( ZImmVal ) != - 1 ) || ( ARM_AM :: getSOImmVal ( ~ ZImmVal ) != - 1 ) ) return 1 ; return ST -> hasV6T2Ops ( ) ? 2 : 3 ; } else if ( ST -> isThumb2 ( ) ) { if ( ( SImmVal >= 0 && SImmVal < 65536 ) || ( ARM_AM :: getT2SOImmVal ( ZImmVal ) != - 1 ) || ( ARM_AM :: getT2SOImmVal ( ~ ZImmVal ) != - 1 ) ) return 1 ; return ST -> hasV6T2Ops ( ) ? 2 : 3 ; } else { if ( SImmVal >= 0 && SImmVal < 256 ) return 1 ; if ( ( ~ ZImmVal < 256 ) || ARM_AM :: isThumbImmShiftedVal ( ZImmVal ) ) return 2 ; return 3 ; } return 2 ; } </s>


<s> unsigned getNumberOfRegisters ( bool Vector ) { if ( Vector ) { if ( ST -> hasNEON ( ) ) return 16 ; return 0 ; } if ( ST -> isThumb1Only ( ) ) return 8 ; return 13 ; } </s>


<s> virtual void initializePass ( ) { pushTTIStack ( this ) ; } </s>


<s> bool isFPVectorizationPotentiallyUnsafe ( ) { return ! ST -> isTargetDarwin ( ) ; } </s>

Indicate that it is potentially unsafe to automatically vectorize floating-point operations because the semantics of vector and scalar floating-point semantics may differ .


<s> bool shouldBuildLookupTablesForConstant ( Constant * C ) const { if ( ST -> isROPI ( ) || ST -> isRWPI ( ) || ST -> isPIP ( ) ) return ! C -> needsRelocation ( ) ; return true ; } </s>


<s> const std :: string & ARMToARMStub :: name ( ) const { return m_Name ; } </s>

Gets the name of the pass we are mixed into .


<s> size_t ARMToARMStub :: size ( ) const { return m_Size ; } </s>

The number of name/type pairs is returned .


<s> const std :: string & ARMToTHMStub :: name ( ) const { return m_Name ; } </s>


<s> size_t ARMToTHMStub :: size ( ) const { return m_Size ; } </s>


<s> void UnwindOpcodeAssembler :: EmitRegSave ( uint32_t RegSave ) { if ( RegSave == 0u ) return ; if ( RegSave & ( 1u << 4 ) ) { uint32_t Mask = RegSave & 0xff0u ; uint32_t Range = countTrailingOnes ( Mask >> 5 ) ; Mask &= ~ ( 0xffffffe0u << Range ) ; uint32_t UnmaskedReg = RegSave & 0xfff0u & ( ~ Mask ) ; if ( UnmaskedReg == 0u ) { EmitInt8 ( ARM :: EHABI :: UNWIND_OPCODE_POP_REG_RANGE_R4 | Range ) ; RegSave &= 0x000fu ; } else if ( UnmaskedReg == ( 1u << 14 ) ) { EmitInt8 ( ARM :: EHABI :: UNWIND_OPCODE_POP_REG_RANGE_R4_R14 | Range ) ; RegSave &= 0x000fu ; } } if ( ( RegSave & 0xfff0u ) != 0 ) EmitInt16 ( ARM :: EHABI :: UNWIND_OPCODE_POP_REG_MASK_R4 | ( RegSave >> 4 ) ) ; if ( ( RegSave & 0x000fu ) != 0 ) EmitInt16 ( ARM :: EHABI :: UNWIND_OPCODE_POP_REG_MASK | ( RegSave & 0x000fu ) ) ; } </s>

Emit unwind opcodes for .save directives .


<s> void UnwindOpcodeAssembler :: EmitSetSP ( uint16_t Reg ) { EmitInt8 ( ARM :: EHABI :: UNWIND_OPCODE_SET_VSP | Reg ) ; } </s>

Emit unwind opcodes to copy address from source register to $ sp .


<s> void UnwindOpcodeAssembler :: EmitSPOffset ( int64_t Offset ) { if ( Offset > 0x200 ) { uint8_t Buff [ 16 ] ; Buff [ 0 ] = ARM :: EHABI :: UNWIND_OPCODE_INC_VSP_ULEB128 ; size_t ULEBSize = encodeULEB128 ( ( Offset - 0x204 ) >> 2 , Buff + 1 ) ; EmitBytes ( Buff , ULEBSize + 1 ) ; } else if ( Offset > 0 ) { if ( Offset > 0x100 ) { EmitInt8 ( ARM :: EHABI :: UNWIND_OPCODE_INC_VSP | 0x3fu ) ; Offset -= 0x100 ; } EmitInt8 ( ARM :: EHABI :: UNWIND_OPCODE_INC_VSP | static_cast < uint8_t > ( ( Offset - 4 ) >> 2 ) ) ; } else if ( Offset < 0 ) { while ( Offset < - 0x100 ) { EmitInt8 ( ARM :: EHABI :: UNWIND_OPCODE_DEC_VSP | 0x3fu ) ; Offset += 0x100 ; } EmitInt8 ( ARM :: EHABI :: UNWIND_OPCODE_DEC_VSP | static_cast < uint8_t > ( ( ( - Offset ) - 4 ) >> 2 ) ) ; } } </s>

Emit unwind opcodes to add $ sp with an offset .


Emit unwind opcodes for .vsave directives .


"too many opcodes for __aeabi_unwind_cpp_pr0"

Finalize the unwind opcode sequence for emitBytes ( )


<s> void Reset ( ) { Ops . clear ( ) ; OpBegins . clear ( ) ; OpBegins . push_back ( 0 ) ; HasPersonality = false ; } </s>


<s> void setPersonality ( const MCSymbol * Per ) { HasPersonality = true ; } </s>


<s> void UnwindOpcodeAssembler :: EmitRegSave ( uint32_t RegSave ) { if ( RegSave == 0u ) { EmitInt8 ( ARM :: EHABI :: UNWIND_OPCODE_POP_RA_AUTH_CODE ) ; return ; } if ( RegSave & ( 1u << 4 ) ) { uint32_t Mask = RegSave & 0xff0u ; uint32_t Range = countTrailingOnes ( Mask >> 5 ) ; Mask &= ~ ( 0xffffffe0u << Range ) ; uint32_t UnmaskedReg = RegSave & 0xfff0u & ( ~ Mask ) ; if ( UnmaskedReg == 0u ) { EmitInt8 ( ARM :: EHABI :: UNWIND_OPCODE_POP_REG_RANGE_R4 | Range ) ; RegSave &= 0x000fu ; } else if ( UnmaskedReg == ( 1u << 14 ) ) { EmitInt8 ( ARM :: EHABI :: UNWIND_OPCODE_POP_REG_RANGE_R4_R14 | Range ) ; RegSave &= 0x000fu ; } } if ( ( RegSave & 0xfff0u ) != 0 ) EmitInt16 ( ARM :: EHABI :: UNWIND_OPCODE_POP_REG_MASK_R4 | ( RegSave >> 4 ) ) ; if ( ( RegSave & 0x000fu ) != 0 ) EmitInt16 ( ARM :: EHABI :: UNWIND_OPCODE_POP_REG_MASK | ( RegSave & 0x000fu ) ) ; } </s>


<s> void UnwindOpcodeAssembler :: EmitSPOffset ( int64_t Offset ) { if ( Offset > 0x200 ) { uint8_t Buff [ 16 ] ; Buff [ 0 ] = ARM :: EHABI :: UNWIND_OPCODE_INC_VSP_ULEB128 ; size_t ULEBSize = encodeULEB128 ( ( Offset - 0x204 ) >> 2 , Buff + 1 ) ; emitBytes ( Buff , ULEBSize + 1 ) ; } else if ( Offset > 0 ) { if ( Offset > 0x100 ) { EmitInt8 ( ARM :: EHABI :: UNWIND_OPCODE_INC_VSP | 0x3fu ) ; Offset -= 0x100 ; } EmitInt8 ( ARM :: EHABI :: UNWIND_OPCODE_INC_VSP | static_cast < uint8_t > ( ( Offset - 4 ) >> 2 ) ) ; } else if ( Offset < 0 ) { while ( Offset < - 0x100 ) { EmitInt8 ( ARM :: EHABI :: UNWIND_OPCODE_DEC_VSP | 0x3fu ) ; Offset += 0x100 ; } EmitInt8 ( ARM :: EHABI :: UNWIND_OPCODE_DEC_VSP | static_cast < uint8_t > ( ( ( - Offset ) - 4 ) >> 2 ) ) ; } } </s>


<s> void emitBytes ( const uint8_t * Opcode , size_t Size ) { Ops . insert ( Ops . end ( ) , Opcode , Opcode + Size ) ; OpBegins . push_back ( OpBegins . back ( ) + Size ) ; } </s>

Emit the bytes in Data into the output .


<s> void UnwindOpcodeAssembler :: EmitRegSave ( uint32_t RegSave ) { if ( RegSave == 0u ) return ; if ( RegSave & ( 1u << 4 ) ) { uint32_t Range = 0 ; uint32_t Mask = ( 1u << 4 ) ; for ( uint32_t Bit = ( 1u << 5 ) ; Bit < ( 1u << 12 ) ; Bit <<= 1 ) { if ( ( RegSave & Bit ) == 0u ) break ; ++ Range ; Mask |= Bit ; } uint32_t UnmaskedReg = RegSave & 0xfff0u & ( ~ Mask ) ; if ( UnmaskedReg == 0u ) { EmitInt8 ( ARM :: EHABI :: UNWIND_OPCODE_POP_REG_RANGE_R4 | Range ) ; RegSave &= 0x000fu ; } else if ( UnmaskedReg == ( 1u << 14 ) ) { EmitInt8 ( ARM :: EHABI :: UNWIND_OPCODE_POP_REG_RANGE_R4_R14 | Range ) ; RegSave &= 0x000fu ; } } if ( ( RegSave & 0xfff0u ) != 0 ) EmitInt16 ( ARM :: EHABI :: UNWIND_OPCODE_POP_REG_MASK_R4 | ( RegSave >> 4 ) ) ; if ( ( RegSave & 0x000fu ) != 0 ) EmitInt16 ( ARM :: EHABI :: UNWIND_OPCODE_POP_REG_MASK | ( RegSave & 0x000fu ) ) ; } </s>


<s> void EmitRaw ( const SmallVectorImpl < uint8_t > & Opcodes ) { Ops . insert ( Ops . end ( ) , Opcodes . begin ( ) , Opcodes . end ( ) ) ; OpBegins . push_back ( OpBegins . back ( ) + Opcodes . size ( ) ) ; } </s>


<s> void Reset ( ) { Ops . clear ( ) ; OpBegins . clear ( ) ; OpBegins . push_back ( 0 ) ; HasPersonality = 0 ; } </s>


<s> void setPersonality ( const MCSymbol * Per ) { HasPersonality = 1 ; } </s>


<s> bool ARMWinCOFFObjectWriter :: recordRelocation ( const MCFixup & Fixup ) const { return static_cast < unsigned > ( Fixup . getKind ( ) ) != ARM :: fixup_t2_movt_hi16 ; } </s>


<s> void ARMWinCOFFStreamer :: finishImpl ( ) { emitFrames ( nullptr ) ; emitWindowsUnwindTables ( ) ; MCWinCOFFStreamer :: finishImpl ( ) ; } </s>


<s> void ARMWinCOFFStreamer :: emitThumbFunc ( MCSymbol * Symbol ) { getAssembler ( ) . setIsThumbFunc ( Symbol ) ; } </s>

Note in the output that the specified Func is a Thumb mode function ( ARM target only ) .


<s> void ARMWinCOFFStreamer :: finishImpl ( ) { emitFrames ( nullptr ) ; MCWinCOFFStreamer :: finishImpl ( ) ; } </s>


<s> bool MLxExpansion :: runOnMachineFunction ( MachineFunction & Fn ) { if ( skipFunction ( * Fn . getFunction ( ) ) ) return false ; TII = static_cast < const ARMBaseInstrInfo * > ( Fn . getSubtarget ( ) . getInstrInfo ( ) ) ; TRI = Fn . getSubtarget ( ) . getRegisterInfo ( ) ; MRI = & Fn . getRegInfo ( ) ; const ARMSubtarget * STI = & Fn . getSubtarget < ARMSubtarget > ( ) ; if ( ! STI -> expandMLx ( ) ) return false ; isLikeA9 = STI -> isLikeA9 ( ) || STI -> isSwift ( ) ; isSwift = STI -> isSwift ( ) ; bool Modified = false ; for ( MachineBasicBlock & MBB : Fn ) Modified |= ExpandFPMLxInstructions ( MBB ) ; return Modified ; } </s>

runOnMachineFunction - Emit the function body .


<s>
const
char
*
getPassName
(
)
const
override
{
return
"ARM MLA / MLS expansion pass"
;
}
</s>

getPassName - Return a nice clean name for a pass .


<s> bool MLxExpansion :: runOnMachineFunction ( MachineFunction & Fn ) { TII = static_cast < const ARMBaseInstrInfo * > ( Fn . getTarget ( ) . getInstrInfo ( ) ) ; TRI = Fn . getTarget ( ) . getRegisterInfo ( ) ; MRI = & Fn . getRegInfo ( ) ; const ARMSubtarget * STI = & Fn . getTarget ( ) . getSubtarget < ARMSubtarget > ( ) ; isLikeA9 = STI -> isLikeA9 ( ) || STI -> isSwift ( ) ; isSwift = STI -> isSwift ( ) ; bool Modified = false ; for ( MachineBasicBlock & MBB : Fn ) Modified |= ExpandFPMLxInstructions ( MBB ) ; return Modified ; } </s>


<s>
virtual
const
char
*
getPassName
(
)
const
{
return
"ARM MLA / MLS expansion pass"
;
}
</s>


<s> bool MLxExpansion :: runOnMachineFunction ( MachineFunction & Fn ) { TII = static_cast < const ARMBaseInstrInfo * > ( Fn . getTarget ( ) . getInstrInfo ( ) ) ; TRI = Fn . getTarget ( ) . getRegisterInfo ( ) ; MRI = & Fn . getRegInfo ( ) ; bool Modified = false ; for ( MachineFunction :: iterator MFI = Fn . begin ( ) , E = Fn . end ( ) ; MFI != E ; ++ MFI ) { MachineBasicBlock & MBB = * MFI ; Modified |= ExpandFPMLxInstructions ( MBB ) ; } return Modified ; } </s>


<s>
StringRef
getPassName
(
)
const
override
{
return
"ARM MLA / MLS expansion pass"
;
}
</s>


<s> bool MLxExpansion :: runOnMachineFunction ( MachineFunction & Fn ) { if ( skipFunction ( Fn . getFunction ( ) ) ) return false ; TII = static_cast < const ARMBaseInstrInfo * > ( Fn . getSubtarget ( ) . getInstrInfo ( ) ) ; TRI = Fn . getSubtarget ( ) . getRegisterInfo ( ) ; MRI = & Fn . getRegInfo ( ) ; const ARMSubtarget * STI = & Fn . getSubtarget < ARMSubtarget > ( ) ; if ( ! STI -> expandMLx ( ) ) return false ; isLikeA9 = STI -> isLikeA9 ( ) || STI -> isSwift ( ) ; isSwift = STI -> isSwift ( ) ; bool Modified = false ; for ( MachineBasicBlock & MBB : Fn ) Modified |= ExpandFPMLxInstructions ( MBB ) ; return Modified ; } </s>


<s> bool MLxExpansion :: runOnMachineFunction ( MachineFunction & Fn ) { TII = static_cast < const ARMBaseInstrInfo * > ( Fn . getTarget ( ) . getInstrInfo ( ) ) ; TRI = Fn . getTarget ( ) . getRegisterInfo ( ) ; MRI = & Fn . getRegInfo ( ) ; const ARMSubtarget * STI = & Fn . getTarget ( ) . getSubtarget < ARMSubtarget > ( ) ; isLikeA9 = STI -> isLikeA9 ( ) || STI -> isSwift ( ) ; isSwift = STI -> isSwift ( ) ; bool Modified = false ; for ( MachineFunction :: iterator MFI = Fn . begin ( ) , E = Fn . end ( ) ; MFI != E ; ++ MFI ) { MachineBasicBlock & MBB = * MFI ; Modified |= ExpandFPMLxInstructions ( MBB ) ; } return Modified ; } </s>


<s> bool MLxExpansion :: runOnMachineFunction ( MachineFunction & Fn ) { TII = static_cast < const ARMBaseInstrInfo * > ( Fn . getTarget ( ) . getInstrInfo ( ) ) ; TRI = Fn . getTarget ( ) . getRegisterInfo ( ) ; MRI = & Fn . getRegInfo ( ) ; const ARMSubtarget * STI = & Fn . getTarget ( ) . getSubtarget < ARMSubtarget > ( ) ; isA9 = STI -> isCortexA9 ( ) ; bool Modified = false ; for ( MachineFunction :: iterator MFI = Fn . begin ( ) , E = Fn . end ( ) ; MFI != E ; ++ MFI ) { MachineBasicBlock & MBB = * MFI ; Modified |= ExpandFPMLxInstructions ( MBB ) ; } return Modified ; } </s>


<s> bool MLxExpansion :: runOnMachineFunction ( MachineFunction & Fn ) { TII = static_cast < const ARMBaseInstrInfo * > ( Fn . getSubtarget ( ) . getInstrInfo ( ) ) ; TRI = Fn . getSubtarget ( ) . getRegisterInfo ( ) ; MRI = & Fn . getRegInfo ( ) ; const ARMSubtarget * STI = & Fn . getTarget ( ) . getSubtarget < ARMSubtarget > ( ) ; isLikeA9 = STI -> isLikeA9 ( ) || STI -> isSwift ( ) ; isSwift = STI -> isSwift ( ) ; bool Modified = false ; for ( MachineBasicBlock & MBB : Fn ) Modified |= ExpandFPMLxInstructions ( MBB ) ; return Modified ; } </s>


<s> bool MLxExpansion :: runOnMachineFunction ( MachineFunction & Fn ) { TII = static_cast < const ARMBaseInstrInfo * > ( Fn . getSubtarget ( ) . getInstrInfo ( ) ) ; TRI = Fn . getSubtarget ( ) . getRegisterInfo ( ) ; MRI = & Fn . getRegInfo ( ) ; const ARMSubtarget * STI = & Fn . getSubtarget < ARMSubtarget > ( ) ; if ( ! STI -> isCortexA9 ( ) ) return false ; isLikeA9 = STI -> isLikeA9 ( ) || STI -> isSwift ( ) ; isSwift = STI -> isSwift ( ) ; bool Modified = false ; for ( MachineBasicBlock & MBB : Fn ) Modified |= ExpandFPMLxInstructions ( MBB ) ; return Modified ; } </s>


<s> bool MLxExpansion :: runOnMachineFunction ( MachineFunction & Fn ) { if ( skipFunction ( * Fn . getFunction ( ) ) ) return false ; TII = static_cast < const ARMBaseInstrInfo * > ( Fn . getSubtarget ( ) . getInstrInfo ( ) ) ; TRI = Fn . getSubtarget ( ) . getRegisterInfo ( ) ; MRI = & Fn . getRegInfo ( ) ; const ARMSubtarget * STI = & Fn . getSubtarget < ARMSubtarget > ( ) ; if ( ! STI -> isCortexA9 ( ) ) return false ; isLikeA9 = STI -> isLikeA9 ( ) || STI -> isSwift ( ) ; isSwift = STI -> isSwift ( ) ; bool Modified = false ; for ( MachineBasicBlock & MBB : Fn ) Modified |= ExpandFPMLxInstructions ( MBB ) ; return Modified ; } </s>


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . setPreservesCFG ( ) ; AU . addRequired < TargetPassConfig > ( ) ; AU . addRequired < LoopInfoWrapperPass > ( ) ; FunctionPass :: getAnalysisUsage ( AU ) ; } </s>

getAnalysisUsage - Subclasses that override getAnalysisUsage must call this .


<s>
StringRef
getPassName
(
)
const
override
{
return
"MVE gather/scatter lowering"
;
}
</s>


<s> bool MVEGatherScatterLowering :: runOnFunction ( Function & F ) { if ( ! EnableMaskedGatherScatters ) return false ; auto & TPC = getAnalysis < TargetPassConfig > ( ) ; auto & TM = TPC . getTM < TargetMachine > ( ) ; auto * ST = & TM . getSubtarget < ARMSubtarget > ( F ) ; if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; DL = & F . getParent ( ) -> getDataLayout ( ) ; SmallVector < IntrinsicInst * , 4 > Gathers ; SmallVector < IntrinsicInst * , 4 > Scatters ; bool Changed = false ; for ( BasicBlock & BB : F ) { Changed |= SimplifyInstructionsInBlock ( & BB ) ; for ( Instruction & I : BB ) { IntrinsicInst * II = dyn_cast < IntrinsicInst > ( & I ) ; if ( II && II -> getIntrinsicID ( ) == Intrinsic :: masked_gather && isa < FixedVectorType > ( II -> getType ( ) ) ) { Gathers . push_back ( II ) ; Changed |= optimiseAddress ( II -> getArgOperand ( 0 ) , II -> getParent ( ) , LI ) ; } else if ( II && II -> getIntrinsicID ( ) == Intrinsic :: masked_scatter && isa < FixedVectorType > ( II -> getArgOperand ( 0 ) -> getType ( ) ) ) { Scatters . push_back ( II ) ; Changed |= optimiseAddress ( II -> getArgOperand ( 1 ) , II -> getParent ( ) , LI ) ; } } } for ( unsigned i = 0 ; i < Gathers . size ( ) ; i ++ ) { IntrinsicInst * I = Gathers [ i ] ; Instruction * L = lowerGather ( I ) ; if ( L == nullptr ) continue ; SimplifyInstructionsInBlock ( L -> getParent ( ) ) ; Changed = true ; } for ( unsigned i = 0 ; i < Scatters . size ( ) ; i ++ ) { IntrinsicInst * I = Scatters [ i ] ; Instruction * S = lowerScatter ( I ) ; if ( S == nullptr ) continue ; SimplifyInstructionsInBlock ( S -> getParent ( ) ) ; Changed = true ; } return Changed ; } </s>

runOnFunction - Virtual method overriden by subclasses to do the per-function processing of the pass .


<s> bool MVEGatherScatterLowering :: runOnFunction ( Function & F ) { if ( ! EnableMaskedGatherScatters ) return false ; auto & TPC = getAnalysis < TargetPassConfig > ( ) ; auto & TM = TPC . getTM < TargetMachine > ( ) ; auto * ST = & TM . getSubtarget < ARMSubtarget > ( F ) ; if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; SmallVector < IntrinsicInst * , 4 > Gathers ; SmallVector < IntrinsicInst * , 4 > Scatters ; bool Changed = false ; for ( BasicBlock & BB : F ) { Changed |= SimplifyInstructionsInBlock ( & BB ) ; for ( Instruction & I : BB ) { IntrinsicInst * II = dyn_cast < IntrinsicInst > ( & I ) ; if ( II && II -> getIntrinsicID ( ) == Intrinsic :: masked_gather && isa < FixedVectorType > ( II -> getType ( ) ) ) { Gathers . push_back ( II ) ; Changed |= optimiseAddress ( II -> getArgOperand ( 0 ) , II -> getParent ( ) , LI ) ; } else if ( II && II -> getIntrinsicID ( ) == Intrinsic :: masked_scatter && isa < FixedVectorType > ( II -> getArgOperand ( 0 ) -> getType ( ) ) ) { Scatters . push_back ( II ) ; Changed |= optimiseAddress ( II -> getArgOperand ( 1 ) , II -> getParent ( ) , LI ) ; } } } for ( unsigned i = 0 ; i < Gathers . size ( ) ; i ++ ) { IntrinsicInst * I = Gathers [ i ] ; Instruction * L = lowerGather ( I ) ; if ( L == nullptr ) continue ; SimplifyInstructionsInBlock ( L -> getParent ( ) ) ; Changed = true ; } for ( unsigned i = 0 ; i < Scatters . size ( ) ; i ++ ) { IntrinsicInst * I = Scatters [ i ] ; Instruction * S = lowerScatter ( I ) ; if ( S == nullptr ) continue ; SimplifyInstructionsInBlock ( S -> getParent ( ) ) ; Changed = true ; } return Changed ; } </s>


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . setPreservesCFG ( ) ; AU . addRequired < TargetPassConfig > ( ) ; FunctionPass :: getAnalysisUsage ( AU ) ; } </s>


<s> bool MVEGatherScatterLowering :: runOnFunction ( Function & F ) { if ( ! EnableMaskedGatherScatters ) return false ; auto & TPC = getAnalysis < TargetPassConfig > ( ) ; auto & TM = TPC . getTM < TargetMachine > ( ) ; auto * ST = & TM . getSubtarget < ARMSubtarget > ( F ) ; if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; SmallVector < IntrinsicInst * , 4 > Gathers ; for ( BasicBlock & BB : F ) { for ( Instruction & I : BB ) { IntrinsicInst * II = dyn_cast < IntrinsicInst > ( & I ) ; if ( II && II -> getIntrinsicID ( ) == Intrinsic :: masked_gather ) Gathers . push_back ( II ) ; } } if ( Gathers . empty ( ) ) return false ; for ( IntrinsicInst * I : Gathers ) lowerGather ( I ) ; return true ; } </s>


<s> bool MVEGatherScatterLowering :: runOnFunction ( Function & F ) { if ( ! EnableMaskedGatherScatters ) return false ; auto & TPC = getAnalysis < TargetPassConfig > ( ) ; auto & TM = TPC . getTM < TargetMachine > ( ) ; auto * ST = & TM . getSubtarget < ARMSubtarget > ( F ) ; if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; SmallVector < IntrinsicInst * , 4 > Gathers ; SmallVector < IntrinsicInst * , 4 > Scatters ; bool Changed = false ; for ( BasicBlock & BB : F ) { Changed |= SimplifyInstructionsInBlock ( & BB ) ; for ( Instruction & I : BB ) { IntrinsicInst * II = dyn_cast < IntrinsicInst > ( & I ) ; if ( II && II -> getIntrinsicID ( ) == Intrinsic :: masked_gather && isa < FixedVectorType > ( II -> getType ( ) ) ) { Gathers . push_back ( II ) ; Changed |= optimiseAddress ( II -> getArgOperand ( 0 ) , II -> getParent ( ) , LI ) ; } else if ( II && II -> getIntrinsicID ( ) == Intrinsic :: masked_scatter && isa < FixedVectorType > ( II -> getArgOperand ( 0 ) -> getType ( ) ) ) { Scatters . push_back ( II ) ; Changed |= optimiseAddress ( II -> getArgOperand ( 1 ) , II -> getParent ( ) , LI ) ; } } } for ( unsigned i = 0 ; i < Gathers . size ( ) ; i ++ ) { IntrinsicInst * I = Gathers [ i ] ; Value * L = lowerGather ( I ) ; if ( L == nullptr ) continue ; SimplifyInstructionsInBlock ( cast < Instruction > ( L ) -> getParent ( ) ) ; Changed = true ; } for ( unsigned i = 0 ; i < Scatters . size ( ) ; i ++ ) { IntrinsicInst * I = Scatters [ i ] ; Value * S = lowerScatter ( I ) ; if ( S == nullptr ) continue ; SimplifyInstructionsInBlock ( cast < Instruction > ( S ) -> getParent ( ) ) ; Changed = true ; } return Changed ; } </s>


<s> bool MVEGatherScatterLowering :: runOnFunction ( Function & F ) { if ( ! EnableMaskedGatherScatters ) return false ; auto & TPC = getAnalysis < TargetPassConfig > ( ) ; auto & TM = TPC . getTM < TargetMachine > ( ) ; auto * ST = & TM . getSubtarget < ARMSubtarget > ( F ) ; if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; SmallVector < IntrinsicInst * , 4 > Gathers ; SmallVector < IntrinsicInst * , 4 > Scatters ; for ( BasicBlock & BB : F ) { for ( Instruction & I : BB ) { IntrinsicInst * II = dyn_cast < IntrinsicInst > ( & I ) ; if ( II && II -> getIntrinsicID ( ) == Intrinsic :: masked_gather ) Gathers . push_back ( II ) ; else if ( II && II -> getIntrinsicID ( ) == Intrinsic :: masked_scatter ) Scatters . push_back ( II ) ; } } bool Changed = false ; for ( IntrinsicInst * I : Gathers ) Changed |= lowerGather ( I ) ; for ( IntrinsicInst * I : Scatters ) Changed |= lowerScatter ( I ) ; return Changed ; } </s>


<s> bool MVEGatherScatterLowering :: runOnFunction ( Function & F ) { if ( ! EnableMaskedGatherScatters ) return false ; auto & TPC = getAnalysis < TargetPassConfig > ( ) ; auto & TM = TPC . getTM < TargetMachine > ( ) ; auto * ST = & TM . getSubtarget < ARMSubtarget > ( F ) ; if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; SmallVector < IntrinsicInst * , 4 > Gathers ; SmallVector < IntrinsicInst * , 4 > Scatters ; LoopInfo & LI = getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; for ( BasicBlock & BB : F ) { for ( Instruction & I : BB ) { IntrinsicInst * II = dyn_cast < IntrinsicInst > ( & I ) ; if ( II && II -> getIntrinsicID ( ) == Intrinsic :: masked_gather ) Gathers . push_back ( II ) ; else if ( II && II -> getIntrinsicID ( ) == Intrinsic :: masked_scatter ) Scatters . push_back ( II ) ; } } bool Changed = false ; for ( unsigned i = 0 ; i < Gathers . size ( ) ; i ++ ) { IntrinsicInst * I = Gathers [ i ] ; if ( isa < GetElementPtrInst > ( I -> getArgOperand ( 0 ) ) ) optimiseOffsets ( cast < Instruction > ( I -> getArgOperand ( 0 ) ) -> getOperand ( 1 ) , I -> getParent ( ) , & LI ) ; Value * L = lowerGather ( I ) ; if ( L == nullptr ) continue ; SimplifyInstructionsInBlock ( cast < Instruction > ( L ) -> getParent ( ) ) ; Changed = true ; } for ( unsigned i = 0 ; i < Scatters . size ( ) ; i ++ ) { IntrinsicInst * I = Scatters [ i ] ; if ( isa < GetElementPtrInst > ( I -> getArgOperand ( 1 ) ) ) optimiseOffsets ( cast < Instruction > ( I -> getArgOperand ( 1 ) ) -> getOperand ( 1 ) , I -> getParent ( ) , & LI ) ; Value * S = lowerScatter ( I ) ; if ( S == nullptr ) continue ; SimplifyInstructionsInBlock ( cast < Instruction > ( S ) -> getParent ( ) ) ; Changed = true ; } return Changed ; } </s>


<s> bool MVEGatherScatterLowering :: runOnFunction ( Function & F ) { if ( ! EnableMaskedGatherScatters ) return false ; auto & TPC = getAnalysis < TargetPassConfig > ( ) ; auto & TM = TPC . getTM < TargetMachine > ( ) ; auto * ST = & TM . getSubtarget < ARMSubtarget > ( F ) ; if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; SmallVector < IntrinsicInst * , 4 > Gathers ; SmallVector < IntrinsicInst * , 4 > Scatters ; bool Changed = false ; for ( BasicBlock & BB : F ) { for ( Instruction & I : BB ) { IntrinsicInst * II = dyn_cast < IntrinsicInst > ( & I ) ; if ( II && II -> getIntrinsicID ( ) == Intrinsic :: masked_gather ) { Gathers . push_back ( II ) ; if ( isa < GetElementPtrInst > ( II -> getArgOperand ( 0 ) ) ) Changed |= optimiseOffsets ( cast < Instruction > ( II -> getArgOperand ( 0 ) ) -> getOperand ( 1 ) , II -> getParent ( ) , LI ) ; } else if ( II && II -> getIntrinsicID ( ) == Intrinsic :: masked_scatter ) { Scatters . push_back ( II ) ; if ( isa < GetElementPtrInst > ( II -> getArgOperand ( 1 ) ) ) Changed |= optimiseOffsets ( cast < Instruction > ( II -> getArgOperand ( 1 ) ) -> getOperand ( 1 ) , II -> getParent ( ) , LI ) ; } } } for ( unsigned i = 0 ; i < Gathers . size ( ) ; i ++ ) { IntrinsicInst * I = Gathers [ i ] ; Value * L = lowerGather ( I ) ; if ( L == nullptr ) continue ; SimplifyInstructionsInBlock ( cast < Instruction > ( L ) -> getParent ( ) ) ; Changed = true ; } for ( unsigned i = 0 ; i < Scatters . size ( ) ; i ++ ) { IntrinsicInst * I = Scatters [ i ] ; Value * S = lowerScatter ( I ) ; if ( S == nullptr ) continue ; SimplifyInstructionsInBlock ( cast < Instruction > ( S ) -> getParent ( ) ) ; Changed = true ; } return Changed ; } </s>


<s> bool MVEGatherScatterLowering :: runOnFunction ( Function & F ) { if ( ! EnableMaskedGatherScatters ) return false ; auto & TPC = getAnalysis < TargetPassConfig > ( ) ; auto & TM = TPC . getTM < TargetMachine > ( ) ; auto * ST = & TM . getSubtarget < ARMSubtarget > ( F ) ; if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; SmallVector < IntrinsicInst * , 4 > Gathers ; SmallVector < IntrinsicInst * , 4 > Scatters ; for ( BasicBlock & BB : F ) { for ( Instruction & I : BB ) { IntrinsicInst * II = dyn_cast < IntrinsicInst > ( & I ) ; if ( II && II -> getIntrinsicID ( ) == Intrinsic :: masked_gather ) { Gathers . push_back ( II ) ; if ( isa < GetElementPtrInst > ( II -> getArgOperand ( 0 ) ) ) optimiseOffsets ( cast < Instruction > ( II -> getArgOperand ( 0 ) ) -> getOperand ( 1 ) , II -> getParent ( ) , LI ) ; } else if ( II && II -> getIntrinsicID ( ) == Intrinsic :: masked_scatter ) { Scatters . push_back ( II ) ; if ( isa < GetElementPtrInst > ( II -> getArgOperand ( 1 ) ) ) optimiseOffsets ( cast < Instruction > ( II -> getArgOperand ( 1 ) ) -> getOperand ( 1 ) , II -> getParent ( ) , LI ) ; } } } bool Changed = false ; for ( unsigned i = 0 ; i < Gathers . size ( ) ; i ++ ) { IntrinsicInst * I = Gathers [ i ] ; Value * L = lowerGather ( I ) ; if ( L == nullptr ) continue ; SimplifyInstructionsInBlock ( cast < Instruction > ( L ) -> getParent ( ) ) ; Changed = true ; } for ( unsigned i = 0 ; i < Scatters . size ( ) ; i ++ ) { IntrinsicInst * I = Scatters [ i ] ; Value * S = lowerScatter ( I ) ; if ( S == nullptr ) continue ; SimplifyInstructionsInBlock ( cast < Instruction > ( S ) -> getParent ( ) ) ; Changed = true ; } return Changed ; } </s>


<s> bool MVEGatherScatterLowering :: runOnFunction ( Function & F ) { if ( ! EnableMaskedGatherScatters ) return false ; auto & TPC = getAnalysis < TargetPassConfig > ( ) ; auto & TM = TPC . getTM < TargetMachine > ( ) ; auto * ST = & TM . getSubtarget < ARMSubtarget > ( F ) ; if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; SmallVector < IntrinsicInst * , 4 > Gathers ; SmallVector < IntrinsicInst * , 4 > Scatters ; bool Changed = false ; for ( BasicBlock & BB : F ) { for ( Instruction & I : BB ) { IntrinsicInst * II = dyn_cast < IntrinsicInst > ( & I ) ; if ( II && II -> getIntrinsicID ( ) == Intrinsic :: masked_gather && isa < FixedVectorType > ( II -> getType ( ) ) ) { Gathers . push_back ( II ) ; Changed |= optimiseAddress ( II -> getArgOperand ( 0 ) , II -> getParent ( ) , LI ) ; } else if ( II && II -> getIntrinsicID ( ) == Intrinsic :: masked_scatter && isa < FixedVectorType > ( II -> getArgOperand ( 0 ) -> getType ( ) ) ) { Scatters . push_back ( II ) ; Changed |= optimiseAddress ( II -> getArgOperand ( 1 ) , II -> getParent ( ) , LI ) ; } } } for ( unsigned i = 0 ; i < Gathers . size ( ) ; i ++ ) { IntrinsicInst * I = Gathers [ i ] ; Value * L = lowerGather ( I ) ; if ( L == nullptr ) continue ; SimplifyInstructionsInBlock ( cast < Instruction > ( L ) -> getParent ( ) ) ; Changed = true ; } for ( unsigned i = 0 ; i < Scatters . size ( ) ; i ++ ) { IntrinsicInst * I = Scatters [ i ] ; Value * S = lowerScatter ( I ) ; if ( S == nullptr ) continue ; SimplifyInstructionsInBlock ( cast < Instruction > ( S ) -> getParent ( ) ) ; Changed = true ; } return Changed ; } </s>


<s>
StringRef
getPassName
(
)
const
override
{
return
"MVE lane interleaving"
;
}
</s>


<s> bool MVELaneInterleaving :: runOnFunction ( Function & F ) { if ( ! EnableInterleave ) return false ; auto & TPC = getAnalysis < TargetPassConfig > ( ) ; auto & TM = TPC . getTM < TargetMachine > ( ) ; auto * ST = & TM . getSubtarget < ARMSubtarget > ( F ) ; if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; bool Changed = false ; SmallPtrSet < Instruction * , 16 > Visited ; for ( Instruction & I : reverse ( instructions ( F ) ) ) { if ( I . getType ( ) -> isVectorTy ( ) && ( isa < TruncInst > ( I ) || isa < FPTruncInst > ( I ) ) && ! Visited . count ( & I ) ) Changed |= tryInterleave ( & I , Visited ) ; } return Changed ; } </s>


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < ScalarEvolutionWrapperPass > ( ) ; AU . addRequired < LoopInfoWrapperPass > ( ) ; AU . addRequired < TargetPassConfig > ( ) ; AU . addRequired < TargetTransformInfoWrapperPass > ( ) ; AU . addPreserved < LoopInfoWrapperPass > ( ) ; AU . setPreservesCFG ( ) ; } </s>


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < ScalarEvolutionWrapperPass > ( ) ; AU . addRequired < LoopInfoWrapperPass > ( ) ; AU . addRequired < TargetPassConfig > ( ) ; AU . addRequired < TargetTransformInfoWrapperPass > ( ) ; AU . addRequired < DominatorTreeWrapperPass > ( ) ; AU . addRequired < TargetLibraryInfoWrapperPass > ( ) ; AU . addPreserved < LoopInfoWrapperPass > ( ) ; AU . setPreservesCFG ( ) ; } </s>


<s>
static
Value
*
getNumElements
(
BasicBlock
*
Preheader
,
Value
*
BTC
)
{
for
(
auto
&
I
:
*
Preheader
)
{
if
(
I
.
getOpcode
(
)
!=
Instruction
::
Add
||
&
I
!=
BTC
)
continue
;
ConstantInt
*
MinusOne
=
nullptr
;
if
(
!
(
MinusOne
=
dyn_cast
<
ConstantInt
>
(
I
.
getOperand
(
1
)
)
)
)
continue
;
if
(
MinusOne
->
getSExtValue
(
)
==
1
)
{
LLVM_DEBUG
(
dbgs
(
)
<<
"ARM TP: Found num elems: "
<<
I
<<
"\n"
)
;
return
I
.
getOperand
(
0
)
;
}
}
IRBuilder
<
>
Builder
(
Preheader
->
getTerminator
(
)
)
;
Value
*
NumElements
=
Builder
.
CreateAdd
(
BTC
,
ConstantInt
::
get
(
BTC
->
getType
(
)
,
1
)
,
"num.elements"
)
;
LLVM_DEBUG
(
dbgs
(
)
<<
"ARM TP: Created num elems: "
<<
*
NumElements
<<
"\n"
)
;
return
NumElements
;
}
</s>


"********** ARM MVE VPT Optimisations **********\n"


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < MachineLoopInfo > ( ) ; AU . addPreserved < MachineLoopInfo > ( ) ; AU . addRequired < MachineDominatorTree > ( ) ; AU . addPreserved < MachineDominatorTree > ( ) ; MachineFunctionPass :: getAnalysisUsage ( AU ) ; } </s>