<s> } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << SmallDataThreshold << <STR_LIT> << GO -> getName ( ) << <STR_LIT> ) ; const GlobalVariable * GVar = dyn_cast < GlobalVariable > ( GO ) ; if ( ! GVar ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( GVar -> hasSection ( ) ) { bool IsSmall = isSmallDataSection ( GVar -> getSection ( ) ) ; LLVM_DEBUG ( dbgs ( ) << ( IsSmall ? <STR_LIT> : <STR_LIT> ) << <STR_LIT> << GVar -> getSection ( ) << '\n' ) ; return IsSmall ; } if ( GVar -> isConstant ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } bool IsLocal = GVar -> hasLocalLinkage ( ) ; if ( ! StaticsInSData && IsLocal ) {


<s> const MCSection * <ISA_LIT>TargetObjectFile :: SelectSectionForGlobal ( const GlobalValue * GV , SectionKind Kind , Mangler & Mang , const TargetMachine & TM ) const { if ( Kind . isBSS ( ) && IsGlobalInSmallSection ( GV , TM , Kind ) ) return SmallBSSSection ;

if ( Kind . isDataNoRel ( ) && IsGlobalInSmallSection ( GV , TM , Kind ) ) return SmallDataSection ;


<s> if ( Section . find ( <STR_LIT> ) != StringRef :: npos ) return getContext ( ) . getELFSection ( GV -> getSection ( ) , ELF :: SHT_PROGBITS , ELF :: SHF_ALLOC | ELF :: SHF_EXECINSTR ) ; if ( Section . find ( <STR_LIT> ) != StringRef :: npos ) return getContext ( ) . getELFSection ( GV -> getSection ( ) , ELF :: SHT_PROGBITS , ELF :: SHF_WRITE | ELF :: SHF_ALLOC ) ; }

if ( isGlobalInSmallSection ( GV , TM ) ) return selectSmallSectionForGlobal ( GV , Kind , Mang , TM ) ;


<s> StringRef Section = GV -> getSection ( ) ; if ( Section . find ( <STR_LIT> ) != StringRef :: npos ) return getContext ( ) . getELFSection ( GV -> getSection ( ) , ELF :: SHT_PROGBITS , ELF :: SHF_ALLOC | ELF :: SHF_EXECINSTR ) ; if ( Section . find ( <STR_LIT> ) != StringRef :: npos ) return getContext ( ) . getELFSection ( GV -> getSection ( ) , ELF :: SHT_PROGBITS , ELF :: SHF_WRITE | ELF :: SHF_ALLOC ) ; } if ( isGlobalInSmallSection ( GV , TM ) ) return selectSmallSectionForGlobal ( GV , Kind , Mang , TM ) ; TRACE ( <STR_LIT> ) ;

return TargetLoweringObjectFileELF :: getExplicitSectionGlobal ( GV , Kind , Mang , TM ) ;


<s> Type * GType = GVar -> getType ( ) ; if ( PointerType * PT = dyn_cast < PointerType > ( GType ) ) GType = PT -> getElementType ( ) ; if ( isa < ArrayType > ( GType ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( StructType * ST = dyn_cast < StructType > ( GType ) ) { if ( ST -> isOpaque ( ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } } unsigned Size = GVar -> getParent ( ) -> getDataLayout ( ) . getTypeAllocSize ( GType ) ; if ( Size == <NUM_LIT> ) {


<s> if ( GVar -> hasSection ( ) ) { bool IsSmall = isSmallDataSection ( GVar -> getSection ( ) ) ; DEBUG ( dbgs ( ) << ( IsSmall ? <STR_LIT> : <STR_LIT> ) << <STR_LIT> << GVar -> getSection ( ) << '\n' ) ; return IsSmall ; } if ( GVar -> isConstant ( ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } bool IsLocal = GVar -> hasLocalLinkage ( ) ; if ( ! StaticsInSData && IsLocal ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } Type * GType = GVar -> getType ( ) ; if ( PointerType * PT = dyn_cast < PointerType > ( GType ) ) GType = PT -> getElementType ( ) ; if ( isa < ArrayType > ( GType ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( StructType * ST = dyn_cast < StructType > ( GType ) ) { if ( ST -> isOpaque ( ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; }


<s> TRACE ( <STR_LIT> << GV -> getSection ( ) << <STR_LIT> ) ; TRACE ( ( GV -> hasPrivateLinkage ( ) ? <STR_LIT> : <STR_LIT> ) << ( GV -> hasLocalLinkage ( ) ? <STR_LIT> : <STR_LIT> ) << ( GV -> hasInternalLinkage ( ) ? <STR_LIT> : <STR_LIT> ) << ( GV -> hasExternalLinkage ( ) ? <STR_LIT> : <STR_LIT> ) << ( GV -> hasCommonLinkage ( ) ? <STR_LIT> : <STR_LIT> ) << ( GV -> hasCommonLinkage ( ) ? <STR_LIT> : <STR_LIT> ) << ( Kind . isCommon ( ) ? <STR_LIT> : <STR_LIT> ) << ( Kind . isBSS ( ) ? <STR_LIT> : <STR_LIT> ) << ( Kind . isBSSLocal ( ) ? <STR_LIT> : <STR_LIT> ) ) ; if ( isGlobalInSmallSection ( GV , TM ) ) return selectSmallSectionForGlobal ( GV , Kind , Mang , TM ) ; if ( Kind . isCommon ( ) ) { return BSSSection ;


<s> if ( Kind . isCommon ( ) ) { if ( NoSmallDataSorting ) return BSSSection ; Twine Name = Twine ( <STR_LIT> ) + getSectionSuffixForSize ( Size ) ; TRACE ( <STR_LIT> << Name << <STR_LIT> ) ; return getContext ( ) . getELFSection ( Name . str ( ) , ELF :: SHT_NOBITS , ELF :: SHF_WRITE | ELF :: SHF_ALLOC | ELF :: SHF_HEX_GPREL ) ; } if ( Kind . isMergeableConst ( ) ) { TRACE ( <STR_LIT> ) ; const GlobalVariable * GVar = dyn_cast < GlobalVariable > ( GV ) ; if ( GVar -> hasSection ( ) && isSmallDataSection ( GVar -> getSection ( ) ) ) Kind = SectionKind :: getData ( ) ; } if ( Kind . isData ( ) ) {


<s> if ( Kind . isBSS ( ) || Kind . isData ( ) || Kind . isCommon ( ) ) { Type * Ty = GV -> getValueType ( ) ; return IsInSmallSection ( GV -> getParent ( ) -> getDataLayout ( ) . getTypeAllocSize ( Ty ) ) ; } return false ;


<s> if ( ! GVA ) return false ; if ( Kind . isBSS ( ) || Kind . isData ( ) || Kind . isCommon ( ) ) { Type * Ty = GV -> getValueType ( ) ; return IsInSmallSection ( GV -> getParent ( ) -> getDataLayout ( ) . getTypeAllocSize ( Ty ) ) ; } return false ;


<s> void <ISA_LIT>TargetObjectFile :: Initialize ( MCContext & Ctx , const TargetMachine & TM ) { TargetLoweringObjectFileELF :: Initialize ( Ctx , TM ) ; InitializeELF ( TM . Options . UseInitArray ) ; SmallDataSection = getContext ( ) . getELFSection ( <STR_LIT> , ELF :: SHT_PROGBITS , ELF :: SHF_WRITE | ELF :: SHF_ALLOC ) ; SmallBSSSection = getContext ( ) . getELFSection ( <STR_LIT> , ELF :: SHT_NOBITS , ELF :: SHF_WRITE | ELF :: SHF_ALLOC ) ;


<s> if ( Kind . isBSS ( ) || Kind . isData ( ) || Kind . isCommon ( ) ) {

Type * Ty = GV -> getType ( ) -> getElementType ( ) ;


<s> if ( ! GVA ) return false ; if ( Kind . isBSS ( ) || Kind . isData ( ) || Kind . isCommon ( ) ) { Type * Ty = GV -> getType ( ) -> getElementType ( ) ; return IsInSmallSection ( GV -> getParent ( ) -> getDataLayout ( ) . getTypeAllocSize ( Ty ) ) ;


<s> static bool IsInSmallSection ( uint64_t Size ) { return Size > <NUM_LIT> && Size <= ( uint64_t ) SmallDataThreshold ;


<s> if ( Kind . isData ( ) && IsGlobalInSmallSection ( GV , TM , Kind ) ) return SmallDataSection ; return TargetLoweringObjectFileELF :: SelectSectionForGlobal ( GV , Kind , Mang , TM ) ;


<s> <ISA_LIT>TargetStreamer ( MCStreamer & S ) : MCTargetStreamer ( S ) {


<s> const <ISA_LIT>Subtarget * getST ( ) const { return ST ;


<s> const <ISA_LIT>TargetLowering * getTLI ( ) const { return TLI ;


<s> <ISA_LIT>TTIImpl ( <ISA_LIT>TTIImpl && Arg ) : BaseT ( std :: move ( static_cast < BaseT & > ( Arg ) ) ) , ST ( std :: move ( Arg . ST ) ) , TLI ( std :: move ( Arg . TLI ) ) {


<s> InstructionCost getCFInstrCost ( unsigned Opcode , TTI :: TargetCostKind CostKind , const Instruction * I = nullptr ) { return <NUM_LIT> ;


<s> void <ISA_LIT>TTIImpl :: getUnrollingPreferences ( Loop * L , ScalarEvolution & SE , TTI :: UnrollingPreferences & UP ) { UP . Runtime = UP . Partial = true ;


<s> int <ISA_LIT>TTIImpl :: getUserCost ( const User * U , ArrayRef < const Value * > Operands ) { auto isCastFoldedIntoLoad = [ ] ( const CastInst * CI ) -> bool { if ( ! CI -> isIntegerCast ( ) ) return false ; const LoadInst * LI = dyn_cast < const LoadInst > ( CI -> getOperand ( <NUM_LIT> ) ) ; if ( ! LI || ! LI -> hasOneUse ( ) ) return false ; unsigned SBW = CI -> getSrcTy ( ) -> getIntegerBitWidth ( ) ; unsigned DBW = CI -> getDestTy ( ) -> getIntegerBitWidth ( ) ; return DBW == <NUM_LIT> && ( SBW < DBW ) ; } ; if ( const CastInst * CI = dyn_cast < const CastInst > ( U ) ) if ( isCastFoldedIntoLoad ( CI ) ) return TargetTransformInfo :: TCC_Free ; return BaseT :: getUserCost ( U , Operands ) ;


<s> bool shouldMaximizeVectorBandwidth ( ) const { return true ;


<s> bool shouldMaximizeVectorBandwidth ( ) const {


<s> if ( Ty -> isVectorTy ( ) ) { std :: pair < int , MVT > LT = TLI . getTypeLegalizationCost ( DL , Ty ) ;

if ( LT . second . isFloatingPoint ( ) ) return LT . first + FloatFactor * getTypeNumElements ( Ty ) ;


<s> std :: pair < int , MVT > LT = TLI . getTypeLegalizationCost ( DL , Ty ) ; if ( LT . second . isFloatingPoint ( ) ) return LT . first + FloatFactor * getTypeNumElements ( Ty ) ; }

return BaseT :: getArithmeticInstrCost ( Opcode , Ty , Opd1Info , Opd2Info , Opd1PropInfo , Opd2PropInfo , Args ) ;


<s> return BaseT :: getCallInstrCost ( F , RetTy , Tys ) ;


<s> std :: pair < int , MVT > DstLT = TLI . getTypeLegalizationCost ( DL , DstTy ) ;

return std :: max ( SrcLT . first , DstLT . first ) + FloatFactor * ( SrcN + DstN ) ;


std :: pair < int , MVT > LT = TLI . getTypeLegalizationCost ( DL , ValTy ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getGatherScatterOpCost ( unsigned Opcode , Type * DataTy , Value * Ptr , bool VariableMask , unsigned Alignment ) {

return BaseT :: getGatherScatterOpCost ( Opcode , DataTy , Ptr , VariableMask , Alignment ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getInterleavedMemoryOpCost ( unsigned Opcode , Type * VecTy , unsigned Factor , ArrayRef < unsigned > Indices , unsigned Alignment , unsigned AddressSpace ) {

return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment , AddressSpace ) ;


<s> return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment , AddressSpace ) ;


<s> std :: pair < int , MVT > LT = TLI . getTypeLegalizationCost ( DL , RetTy ) ; return LT . first + <NUM_LIT> ; } return BaseT :: getIntrinsicInstrCost ( ID , RetTy , Tys , FMF , ScalarizationCostPassed ) ;


<s> unsigned AlignWidth = <NUM_LIT> * std :: max ( <NUM_LIT> , Alignment ) ; unsigned NumLoads = alignTo ( VecWidth , AlignWidth ) / AlignWidth ; return <NUM_LIT> * NumLoads ; } unsigned Cost = VecTy -> getElementType ( ) -> isFloatingPointTy ( ) ? FloatFactor : <NUM_LIT> ; Alignment = std :: min ( Alignment , <NUM_LIT> ) ; unsigned AlignWidth = <NUM_LIT> * std :: max ( <NUM_LIT> , Alignment ) ; unsigned NumLoads = alignTo ( VecWidth , AlignWidth ) / AlignWidth ; if ( Alignment == <NUM_LIT> || Alignment == <NUM_LIT> ) return Cost * NumLoads ;


<s> Alignment = std :: min ( Alignment , RegWidth / <NUM_LIT> ) ; if ( VecWidth % RegWidth == <NUM_LIT> ) return VecWidth / RegWidth ; unsigned AlignWidth = <NUM_LIT> * std :: max ( <NUM_LIT> , Alignment ) ; unsigned NumLoads = alignTo ( VecWidth , AlignWidth ) / AlignWidth ; return <NUM_LIT> * NumLoads ; } unsigned Cost = VecTy -> getElementType ( ) -> isFloatingPointTy ( ) ? FloatFactor : <NUM_LIT> ; Alignment = std :: min ( Alignment , <NUM_LIT> ) ; unsigned AlignWidth = <NUM_LIT> * std :: max ( <NUM_LIT> , Alignment ) ; unsigned NumLoads = alignTo ( VecWidth , AlignWidth ) / AlignWidth ;

if ( Alignment == <NUM_LIT> || Alignment == <NUM_LIT> ) return Cost * NumLoads ;


<s> return BaseT :: getScalarizationOverhead ( Ty , Insert , Extract ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getScalarizationOverhead ( Type * Ty , bool Insert , bool Extract ) { return BaseT :: getScalarizationOverhead ( Ty , Insert , Extract ) ;


<s> auto isCastFoldedIntoLoad = [ this ] ( const CastInst * CI ) -> bool { if ( ! CI -> isIntegerCast ( ) ) return false ; const DataLayout & DL = getDataLayout ( ) ; unsigned SBW = DL . getTypeSizeInBits ( CI -> getSrcTy ( ) ) ; unsigned DBW = DL . getTypeSizeInBits ( CI -> getDestTy ( ) ) ; if ( DBW != <NUM_LIT> || SBW >= DBW ) return false ; const LoadInst * LI = dyn_cast < const LoadInst > ( CI -> getOperand ( <NUM_LIT> ) ) ; return LI && LI -> hasOneUse ( ) ; } ; if ( const CastInst * CI = dyn_cast < const CastInst > ( U ) ) if ( isCastFoldedIntoLoad ( CI ) ) return TargetTransformInfo :: TCC_Free ;

return BaseT :: getUserCost ( U , Operands ) ;


<s> if ( ! CI -> isIntegerCast ( ) ) return false ; const DataLayout & DL = getDataLayout ( ) ; unsigned SBW = DL . getTypeSizeInBits ( CI -> getSrcTy ( ) ) ; unsigned DBW = DL . getTypeSizeInBits ( CI -> getDestTy ( ) ) ; if ( DBW != <NUM_LIT> || SBW >= DBW ) return false ; const LoadInst * LI = dyn_cast < const LoadInst > ( CI -> getOperand ( <NUM_LIT> ) ) ;


<s> if ( ST . isHVXVectorType ( VecVT . getSimpleVT ( ) ) ) return true ; auto Action = TLI . getPreferredVectorAction ( VecVT ) ; return Action == TargetLoweringBase :: TypeWidenVector ;


<s> EVT VecVT = EVT :: getEVT ( VecTy ) ; if ( ! VecVT . isSimple ( ) || VecVT . getSizeInBits ( ) <= <NUM_LIT> ) return false ; if ( ST . isHVXVectorType ( VecVT . getSimpleVT ( ) ) ) return true ;

auto Action = TLI . getPreferredVectorAction ( VecVT ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getArithmeticInstrCost ( unsigned Opcode , Type * Ty , TTI :: OperandValueKind Opd1Info , TTI :: OperandValueKind Opd2Info , TTI :: OperandValueProperties Opd1PropInfo , TTI :: OperandValueProperties Opd2PropInfo , ArrayRef < const Value * > Args ) {


<s> unsigned <ISA_LIT>TTIImpl :: getCastInstrCost ( unsigned Opcode , Type * Dst , Type * Src , const Instruction * I ) { return <NUM_LIT> ;


<s> unsigned <ISA_LIT>TTIImpl :: getCmpSelInstrCost ( unsigned Opcode , Type * ValTy , Type * CondTy , const Instruction * I ) { if ( ValTy -> isVectorTy ( ) ) { auto * VecTy = dyn_cast < VectorType > ( ValTy ) ; std :: pair < int , MVT > LT = TLI . getTypeLegalizationCost ( DL , ValTy ) ; if ( Opcode == Instruction :: FCmp ) return LT . first + <NUM_LIT> * VecTy -> getNumElements ( ) ; } return BaseT :: getCmpSelInstrCost ( Opcode , ValTy , CondTy , I ) ;


<s> assert ( Opcode == Instruction :: Load || Opcode == Instruction :: Store ) ; if ( Opcode == Instruction :: Store ) return BaseT :: getMemoryOpCost ( Opcode , Src , Alignment , AddressSpace , I ) ; if ( Src -> isVectorTy ( ) ) { VectorType * VecTy = cast < VectorType > ( Src ) ; unsigned VecWidth = VecTy -> getBitWidth ( ) ; if ( useHVX ( ) && isTypeForHVX ( VecTy ) ) { unsigned RegWidth = getRegisterBitWidth ( true ) ;

Alignment = std :: min ( Alignment , RegWidth / <NUM_LIT> ) ;


<s> bool shouldMaximizeVectorBandwidth ( TargetTransformInfo :: RegisterKind K ) const {


<s> unsigned getCFInstrCost ( unsigned Opcode , TTI :: TargetCostKind CostKind ) {


<s> unsigned <ISA_LIT>TTIImpl :: getInterleavedMemoryOpCost ( unsigned Opcode , Type * VecTy , unsigned Factor , ArrayRef < unsigned > Indices , unsigned Alignment , unsigned AddressSpace , bool UseMaskForCond , bool UseMaskForGaps ) { if ( Indices . size ( ) != Factor || UseMaskForCond || UseMaskForGaps ) return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment , AddressSpace , UseMaskForCond , UseMaskForGaps ) ;

return getMemoryOpCost ( Opcode , VecTy , Alignment , AddressSpace , nullptr ) ;


<s> if ( ST . isHVXVectorType ( VecVT . getSimpleVT ( ) ) ) return true ; auto Action = TLI . getPreferredVectorAction ( VecVT . getSimpleVT ( ) ) ;

return Action == TargetLoweringBase :: TypeWidenVector ;


<s> EVT VecVT = EVT :: getEVT ( VecTy ) ; if ( ! VecVT . isSimple ( ) || VecVT . getSizeInBits ( ) <= <NUM_LIT> ) return false ; if ( ST . isHVXVectorType ( VecVT . getSimpleVT ( ) ) ) return true ; auto Action = TLI . getPreferredVectorAction ( VecVT . getSimpleVT ( ) ) ;


<s> if ( Ty -> isVectorTy ( ) ) { std :: pair < int , MVT > LT = TLI . getTypeLegalizationCost ( DL , Ty ) ; if ( LT . second . isFloatingPoint ( ) ) return LT . first + FloatFactor * getTypeNumElements ( Ty ) ; } return BaseT :: getArithmeticInstrCost ( Opcode , Ty , CostKind , Opd1Info , Opd2Info , Opd1PropInfo , Opd2PropInfo , Args , CxtI ) ;


<s> std :: pair < int , MVT > LT = TLI . getTypeLegalizationCost ( DL , ValTy ) ; if ( Opcode == Instruction :: FCmp ) return LT . first + FloatFactor * getTypeNumElements ( ValTy ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getCmpSelInstrCost ( unsigned Opcode , Type * ValTy , Type * CondTy , TTI :: TargetCostKind CostKind , const Instruction * I ) { if ( ValTy -> isVectorTy ( ) ) {


<s> unsigned VecWidth = VecTy -> getPrimitiveSizeInBits ( ) . getFixedSize ( ) ; if ( useHVX ( ) && isTypeForHVX ( VecTy ) ) { unsigned RegWidth = getRegisterBitWidth ( true ) ; assert ( RegWidth && <STR_LIT> ) ; if ( VecWidth % RegWidth == <NUM_LIT> ) return VecWidth / RegWidth ; const Align RegAlign ( RegWidth / <NUM_LIT> ) ; if ( ! Alignment || * Alignment > RegAlign ) Alignment = RegAlign ; assert ( Alignment ) ; unsigned AlignWidth = <NUM_LIT> * Alignment -> value ( ) ; unsigned NumLoads = alignTo ( VecWidth , AlignWidth ) / AlignWidth ; return <NUM_LIT> * NumLoads ; } unsigned Cost = VecTy -> getElementType ( ) -> isFloatingPointTy ( ) ? FloatFactor : <NUM_LIT> ; const Align BoundAlignment = std :: min ( Alignment . valueOrOne ( ) , Align ( <NUM_LIT> ) ) ; unsigned AlignWidth = <NUM_LIT> * BoundAlignment . value ( ) ;


<s> unsigned AlignWidth = <NUM_LIT> * Alignment -> value ( ) ; unsigned NumLoads = alignTo ( VecWidth , AlignWidth ) / AlignWidth ; return <NUM_LIT> * NumLoads ; } unsigned Cost = VecTy -> getElementType ( ) -> isFloatingPointTy ( ) ? FloatFactor : <NUM_LIT> ; const Align BoundAlignment = std :: min ( Alignment . valueOrOne ( ) , Align ( <NUM_LIT> ) ) ; unsigned AlignWidth = <NUM_LIT> * BoundAlignment . value ( ) ; unsigned NumLoads = alignTo ( VecWidth , AlignWidth ) / AlignWidth ; if ( Alignment == Align ( <NUM_LIT> ) || Alignment == Align ( <NUM_LIT> ) ) return Cost * NumLoads ; assert ( BoundAlignment <= Align ( <NUM_LIT> ) ) ; unsigned LogA = Log2 ( BoundAlignment ) ; return ( <NUM_LIT> - LogA ) * Cost * NumLoads ;


<s> return <NUM_LIT> * NumLoads ; } unsigned Cost = VecTy -> getElementType ( ) -> isFloatingPointTy ( ) ? FloatFactor : <NUM_LIT> ; Alignment = std :: min ( Alignment , <NUM_LIT> ) ; unsigned AlignWidth = <NUM_LIT> * std :: max ( <NUM_LIT> , Alignment ) ; unsigned NumLoads = alignTo ( VecWidth , AlignWidth ) / AlignWidth ; if ( Alignment == <NUM_LIT> || Alignment == <NUM_LIT> ) return Cost * NumLoads ; unsigned LogA = Log2_32 ( Alignment ) ; return ( <NUM_LIT> - LogA ) * Cost * NumLoads ; } return BaseT :: getMemoryOpCost ( Opcode , Src , Alignment , AddressSpace , I ) ;


<s> unsigned NumLoads = alignTo ( VecWidth , AlignWidth ) / AlignWidth ; return <NUM_LIT> * NumLoads ; } unsigned Cost = VecTy -> getElementType ( ) -> isFloatingPointTy ( ) ? FloatFactor : <NUM_LIT> ; Alignment = std :: min ( Alignment , <NUM_LIT> ) ; unsigned AlignWidth = <NUM_LIT> * std :: max ( <NUM_LIT> , Alignment ) ; unsigned NumLoads = alignTo ( VecWidth , AlignWidth ) / AlignWidth ;


<s> if ( cast < VectorType > ( VecTy ) -> isScalable ( ) ) return false ; if ( ! cast < VectorType > ( VecTy ) -> getElementType ( ) -> isIntegerTy ( ) ) return false ; EVT VecVT = EVT :: getEVT ( VecTy ) ; if ( ! VecVT . isSimple ( ) || VecVT . getSizeInBits ( ) <= <NUM_LIT> ) return false ; if ( ST . isHVXVectorType ( VecVT . getSimpleVT ( ) ) ) return true ; auto Action = TLI . getPreferredVectorAction ( VecVT . getSimpleVT ( ) ) ;


<s> if ( Ty -> isVectorTy ( ) ) { std :: pair < InstructionCost , MVT > LT = TLI . getTypeLegalizationCost ( DL , Ty ) ; if ( LT . second . isFloatingPoint ( ) ) return LT . first + FloatFactor * getTypeNumElements ( Ty ) ; } return BaseT :: getArithmeticInstrCost ( Opcode , Ty , CostKind , Opd1Info , Opd2Info , Opd1PropInfo , Opd2PropInfo , Args , CxtI ) ;


<s> return BaseT :: getCallInstrCost ( F , RetTy , Tys , CostKind ) ;


<s> if ( SrcTy -> isFPOrFPVectorTy ( ) || DstTy -> isFPOrFPVectorTy ( ) ) { unsigned SrcN = SrcTy -> isFPOrFPVectorTy ( ) ? getTypeNumElements ( SrcTy ) : <NUM_LIT> ; unsigned DstN = DstTy -> isFPOrFPVectorTy ( ) ? getTypeNumElements ( DstTy ) : <NUM_LIT> ; std :: pair < InstructionCost , MVT > SrcLT = TLI . getTypeLegalizationCost ( DL , SrcTy ) ;

std :: pair < InstructionCost , MVT > DstLT = TLI . getTypeLegalizationCost ( DL , DstTy ) ;


<s> unsigned SrcN = SrcTy -> isFPOrFPVectorTy ( ) ? getTypeNumElements ( SrcTy ) : <NUM_LIT> ; unsigned DstN = DstTy -> isFPOrFPVectorTy ( ) ? getTypeNumElements ( DstTy ) : <NUM_LIT> ; std :: pair < InstructionCost , MVT > SrcLT = TLI . getTypeLegalizationCost ( DL , SrcTy ) ; std :: pair < InstructionCost , MVT > DstLT = TLI . getTypeLegalizationCost ( DL , DstTy ) ;

InstructionCost Cost = std :: max ( SrcLT . first , DstLT . first ) + FloatFactor * ( SrcN + DstN ) ;


<s> std :: pair < InstructionCost , MVT > LT = TLI . getTypeLegalizationCost ( DL , ValTy ) ;

if ( Opcode == Instruction :: FCmp ) return LT . first + FloatFactor * getTypeNumElements ( ValTy ) ;


<s> InstructionCost <ISA_LIT>TTIImpl :: getIntrinsicInstrCost ( const IntrinsicCostAttributes & ICA , TTI :: TargetCostKind CostKind ) { if ( ICA . getID ( ) == <STR_LIT> ) { std :: pair < InstructionCost , MVT > LT = TLI . getTypeLegalizationCost ( DL , ICA . getReturnType ( ) ) ; return LT . first + <NUM_LIT> ; }

return BaseT :: getIntrinsicInstrCost ( ICA , CostKind ) ;


<s> InstructionCost <ISA_LIT>TTIImpl :: getIntrinsicInstrCost ( const IntrinsicCostAttributes & ICA , TTI :: TargetCostKind CostKind ) { if ( ICA . getID ( ) == <STR_LIT> ) {

std :: pair < InstructionCost , MVT > LT = TLI . getTypeLegalizationCost ( DL , ICA . getReturnType ( ) ) ;


<s> return BaseT :: getMaskedMemoryOpCost ( Opcode , Src , Alignment , AddressSpace , CostKind ) ;


<s> if ( VecWidth % RegWidth == <NUM_LIT> ) return VecWidth / RegWidth ; const Align RegAlign ( RegWidth / <NUM_LIT> ) ; if ( ! Alignment || * Alignment > RegAlign ) Alignment = RegAlign ; assert ( Alignment ) ; unsigned AlignWidth = <NUM_LIT> * Alignment -> value ( ) ; unsigned NumLoads = alignTo ( VecWidth , AlignWidth ) / AlignWidth ; return <NUM_LIT> * NumLoads ; } unsigned Cost = VecTy -> getElementType ( ) -> isFloatingPointTy ( ) ? FloatFactor : <NUM_LIT> ; const Align BoundAlignment = std :: min ( Alignment . valueOrOne ( ) , Align ( <NUM_LIT> ) ) ; unsigned AlignWidth = <NUM_LIT> * BoundAlignment . value ( ) ;


<s> assert ( ! IsScalable && <STR_LIT> ) ;

return ElementCount :: getFixed ( ( <NUM_LIT> * ST . getVectorLength ( ) ) / ElemWidth ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getMinVectorRegisterBitWidth ( ) const { return useHVX ( ) ? ST . getVectorLength ( ) * <NUM_LIT> : <NUM_LIT> ;


<s> return useHVX ( ) ? ST . getVectorLength ( ) * <NUM_LIT> : <NUM_LIT> ;


<s> InstructionCost <ISA_LIT>TTIImpl :: getOperandsScalarizationOverhead ( ArrayRef < const Value * > Args , ArrayRef < Type * > Tys ) { return BaseT :: getOperandsScalarizationOverhead ( Args , Tys ) ;


<s> return BaseT :: getOperandsScalarizationOverhead ( Args , Tys ) ;


<s> BaseT :: getPeelingPreferences ( L , SE , PP ) ;

if ( L && L -> isInnermost ( ) && canPeel ( L ) && SE . getSmallConstantTripCount ( L ) == <NUM_LIT> && SE . getSmallConstantMaxTripCount ( L ) > <NUM_LIT> && SE . getSmallConstantMaxTripCount ( L ) <= <NUM_LIT> ) {


<s> TTI :: AddressingModeKind <ISA_LIT>TTIImpl :: getPreferredAddressingMode ( const Loop * L , ScalarEvolution * SE ) const { return TTI :: AMK_PostIndexed ;


<s> return TypeSize :: getFixed ( <NUM_LIT> ) ; case TargetTransformInfo :: RGK_FixedWidthVector : return TypeSize :: getFixed ( getMinVectorRegisterBitWidth ( ) ) ; case TargetTransformInfo :: RGK_ScalableVector :

return TypeSize :: getScalable ( <NUM_LIT> ) ;


<s> InstructionCost <ISA_LIT>TTIImpl :: getShuffleCost ( TTI :: ShuffleKind Kind , Type * Tp , ArrayRef < int > Mask , int Index , Type * SubTp , ArrayRef < Value * > Args ) { return <NUM_LIT> ;


<s> void <ISA_LIT>TTIImpl :: getUnrollingPreferences ( Loop * L , ScalarEvolution & SE , TTI :: UnrollingPreferences & UP , OptimizationRemarkEmitter * ORE ) {


<s> unsigned SBW = DL . getTypeSizeInBits ( CI -> getSrcTy ( ) ) ; unsigned DBW = DL . getTypeSizeInBits ( CI -> getDestTy ( ) ) ; if ( DBW != <NUM_LIT> || SBW >= DBW ) return false ;

const LoadInst * LI = dyn_cast < const LoadInst > ( CI -> getOperand ( <NUM_LIT> ) ) ;


<s> return LI && LI -> hasOneUse ( ) ; } ; if ( const CastInst * CI = dyn_cast < const CastInst > ( U ) ) if ( isCastFoldedIntoLoad ( CI ) ) return TargetTransformInfo :: TCC_Free ; return BaseT :: getUserCost ( U , Operands , CostKind ) ;


<s> Type * ElemTy = Val -> isVectorTy ( ) ? cast < VectorType > ( Val ) -> getElementType ( ) : Val ; if ( Opcode == Instruction :: InsertElement ) { unsigned Cost = ( Index != <NUM_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; if ( ElemTy -> isIntegerTy ( <NUM_LIT> ) ) return Cost ; return Cost + getVectorInstrCost ( Instruction :: ExtractElement , Val , Index ) ; }

if ( Opcode == Instruction :: ExtractElement ) return <NUM_LIT> ;


<s> bool <ISA_LIT>TTIImpl :: isLegalMaskedLoad ( Type * DataType , Align ) { return <ISA_LIT>MaskedVMem && ST . isTypeForHVX ( DataType ) ;


<s> return <ISA_LIT>MaskedVMem && ST . isTypeForHVX ( DataType ) ;


<s> bool <ISA_LIT>TTIImpl :: isLegalMaskedStore ( Type * DataType , Align ) { return <ISA_LIT>MaskedVMem && ST . isTypeForHVX ( DataType ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getCastInstrCost ( unsigned Opcode , Type * DstTy , Type * SrcTy , TTI :: CastContextHint CCH , TTI :: TargetCostKind CostKind , const Instruction * I ) { if ( SrcTy -> isFPOrFPVectorTy ( ) || DstTy -> isFPOrFPVectorTy ( ) ) { unsigned SrcN = SrcTy -> isFPOrFPVectorTy ( ) ? getTypeNumElements ( SrcTy ) : <NUM_LIT> ; unsigned DstN = DstTy -> isFPOrFPVectorTy ( ) ? getTypeNumElements ( DstTy ) : <NUM_LIT> ; std :: pair < int , MVT > SrcLT = TLI . getTypeLegalizationCost ( DL , SrcTy ) ; std :: pair < int , MVT > DstLT = TLI . getTypeLegalizationCost ( DL , DstTy ) ; unsigned Cost = std :: max ( SrcLT . first , DstLT . first ) + FloatFactor * ( SrcN + DstN ) ; if ( CostKind != TTI :: TCK_RecipThroughput ) return Cost == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ; return Cost ;


<s> std :: pair < int , MVT > LT = TLI . getTypeLegalizationCost ( DL , ValTy ) ; if ( Opcode == Instruction :: FCmp ) return LT . first + FloatFactor * getTypeNumElements ( ValTy ) ; }

return BaseT :: getCmpSelInstrCost ( Opcode , ValTy , CondTy , VecPred , CostKind , I ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getCmpSelInstrCost ( unsigned Opcode , Type * ValTy , Type * CondTy , CmpInst :: Predicate VecPred , TTI :: TargetCostKind CostKind , const Instruction * I ) { if ( ValTy -> isVectorTy ( ) && CostKind == TTI :: TCK_RecipThroughput ) {

std :: pair < int , MVT > LT = TLI . getTypeLegalizationCost ( DL , ValTy ) ;


<s> return BaseT :: getGatherScatterOpCost ( Opcode , DataTy , Ptr , VariableMask , Alignment , CostKind , I ) ;


<s> if ( Indices . size ( ) != Factor || UseMaskForCond || UseMaskForGaps ) return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment , AddressSpace , CostKind , UseMaskForCond , UseMaskForGaps ) ;

return getMemoryOpCost ( Opcode , VecTy , MaybeAlign ( Alignment ) , AddressSpace , CostKind ) ;


<s> return BaseT :: getMaskedMemoryOpCost ( Opcode , Src , Alignment , AddressSpace , CostKind ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getMaskedMemoryOpCost ( unsigned Opcode , Type * Src , Align Alignment , unsigned AddressSpace , TTI :: TargetCostKind CostKind ) { return BaseT :: getMaskedMemoryOpCost ( Opcode , Src , Alignment , AddressSpace , CostKind ) ;


<s> VectorType * VecTy = cast < VectorType > ( Src ) ; unsigned VecWidth = VecTy -> getPrimitiveSizeInBits ( ) . getFixedSize ( ) ; if ( useHVX ( ) && ST . isTypeForHVX ( VecTy ) ) { unsigned RegWidth = getRegisterBitWidth ( true ) ; assert ( RegWidth && <STR_LIT> ) ; if ( VecWidth % RegWidth == <NUM_LIT> ) return VecWidth / RegWidth ; const Align RegAlign ( RegWidth / <NUM_LIT> ) ; if ( ! Alignment || * Alignment > RegAlign ) Alignment = RegAlign ; assert ( Alignment ) ; unsigned AlignWidth = <NUM_LIT> * Alignment -> value ( ) ;


<s> bool enableAggressiveInterleaving ( bool LoopHasReductions ) {


<s> unsigned <ISA_LIT>TTIImpl :: getCacheLineSize ( ) const {

return getST ( ) -> getL1CacheLineSize ( ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getNumberOfRegisters ( bool vector ) const {


<s> return vector ? <NUM_LIT> : <NUM_LIT> ;


<s> const <ISA_LIT>TargetLowering * getTLI ( ) const {


<s> explicit <ISA_LIT>TTIImpl ( const <ISA_LIT>TargetMachine * TM , const Function & F ) : BaseT ( TM , F . getParent ( ) -> getDataLayout ( ) ) , ST ( * TM -> getSubtargetImpl ( F ) ) , TLI ( * ST . getTargetLowering ( ) ) {


<s> bool prefersVectorizedAddressing ( ) { return false ;


<s> unsigned getAddressComputationCost ( Type * Tp , ScalarEvolution * , const SCEV * ) {


<s> unsigned getAddressComputationCost ( Type * Tp , ScalarEvolution * , const SCEV * ) { return <NUM_LIT> ;


<s> unsigned getArithmeticInstrCost ( unsigned Opcode , Type * Ty , TTI :: OperandValueKind Opd1Info = TTI :: OK_AnyValue , TTI :: OperandValueKind Opd2Info = TTI :: OK_AnyValue , TTI :: OperandValueProperties Opd1PropInfo = TTI :: OP_None , TTI :: OperandValueProperties Opd2PropInfo = TTI :: OP_None , ArrayRef < const Value * > Args = ArrayRef < const Value * > ( ) ) { return BaseT :: getArithmeticInstrCost ( Opcode , Ty , Opd1Info , Opd2Info , Opd1PropInfo , Opd2PropInfo , Args ) ;


<s> unsigned getCallInstrCost ( Function * F , Type * RetTy , ArrayRef < Type * > Tys ) {


<s> unsigned getCastInstrCost ( unsigned Opcode , Type * Dst , Type * Src , const Instruction * I = nullptr ) { return <NUM_LIT> ;


<s> unsigned getCastInstrCost ( unsigned Opcode , Type * Dst , Type * Src , const Instruction * I = nullptr ) {


<s> unsigned getGatherScatterOpCost ( unsigned Opcode , Type * DataTy , Value * Ptr , bool VariableMask , unsigned Alignment ) { return <NUM_LIT> ;


<s> unsigned getInterleavedMemoryOpCost ( unsigned Opcode , Type * VecTy , unsigned Factor , ArrayRef < unsigned > Indices , unsigned Alignment , unsigned AddressSpace ) { return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment , AddressSpace ) ;


<s> return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment , AddressSpace ) ;


<s> unsigned getIntrinsicInstrCost ( <STR_LIT> ID , Type * RetTy , ArrayRef < Type * > Tys , FastMathFlags FMF , unsigned ScalarizationCostPassed = UINT_MAX ) { return <NUM_LIT> ;


<s> unsigned getMaskedMemoryOpCost ( unsigned Opcode , Type * Src , unsigned Alignment , unsigned AddressSpace ) {


<s> unsigned getNumberOfParts ( Type * Tp ) { return BaseT :: getNumberOfParts ( Tp ) ;


<s> unsigned getOperandsScalarizationOverhead ( ArrayRef < const Value * > Args , unsigned VF ) { return <NUM_LIT> ;


<s> unsigned getScalarizationOverhead ( Type * Ty , bool Insert , bool Extract ) {


<s> unsigned getShuffleCost ( TTI :: ShuffleKind Kind , Type * Tp , int Index , Type * SubTp ) {


<s> unsigned getVectorInstrCost ( unsigned Opcode , Type * Val , unsigned Index ) {


<s> if ( L && L -> empty ( ) && canPeel ( L ) && SE . getSmallConstantTripCount ( L ) == <NUM_LIT> && SE . getSmallConstantMaxTripCount ( L ) > <NUM_LIT> && SE . getSmallConstantMaxTripCount ( L ) <= <NUM_LIT> ) {


<s> if ( LT . second . isFloatingPoint ( ) ) return LT . first + FloatFactor * getTypeNumElements ( Ty ) ; } return BaseT :: getArithmeticInstrCost ( Opcode , Ty , Opd1Info , Opd2Info , Opd1PropInfo , Opd2PropInfo , Args , CxtI ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getGatherScatterOpCost ( unsigned Opcode , Type * DataTy , Value * Ptr , bool VariableMask , unsigned Alignment , const Instruction * I ) { return BaseT :: getGatherScatterOpCost ( Opcode , DataTy , Ptr , VariableMask , Alignment , I ) ;


<s> return BaseT :: getGatherScatterOpCost ( Opcode , DataTy , Ptr , VariableMask , Alignment , I ) ;


<s> if ( Indices . size ( ) != Factor || UseMaskForCond || UseMaskForGaps ) return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment , AddressSpace , UseMaskForCond , UseMaskForGaps ) ;

return getMemoryOpCost ( Opcode , VecTy , MaybeAlign ( Alignment ) , AddressSpace , nullptr ) ;


<s> if ( ID == <STR_LIT> ) { std :: pair < int , MVT > LT = TLI . getTypeLegalizationCost ( DL , RetTy ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getMemoryOpCost ( unsigned Opcode , Type * Src , MaybeAlign Alignment , unsigned AddressSpace , const Instruction * I ) { assert ( Opcode == Instruction :: Load || Opcode == Instruction :: Store ) ; if ( Opcode == Instruction :: Store ) return BaseT :: getMemoryOpCost ( Opcode , Src , Alignment , AddressSpace , I ) ; if ( Src -> isVectorTy ( ) ) { VectorType * VecTy = cast < VectorType > ( Src ) ; unsigned VecWidth = VecTy -> getBitWidth ( ) ; if ( useHVX ( ) && isTypeForHVX ( VecTy ) ) { unsigned RegWidth = getRegisterBitWidth ( true ) ; assert ( RegWidth && <STR_LIT> ) ; if ( VecWidth % RegWidth == <NUM_LIT> ) return VecWidth / RegWidth ; const Align RegAlign ( RegWidth / <NUM_LIT> ) ; if ( ! Alignment || * Alignment > RegAlign ) Alignment = RegAlign ; assert ( Alignment ) ; unsigned AlignWidth = <NUM_LIT> * Alignment -> value ( ) ;


<s> explicit <ISA_LIT>TTIImpl ( const <ISA_LIT>TargetMachine * TM , Function & F ) : BaseT ( TM , F . getParent ( ) -> getDataLayout ( ) ) , ST ( TM -> getSubtargetImpl ( F ) ) , TLI ( ST -> getTargetLowering ( ) ) {


<s> bool <ISA_LIT>TTIImpl :: isLegalMaskedLoad ( Type * DataType , Align ) {

return <ISA_LIT>MaskedVMem && isTypeForHVX ( DataType ) ;


<s> EVT VecVT = EVT :: getEVT ( VecTy ) ; if ( ! VecVT . isSimple ( ) || VecVT . getSizeInBits ( ) <= <NUM_LIT> ) return false ; if ( ST . isHVXVectorType ( VecVT . getSimpleVT ( ) ) ) return true ;

auto Action = TLI . getPreferredVectorAction ( VecVT . getSimpleVT ( ) ) ;


<s> return BaseT :: getScalarizationOverhead ( Ty , DemandedElts , Insert , Extract ) ;


<s> if ( auto * VTy = dyn_cast < VectorType > ( Ty ) ) return VTy -> getNumElements ( ) ; assert ( ( Ty -> isIntegerTy ( ) || Ty -> isFloatingPointTy ( ) ) && <STR_LIT> ) ; return <NUM_LIT> ;


<s> assert ( ( Ty -> isIntegerTy ( ) || Ty -> isFloatingPointTy ( ) ) && <STR_LIT> ) ; return <NUM_LIT> ;


<s> InstructionCost <ISA_LIT>TTIImpl :: getShuffleCost ( TTI :: ShuffleKind Kind , Type * Tp , ArrayRef < int > Mask , int Index , Type * SubTp ) {


<s> unsigned <ISA_LIT>TTIImpl :: getMemoryOpCost ( unsigned Opcode , Type * Src , unsigned Alignment , unsigned AddressSpace , const Instruction * I ) { if ( Opcode == Instruction :: Load && Src -> isVectorTy ( ) ) { VectorType * VecTy = cast < VectorType > ( Src ) ; unsigned VecWidth = VecTy -> getBitWidth ( ) ; if ( VecWidth > <NUM_LIT> ) { if ( getNumberOfRegisters ( true ) > <NUM_LIT> ) { if ( VecWidth % getRegisterBitWidth ( true ) == <NUM_LIT> ) return <NUM_LIT> ;


<s> return getST ( ) -> useHVXOps ( ) ? getST ( ) -> getVectorLength ( ) * <NUM_LIT> : <NUM_LIT> ;


<s> unsigned <ISA_LIT>TTIImpl :: getNumberOfRegisters ( bool Vector ) const {

if ( Vector ) return <ISA_LIT>AutoHVX && getST ( ) -> useHVXOps ( ) ? <NUM_LIT> : <NUM_LIT> ;


<s> unsigned <ISA_LIT>TTIImpl :: getNumberOfRegisters ( bool Vector ) const { if ( Vector ) return <ISA_LIT>AutoHVX && getST ( ) -> useHVXOps ( ) ? <NUM_LIT> : <NUM_LIT> ;


<s> int <ISA_LIT>TTIImpl :: getUserCost ( const User * U ) { auto isCastFoldedIntoLoad = [ ] ( const CastInst * CI ) -> bool { if ( ! CI -> isIntegerCast ( ) ) return false ; const LoadInst * LI = dyn_cast < const LoadInst > ( CI -> getOperand ( <NUM_LIT> ) ) ; if ( ! LI || ! LI -> hasOneUse ( ) ) return false ; unsigned SBW = CI -> getSrcTy ( ) -> getIntegerBitWidth ( ) ;

unsigned DBW = CI -> getDestTy ( ) -> getIntegerBitWidth ( ) ;


<s> if ( ! LI || ! LI -> hasOneUse ( ) ) return false ; unsigned SBW = CI -> getSrcTy ( ) -> getIntegerBitWidth ( ) ; unsigned DBW = CI -> getDestTy ( ) -> getIntegerBitWidth ( ) ; return DBW == <NUM_LIT> && ( SBW < DBW ) ;


<s> std :: pair < int , MVT > SrcLT = TLI . getTypeLegalizationCost ( DL , SrcTy ) ; std :: pair < int , MVT > DstLT = TLI . getTypeLegalizationCost ( DL , DstTy ) ;

return std :: max ( SrcLT . first , DstLT . first ) + FloatFactor * ( SrcN + DstN ) ;


<s> if ( Indices . size ( ) != Factor ) return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment , AddressSpace ) ; return getMemoryOpCost ( Opcode , VecTy , Alignment , AddressSpace , nullptr ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getInterleavedMemoryOpCost ( unsigned Opcode , Type * VecTy , unsigned Factor , ArrayRef < unsigned > Indices , unsigned Alignment , unsigned AddressSpace ) { if ( Indices . size ( ) != Factor ) return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment , AddressSpace ) ;

return getMemoryOpCost ( Opcode , VecTy , Alignment , AddressSpace , nullptr ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getArithmeticInstrCost ( unsigned Opcode , Type * Ty , TTI :: TargetCostKind CostKind , TTI :: OperandValueKind Opd1Info , TTI :: OperandValueKind Opd2Info , TTI :: OperandValueProperties Opd1PropInfo , TTI :: OperandValueProperties Opd2PropInfo , ArrayRef < const Value * > Args , const Instruction * CxtI ) {

if ( CostKind != TTI :: TCK_RecipThroughput ) return BaseT :: getArithmeticInstrCost ( Opcode , Ty , CostKind , Opd1Info , Opd2Info , Opd1PropInfo , Opd2PropInfo , Args , CxtI ) ;


<s> if ( Ty -> isVectorTy ( ) ) { std :: pair < int , MVT > LT = TLI . getTypeLegalizationCost ( DL , Ty ) ; if ( LT . second . isFloatingPoint ( ) ) return LT . first + FloatFactor * getTypeNumElements ( Ty ) ; } return BaseT :: getArithmeticInstrCost ( Opcode , Ty , CostKind , Opd1Info , Opd2Info , Opd1PropInfo , Opd2PropInfo , Args , CxtI ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getCacheLineSize ( ) const {


<s> return ST . getL1CacheLineSize ( ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getCallInstrCost ( Function * F , Type * RetTy , ArrayRef < Type * > Tys , TTI :: TargetCostKind CostKind ) {

return BaseT :: getCallInstrCost ( F , RetTy , Tys , CostKind ) ;


<s> return BaseT :: getCallInstrCost ( F , RetTy , Tys , CostKind ) ;


<s> std :: pair < int , MVT > SrcLT = TLI . getTypeLegalizationCost ( DL , SrcTy ) ; std :: pair < int , MVT > DstLT = TLI . getTypeLegalizationCost ( DL , DstTy ) ; unsigned Cost = std :: max ( SrcLT . first , DstLT . first ) + FloatFactor * ( SrcN + DstN ) ; if ( CostKind != TTI :: TCK_RecipThroughput ) return Cost == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ; return Cost ; } return <NUM_LIT> ;


<s> std :: pair < int , MVT > LT = TLI . getTypeLegalizationCost ( DL , ValTy ) ;

if ( Opcode == Instruction :: FCmp ) return LT . first + FloatFactor * getTypeNumElements ( ValTy ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getGatherScatterOpCost ( unsigned Opcode , Type * DataTy , Value * Ptr , bool VariableMask , unsigned Alignment , TTI :: TargetCostKind CostKind , const Instruction * I ) { return BaseT :: getGatherScatterOpCost ( Opcode , DataTy , Ptr , VariableMask , Alignment , CostKind , I ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getGatherScatterOpCost ( unsigned Opcode , Type * DataTy , Value * Ptr , bool VariableMask , unsigned Alignment , TTI :: TargetCostKind CostKind , const Instruction * I ) {

return BaseT :: getGatherScatterOpCost ( Opcode , DataTy , Ptr , VariableMask , Alignment , CostKind , I ) ;


<s> if ( ICA . getID ( ) == <STR_LIT> ) {

std :: pair < int , MVT > LT = TLI . getTypeLegalizationCost ( DL , ICA . getReturnType ( ) ) ;


<s> return BaseT :: getMaskedMemoryOpCost ( Opcode , Src , Alignment , AddressSpace , CostKind ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getMaxInterleaveFactor ( unsigned VF ) { return useHVX ( ) ? <NUM_LIT> : <NUM_LIT> ;


<s> assert ( RegWidth && <STR_LIT> ) ; if ( VecWidth % RegWidth == <NUM_LIT> ) return VecWidth / RegWidth ; const Align RegAlign ( RegWidth / <NUM_LIT> ) ; if ( ! Alignment || * Alignment > RegAlign ) Alignment = RegAlign ; assert ( Alignment ) ; unsigned AlignWidth = <NUM_LIT> * Alignment -> value ( ) ; unsigned NumLoads = alignTo ( VecWidth , AlignWidth ) / AlignWidth ; return <NUM_LIT> * NumLoads ; } unsigned Cost = VecTy -> getElementType ( ) -> isFloatingPointTy ( ) ? FloatFactor : <NUM_LIT> ; const Align BoundAlignment = std :: min ( Alignment . valueOrOne ( ) , Align ( <NUM_LIT> ) ) ; unsigned AlignWidth = <NUM_LIT> * BoundAlignment . value ( ) ; unsigned NumLoads = alignTo ( VecWidth , AlignWidth ) / AlignWidth ; if ( Alignment == Align ( <NUM_LIT> ) || Alignment == Align ( <NUM_LIT> ) ) return Cost * NumLoads ;

assert ( BoundAlignment <= Align ( <NUM_LIT> ) ) ;


<s> return ( <NUM_LIT> * ST . getVectorLength ( ) ) / ElemWidth ;


<s> unsigned <ISA_LIT>TTIImpl :: getMinimumVF ( unsigned ElemWidth ) const {

return ( <NUM_LIT> * ST . getVectorLength ( ) ) / ElemWidth ;


<s> unsigned <ISA_LIT>TTIImpl :: getMinVectorRegisterBitWidth ( ) const {

return useHVX ( ) ? ST . getVectorLength ( ) * <NUM_LIT> : <NUM_LIT> ;


<s> if ( Vector ) return useHVX ( ) ? <NUM_LIT> : <NUM_LIT> ;


<s> unsigned <ISA_LIT>TTIImpl :: getOperandsScalarizationOverhead ( ArrayRef < const Value * > Args , unsigned VF ) {

return BaseT :: getOperandsScalarizationOverhead ( Args , VF ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getOperandsScalarizationOverhead ( ArrayRef < const Value * > Args , unsigned VF ) { return BaseT :: getOperandsScalarizationOverhead ( Args , VF ) ;


<s> TargetTransformInfo :: PopcntSupportKind <ISA_LIT>TTIImpl :: getPopcntSupport ( unsigned IntTyWidthInBit ) const {

return TargetTransformInfo :: PSK_FastHardware ;


<s> unsigned <ISA_LIT>TTIImpl :: getPrefetchDistance ( ) const {


<s> unsigned <ISA_LIT>TTIImpl :: getRegisterBitWidth ( bool Vector ) const { return Vector ? getMinVectorRegisterBitWidth ( ) : <NUM_LIT> ;


<s> return Vector ? getMinVectorRegisterBitWidth ( ) : <NUM_LIT> ;


<s> return BaseT :: getScalarizationOverhead ( Ty , DemandedElts , Insert , Extract ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getShuffleCost ( TTI :: ShuffleKind Kind , Type * Tp , int Index , Type * SubTp ) { return <NUM_LIT> ;


<s> if ( auto * VTy = dyn_cast < FixedVectorType > ( Ty ) ) return VTy -> getNumElements ( ) ; assert ( ( Ty -> isIntegerTy ( ) || Ty -> isFloatingPointTy ( ) ) && <STR_LIT> ) ;


<s> unsigned <ISA_LIT>TTIImpl :: getTypeNumElements ( Type * Ty ) const {

if ( auto * VTy = dyn_cast < FixedVectorType > ( Ty ) ) return VTy -> getNumElements ( ) ;


<s> if ( L && L -> empty ( ) && canPeel ( L ) && SE . getSmallConstantTripCount ( L ) == <NUM_LIT> && SE . getSmallConstantMaxTripCount ( L ) > <NUM_LIT> && SE . getSmallConstantMaxTripCount ( L ) <= <NUM_LIT> ) {


<s> UP . Runtime = UP . Partial = true ; if ( L && L -> empty ( ) && canPeel ( L ) && SE . getSmallConstantTripCount ( L ) == <NUM_LIT> && SE . getSmallConstantMaxTripCount ( L ) > <NUM_LIT> && SE . getSmallConstantMaxTripCount ( L ) <= <NUM_LIT> ) { UP . PeelCount = <NUM_LIT> ;


<s> auto isCastFoldedIntoLoad = [ this ] ( const CastInst * CI ) -> bool { if ( ! CI -> isIntegerCast ( ) ) return false ; const DataLayout & DL = getDataLayout ( ) ; unsigned SBW = DL . getTypeSizeInBits ( CI -> getSrcTy ( ) ) ; unsigned DBW = DL . getTypeSizeInBits ( CI -> getDestTy ( ) ) ; if ( DBW != <NUM_LIT> || SBW >= DBW ) return false ; const LoadInst * LI = dyn_cast < const LoadInst > ( CI -> getOperand ( <NUM_LIT> ) ) ; return LI && LI -> hasOneUse ( ) ; } ; if ( const CastInst * CI = dyn_cast < const CastInst > ( U ) ) if ( isCastFoldedIntoLoad ( CI ) ) return TargetTransformInfo :: TCC_Free ;

return BaseT :: getUserCost ( U , Operands , CostKind ) ;


<s> if ( ElemTy -> isIntegerTy ( <NUM_LIT> ) ) return Cost ; return Cost + getVectorInstrCost ( Instruction :: ExtractElement , Val , Index ) ; } if ( Opcode == Instruction :: ExtractElement ) return <NUM_LIT> ; return <NUM_LIT> ;


<s> Type * ElemTy = Val -> isVectorTy ( ) ? cast < VectorType > ( Val ) -> getElementType ( ) : Val ; if ( Opcode == Instruction :: InsertElement ) { unsigned Cost = ( Index != <NUM_LIT> ) ? <NUM_LIT> : <NUM_LIT> ;

if ( ElemTy -> isIntegerTy ( <NUM_LIT> ) ) return Cost ;


<s> if ( ! cast < VectorType > ( VecTy ) -> getElementType ( ) -> isIntegerTy ( ) ) return false ; EVT VecVT = EVT :: getEVT ( VecTy ) ;

if ( ! VecVT . isSimple ( ) || VecVT . getSizeInBits ( ) <= <NUM_LIT> ) return false ;


<s> bool <ISA_LIT>TTIImpl :: shouldBuildLookupTables ( ) const { return EmitLookupTables ;


<s> bool <ISA_LIT>TTIImpl :: shouldBuildLookupTables ( ) const {


<s> bool <ISA_LIT>TTIImpl :: shouldFavorPostInc ( ) const { return true ;


<s> State . addLoc ( CCValAssign :: getReg ( ValNo , ValVT . getSimpleVT ( ) , Reg , LocVT . getSimpleVT ( ) , LocInfo ) ) ; return false ; } } } const Type * ArgTy = LocVT . getTypeForEVT ( State . getContext ( ) ) ; unsigned Alignment = State . getTarget ( ) . getSubtargetImpl ( ) -> getDataLayout ( ) -> getABITypeAlignment ( ArgTy ) ; unsigned Size = State . getTarget ( ) . getSubtargetImpl ( ) -> getDataLayout ( ) -> getTypeSizeInBits ( ArgTy ) / <NUM_LIT> ; if ( ArgFlags . isByVal ( ) ) { Size = ByValSize ; Alignment = <NUM_LIT> ; } unsigned Offset3 = State . AllocateStack ( Size , Alignment ) ; State . addLoc ( CCValAssign :: getMem ( ValNo , ValVT . getSimpleVT ( ) , Offset3 , LocVT . getSimpleVT ( ) , LocInfo ) ) ;


<s> <STR_LIT> , <STR_LIT> , <STR_LIT> } ; if ( unsigned Reg = State . AllocateReg ( RegList2 , <NUM_LIT> ) ) { State . addLoc ( CCValAssign :: getReg ( ValNo , ValVT . getSimpleVT ( ) , Reg , LocVT . getSimpleVT ( ) , LocInfo ) ) ; return false ; } } } const Type * ArgTy = LocVT . getTypeForEVT ( State . getContext ( ) ) ; unsigned Alignment = State . getTarget ( ) . getSubtargetImpl ( ) -> getDataLayout ( ) -> getABITypeAlignment ( ArgTy ) ; unsigned Size = State . getTarget ( ) . getSubtargetImpl ( ) -> getDataLayout ( ) -> getTypeSizeInBits ( ArgTy ) / <NUM_LIT> ; if ( ArgFlags . isByVal ( ) ) { Size = ByValSize ; Alignment = <NUM_LIT> ;


<s> <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; if ( unsigned Reg = State . AllocateReg ( RegList1 , <NUM_LIT> ) ) { State . addLoc ( CCValAssign :: getReg ( ValNo , ValVT . getSimpleVT ( ) , Reg , LocVT . getSimpleVT ( ) , LocInfo ) ) ; return false ; } } if ( LocVT == <STR_LIT> || LocVT == <STR_LIT> ) { static const unsigned RegList2 [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> } ; if ( unsigned Reg = State . AllocateReg ( RegList2 , <NUM_LIT> ) ) { State . addLoc ( CCValAssign :: getReg ( ValNo , ValVT . getSimpleVT ( ) , Reg , LocVT . getSimpleVT ( ) , LocInfo ) ) ; return false ; } } const Type * ArgTy = LocVT . getTypeForEVT ( State . getContext ( ) ) ; unsigned Alignment = State . getTarget ( ) . getSubtargetImpl ( ) -> getDataLayout ( ) -> getABITypeAlignment ( ArgTy ) ; unsigned Size = State . getTarget ( ) . getSubtargetImpl ( ) -> getDataLayout ( ) -> getTypeSizeInBits ( ArgTy ) / <NUM_LIT> ; unsigned Offset3 = State . AllocateStack ( Size , Alignment ) ;

State . addLoc ( CCValAssign :: getMem ( ValNo , ValVT . getSimpleVT ( ) , Offset3 , LocVT . getSimpleVT ( ) , LocInfo ) ) ;


<s> static bool CC_<ISA_LIT>32_VarArgs ( unsigned ValNo , EVT ValVT , EVT LocVT , CCValAssign :: LocInfo LocInfo , <STR_LIT> ArgFlags , <ISA_LIT>_CCState & State , int NonVarArgsParams , int CurrentParam , bool ForceMem ) { unsigned ByValSize = <NUM_LIT> ; if ( ArgFlags . isByVal ( ) && ( ( ByValSize = ArgFlags . getByValSize ( ) ) > ( MVT ( <STR_LIT> ) . getSizeInBits ( ) / <NUM_LIT> ) ) ) { ForceMem = true ; } if ( ! ForceMem && ( ( NonVarArgsParams == - <NUM_LIT> ) || ( CurrentParam <= NonVarArgsParams ) ) ) { if ( LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> ) { static const unsigned RegList1 [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; if ( unsigned Reg = State . AllocateReg ( RegList1 , <NUM_LIT> ) ) { State . addLoc ( CCValAssign :: getReg ( ValNo , ValVT . getSimpleVT ( ) , Reg , LocVT . getSimpleVT ( ) , LocInfo ) ) ; return false ; } }

if ( LocVT == <STR_LIT> || LocVT == <STR_LIT> ) {


<s> <STR_LIT> , <STR_LIT> , <STR_LIT> } ; if ( unsigned Reg = State . AllocateReg ( RegList2 , <NUM_LIT> ) ) { State . addLoc ( CCValAssign :: getReg ( ValNo , ValVT . getSimpleVT ( ) , Reg , LocVT . getSimpleVT ( ) , LocInfo ) ) ; return false ; } } const Type * ArgTy = LocVT . getTypeForEVT ( State . getContext ( ) ) ; unsigned Alignment = State . getTarget ( ) . getTargetData ( ) -> getABITypeAlignment ( ArgTy ) ; unsigned Size = State . getTarget ( ) . getTargetData ( ) -> getTypeSizeInBits ( ArgTy ) / <NUM_LIT> ; unsigned Offset3 = State . AllocateStack ( Size , Alignment ) ; State . addLoc ( CCValAssign :: getMem ( ValNo , ValVT . getSimpleVT ( ) , Offset3 , LocVT . getSimpleVT ( ) , LocInfo ) ) ; return false ;


<s> static bool CC_<ISA_LIT>32_VarArgs ( unsigned ValNo , EVT ValVT , EVT LocVT , CCValAssign :: LocInfo LocInfo , <STR_LIT> ArgFlags , <ISA_LIT>_CCState & State , int NonVarArgsParams , int CurrentParam , bool ForceMem ) { unsigned ByValSize = <NUM_LIT> ; if ( ArgFlags . isByVal ( ) && ( ( ByValSize = ArgFlags . getByValSize ( ) ) > ( MVT ( <STR_LIT> ) . getSizeInBits ( ) / <NUM_LIT> ) ) ) { ForceMem = true ; } if ( ! ForceMem && ( ( NonVarArgsParams == - <NUM_LIT> ) || ( CurrentParam <= NonVarArgsParams ) ) ) { if ( LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> ) { static const unsigned RegList1 [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; if ( unsigned Reg = State . AllocateReg ( RegList1 , <NUM_LIT> ) ) { State . addLoc ( CCValAssign :: getReg ( ValNo , ValVT . getSimpleVT ( ) , Reg , LocVT . getSimpleVT ( ) , LocInfo ) ) ; return false ; } } if ( LocVT == <STR_LIT> || LocVT == <STR_LIT> ) { static const unsigned RegList2 [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> } ; if ( unsigned Reg = State . AllocateReg ( RegList2 , <NUM_LIT> ) ) { State . addLoc ( CCValAssign :: getReg ( ValNo , ValVT . getSimpleVT ( ) , Reg , LocVT . getSimpleVT ( ) , LocInfo ) ) ; return false ; } } } const Type * ArgTy = LocVT . getTypeForEVT ( State . getContext ( ) ) ; unsigned Alignment = State . getTarget ( ) . getDataLayout ( ) -> getABITypeAlignment ( ArgTy ) ; unsigned Size = State . getTarget ( ) . getDataLayout ( ) -> getTypeSizeInBits ( ArgTy ) / <NUM_LIT> ; if ( ArgFlags . isByVal ( ) ) { Size = ByValSize ; Alignment = <NUM_LIT> ; }

unsigned Offset3 = State . AllocateStack ( Size , Alignment ) ;


<s> static const unsigned RegList2 [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> } ; if ( unsigned Reg = State . AllocateReg ( RegList2 , <NUM_LIT> ) ) { State . addLoc ( CCValAssign :: getReg ( ValNo , ValVT . getSimpleVT ( ) , Reg , LocVT . getSimpleVT ( ) , LocInfo ) ) ; return false ; } } } const Type * ArgTy = LocVT . getTypeForEVT ( State . getContext ( ) ) ; unsigned Alignment = State . getTarget ( ) . getDataLayout ( ) -> getABITypeAlignment ( ArgTy ) ; unsigned Size = State . getTarget ( ) . getDataLayout ( ) -> getTypeSizeInBits ( ArgTy ) / <NUM_LIT> ; if ( ArgFlags . isByVal ( ) ) { Size = ByValSize ; Alignment = <NUM_LIT> ; } unsigned Offset3 = State . AllocateStack ( Size , Alignment ) ;


<s> static const unsigned RegList1 [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; if ( unsigned Reg = State . AllocateReg ( RegList1 , <NUM_LIT> ) ) { State . addLoc ( CCValAssign :: getReg ( ValNo , ValVT . getSimpleVT ( ) , Reg , LocVT . getSimpleVT ( ) , LocInfo ) ) ; return false ; } } if ( LocVT == <STR_LIT> || LocVT == <STR_LIT> ) { static const unsigned RegList2 [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> } ; if ( unsigned Reg = State . AllocateReg ( RegList2 , <NUM_LIT> ) ) { State . addLoc ( CCValAssign :: getReg ( ValNo , ValVT . getSimpleVT ( ) , Reg , LocVT . getSimpleVT ( ) , LocInfo ) ) ; return false ; } } const Type * ArgTy = LocVT . getTypeForEVT ( State . getContext ( ) ) ;

unsigned Alignment = State . getTarget ( ) . getDataLayout ( ) -> getABITypeAlignment ( ArgTy ) ;


<s> if ( ! AI ) continue ; auto F = findBaseAndOffset ( * AI ) ; Instruction * GroupInst ; if ( Instruction * BI = F . first ) { AI -> Offset = F . second ; GroupInst = BI ; } else { WorkStack . push_back ( * AI ) ; GroupInst = AI -> Inst ; } AddrGroups [ GroupInst ] . push_back ( * AI ) ; } for ( DomTreeNode * C : DomN -> children ( ) ) Visit ( C , Visit ) ; while ( ! WorkStack . empty ( ) && WorkStack . back ( ) . Inst -> getParent ( ) == & Block ) WorkStack . pop_back ( ) ; } ; traverseBlock ( HVC . DT . getRootNode ( ) , traverseBlock ) ; assert ( WorkStack . empty ( ) ) ; erase_if ( AddrGroups , [ ] ( auto & G ) { return G . second . size ( ) == <NUM_LIT> ; } ) ; erase_if ( AddrGroups , [ & ] ( auto & G ) { return ! llvm :: any_of ( G . second , [ & ] ( auto & I ) { return HVC . HST . isTypeForHVX ( I . ValTy ) ; } ) ; } ) ; erase_if ( AddrGroups , [ & ] ( auto & G ) {

return llvm :: all_of ( G . second , [ & ] ( auto & I ) { return I . HaveAlign >= I . NeedAlign ; } ) ;


<s> if ( Instruction * BI = F . first ) { AI -> Offset = F . second ; GroupInst = BI ; } else { WorkStack . push_back ( * AI ) ; GroupInst = AI -> Inst ; } AddrGroups [ GroupInst ] . push_back ( * AI ) ; } for ( DomTreeNode * C : DomN -> children ( ) ) Visit ( C , Visit ) ; while ( ! WorkStack . empty ( ) && WorkStack . back ( ) . Inst -> getParent ( ) == & Block ) WorkStack . pop_back ( ) ; } ; traverseBlock ( HVC . DT . getRootNode ( ) , traverseBlock ) ; assert ( WorkStack . empty ( ) ) ; erase_if ( AddrGroups , [ ] ( auto & G ) { return G . second . size ( ) == <NUM_LIT> ; } ) ; erase_if ( AddrGroups , [ & ] ( auto & G ) { return ! llvm :: any_of ( G . second , [ & ] ( auto & I ) { return HVC . HST . isTypeForHVX ( I . ValTy ) ; } ) ; } ) ; erase_if ( AddrGroups , [ & ] ( auto & G ) { return llvm :: all_of ( G . second , [ & ] ( auto & I ) { return I . HaveAlign >= I . NeedAlign ; } ) ;


<s> if ( Adjust % ElemSize == <NUM_LIT> ) { Value * Tmp0 = Builder . CreateGEP ( Ptr , HVC . getConstInt ( Adjust / ElemSize ) ) ; return Builder . CreatePointerCast ( Tmp0 , ValTy -> getPointerTo ( ) ) ; }

PointerType * CharPtrTy = Type :: getInt8PtrTy ( HVC . F . getContext ( ) ) ;


<s> return Builder . CreatePointerCast ( Tmp0 , ValTy -> getPointerTo ( ) ) ; } PointerType * CharPtrTy = Type :: getInt8PtrTy ( HVC . F . getContext ( ) ) ; Value * Tmp0 = Builder . CreatePointerCast ( Ptr , CharPtrTy ) ; Value * Tmp1 = Builder . CreateGEP ( Tmp0 , HVC . getConstInt ( Adjust ) ) ;

return Builder . CreatePointerCast ( Tmp1 , ValTy -> getPointerTo ( ) ) ;


<s> auto AlignVectors :: createAlignedLoad ( IRBuilder < > & Builder , Type * ValTy , Value * Ptr , int Alignment , Value * Mask , Value * PassThru ) const -> Value * { assert ( ! HVC . isUndef ( Mask ) ) ; if ( HVC . isZero ( Mask ) ) return PassThru ;

if ( Mask == ConstantInt :: getTrue ( Mask -> getType ( ) ) ) return Builder . CreateAlignedLoad ( ValTy , Ptr , Align ( Alignment ) ) ;


<s> auto getLocOrNone = [ this ] ( const Instruction & I ) -> Optional < MemoryLocation > { if ( const auto * II = dyn_cast < IntrinsicInst > ( & I ) ) { switch ( II -> getIntrinsicID ( ) ) { case <STR_LIT> : return MemoryLocation :: getForArgument ( II , <NUM_LIT> , TLI ) ; case <STR_LIT> : return MemoryLocation :: getForArgument ( II , <NUM_LIT> , TLI ) ; } } return MemoryLocation :: getOrNone ( & I ) ; } ; const BasicBlock & Block = * In . getParent ( ) ; assert ( Block . begin ( ) == To || Block . end ( ) == To || To -> getParent ( ) == & Block ) ; if ( isa < PHINode > ( In ) || ( To != Block . end ( ) && isa < PHINode > ( * To ) ) ) return false ; if ( ! mayBeMemoryDependent ( In ) ) return true ; bool MayWrite = In . mayWriteToMemory ( ) ; auto MaybeLoc = getLocOrNone ( In ) ;


<s> bool MoveUp = ( To != Block . end ( ) && To -> comesBefore ( & In ) ) ; auto Range = MoveUp ? std :: make_pair ( To , From ) : std :: make_pair ( std :: next ( From ) , To ) ; for ( auto It = Range . first ; It != Range . second ; ++ It ) { const Instruction & I = * It ; if ( llvm :: is_contained ( Ignore , & I ) ) continue ; if ( I . mayThrow ( ) ) return false ; if ( auto * CB = dyn_cast < CallBase > ( & I ) ) { if ( ! CB -> hasFnAttr ( Attribute :: WillReturn ) ) return false ; if ( ! CB -> hasFnAttr ( Attribute :: NoSync ) ) return false ; } if ( I . mayReadOrWriteMemory ( ) ) { auto MaybeLocI = getLocOrNone ( I ) ; if ( MayWrite || I . mayWriteToMemory ( ) ) { if ( ! MaybeLoc || ! MaybeLocI ) return false ; if ( ! AA . isNoAlias ( * MaybeLoc , * MaybeLocI ) ) return false ;


<s> auto & TM = getAnalysis < TargetPassConfig > ( ) . getTM < <ISA_LIT>TargetMachine > ( ) ;

<ISA_LIT>VectorCombine HVC ( F , AA , AC , DT , TLI , TM ) ;


<s> } else { WorkStack . push_back ( * AI ) ; GroupInst = AI -> Inst ; } AddrGroups [ GroupInst ] . push_back ( * AI ) ; } for ( DomTreeNode * C : DomN -> children ( ) ) Visit ( C , Visit ) ; while ( ! WorkStack . empty ( ) && WorkStack . back ( ) . Inst -> getParent ( ) == & Block ) WorkStack . pop_back ( ) ; } ; traverseBlock ( HVC . DT . getRootNode ( ) , traverseBlock ) ; assert ( WorkStack . empty ( ) ) ; erase_if ( AddrGroups , [ ] ( auto & G ) { return G . second . size ( ) == <NUM_LIT> ; } ) ; erase_if ( AddrGroups , [ & ] ( auto & G ) { return llvm :: none_of ( G . second , [ & ] ( auto & I ) { return HVC . HST . isTypeForHVX ( I . ValTy ) ; } ) ; } ) ;


<s> int ElemSize = HVC . getAllocSizeOf ( ElemTy ) ; if ( Adjust % ElemSize == <NUM_LIT> && Adjust != <NUM_LIT> ) { Value * Tmp0 = Builder . CreateGEP ( ElemTy , Ptr , HVC . getConstInt ( Adjust / ElemSize ) ) ; return Builder . CreatePointerCast ( Tmp0 , ValTy -> getPointerTo ( ) ) ; } } PointerType * CharPtrTy = Type :: getInt8PtrTy ( HVC . F . getContext ( ) ) ; Value * Tmp0 = Builder . CreatePointerCast ( Ptr , CharPtrTy ) ; Value * Tmp1 = Builder . CreateGEP ( Type :: getInt8Ty ( HVC . F . getContext ( ) ) , Tmp0 , HVC . getConstInt ( Adjust ) ) ;


<s> if ( isa < PHINode > ( In ) || ( To != Block . end ( ) && isa < PHINode > ( * To ) ) ) return false ; if ( ! mayHaveNonDefUseDependency ( In ) ) return true ; bool MayWrite = In . mayWriteToMemory ( ) ; auto MaybeLoc = getLocOrNone ( In ) ; auto From = In . getIterator ( ) ; if ( From == To ) return true ; bool MoveUp = ( To != Block . end ( ) && To -> comesBefore ( & In ) ) ; auto Range = MoveUp ? std :: make_pair ( To , From ) : std :: make_pair ( std :: next ( From ) , To ) ; for ( auto It = Range . first ; It != Range . second ; ++ It ) { const Instruction & I = * It ; if ( llvm :: is_contained ( Ignore , & I ) ) continue ; if ( auto * II = dyn_cast < IntrinsicInst > ( & I ) ) { if ( II -> getIntrinsicID ( ) == <STR_LIT> ) continue ; } if ( I . mayThrow ( ) ) return false ; if ( auto * CB = dyn_cast < CallBase > ( & I ) ) { if ( ! CB -> hasFnAttr ( Attribute :: WillReturn ) ) return false ; if ( ! CB -> hasFnAttr ( Attribute :: NoSync ) ) return false ; } if ( I . mayReadOrWriteMemory ( ) ) { auto MaybeLocI = getLocOrNone ( I ) ; if ( MayWrite || I . mayWriteToMemory ( ) ) { if ( ! MaybeLoc || ! MaybeLocI ) return false ;

if ( ! AA . isNoAlias ( * MaybeLoc , * MaybeLocI ) ) return false ;


<s> auto MaybeLoc = getLocOrNone ( In ) ; auto From = In . getIterator ( ) ; if ( From == To ) return true ; bool MoveUp = ( To != Block . end ( ) && To -> comesBefore ( & In ) ) ; auto Range = MoveUp ? std :: make_pair ( To , From ) : std :: make_pair ( std :: next ( From ) , To ) ; for ( auto It = Range . first ; It != Range . second ; ++ It ) { const Instruction & I = * It ; if ( llvm :: is_contained ( Ignore , & I ) ) continue ; if ( auto * II = dyn_cast < IntrinsicInst > ( & I ) ) { if ( II -> getIntrinsicID ( ) == <STR_LIT> ) continue ;


<s> int FromSize = getSizeOf ( FromSTy ) ; int ToSize = getSizeOf ( ToSTy ) ; assert ( FromSize % ToSize == <NUM_LIT> || ToSize % FromSize == <NUM_LIT> ) ; auto * MaskTy = cast < VectorType > ( Mask -> getType ( ) ) ; int FromCount = MaskTy -> getElementCount ( ) . getFixedValue ( ) ; int ToCount = ( FromCount * FromSize ) / ToSize ; assert ( ( FromCount * FromSize ) % ToSize == <NUM_LIT> ) ;

auto * FromITy = IntegerType :: get ( F . getContext ( ) , FromSize * <NUM_LIT> ) ;


<s> assert ( ( FromCount * FromSize ) % ToSize == <NUM_LIT> ) ; auto * FromITy = IntegerType :: get ( F . getContext ( ) , FromSize * <NUM_LIT> ) ; auto * ToITy = IntegerType :: get ( F . getContext ( ) , ToSize * <NUM_LIT> ) ; Value * Ext = Builder . CreateSExt ( Mask , VectorType :: get ( FromITy , FromCount , false ) ) ; Value * Cast = Builder . CreateBitCast ( Ext , VectorType :: get ( ToITy , ToCount , false ) ) ;

return Builder . CreateTrunc ( Cast , VectorType :: get ( getBoolTy ( ) , ToCount , false ) ) ;


<s> std :: vector < Value * > Work [ <NUM_LIT> ] ; int ThisW = <NUM_LIT> , OtherW = <NUM_LIT> ; Work [ ThisW ] . assign ( Vecs . begin ( ) , Vecs . end ( ) ) ; while ( Work [ ThisW ] . size ( ) > <NUM_LIT> ) { auto * Ty = cast < VectorType > ( Work [ ThisW ] . front ( ) -> getType ( ) ) ; int ElemCount = Ty -> getElementCount ( ) . getFixedValue ( ) ; SMask . resize ( ElemCount * <NUM_LIT> ) ; std :: iota ( SMask . begin ( ) , SMask . end ( ) , <NUM_LIT> ) ; Work [ OtherW ] . clear ( ) ;

if ( Work [ ThisW ] . size ( ) % <NUM_LIT> != <NUM_LIT> ) Work [ ThisW ] . push_back ( UndefValue :: get ( Ty ) ) ;


<s> std :: iota ( SMask . begin ( ) , SMask . end ( ) , <NUM_LIT> ) ; Work [ OtherW ] . clear ( ) ; if ( Work [ ThisW ] . size ( ) % <NUM_LIT> != <NUM_LIT> ) Work [ ThisW ] . push_back ( UndefValue :: get ( Ty ) ) ; for ( int i = <NUM_LIT> , e = Work [ ThisW ] . size ( ) ; i < e ; i += <NUM_LIT> ) { Value * Joined = Builder . CreateShuffleVector ( Work [ ThisW ] [ i ] , Work [ ThisW ] [ i + <NUM_LIT> ] , SMask ) ; Work [ OtherW ] . push_back ( Joined ) ; } std :: swap ( ThisW , OtherW ) ; } SMask . resize ( Vecs . size ( ) * getSizeOf ( Vecs . front ( ) -> getType ( ) ) ) ; std :: iota ( SMask . begin ( ) , SMask . end ( ) , <NUM_LIT> ) ; Value * Total = Work [ OtherW ] . front ( ) ;

return Builder . CreateShuffleVector ( Total , UndefValue :: get ( Total -> getType ( ) ) , SMask ) ;


<s> if ( auto * I = dyn_cast < Instruction > ( Op ) ) { if ( I -> getParent ( ) == Parent && Base -> comesBefore ( I ) ) WorkQ . push_back ( I ) ; } } } return Deps ; } ; auto tryAddTo = [ & ] ( const AddrInfo & Info , MoveGroup & Move ) { assert ( ! Move . Main . empty ( ) && <STR_LIT> ) ; if ( Move . IsHvx != isHvx ( Info ) ) return false ; Instruction * Base = Move . Main . front ( ) ; if ( Base -> getParent ( ) != Info . Inst -> getParent ( ) ) return false ; auto isSafeToMoveToBase = [ & ] ( const Instruction * I ) { return HVC . isSafeToMoveBeforeInBB ( * I , Base -> getIterator ( ) ) ; } ;

DepList Deps = getUpwardDeps ( Info . Inst , Base ) ;


<s> auto * PtrTy = cast < PointerType > ( Ptr -> getType ( ) ) ; if ( ! PtrTy -> isOpaque ( ) ) { Type * ElemTy = PtrTy -> getElementType ( ) ; int ElemSize = HVC . getSizeOf ( ElemTy ) ; if ( Adjust % ElemSize == <NUM_LIT> ) {

Value * Tmp0 = Builder . CreateGEP ( ElemTy , Ptr , HVC . getConstInt ( Adjust / ElemSize ) ) ;


<ISA_LIT>VectorCombineLegacy ( ) : FunctionPass ( ID ) {


<s> AddrInfo ( const <ISA_LIT>VectorCombine & HVC , Instruction * I , Value * A , Type * T , Align H ) : Inst ( I ) , Addr ( A ) , ValTy ( T ) , HaveAlign ( H ) , NeedAlign ( HVC . getTypeAlignment ( ValTy ) ) {


<s> AlignVectors ( <ISA_LIT>VectorCombine & HVC_ ) : HVC ( HVC_ ) {


<s> Block ( Value * Val , int Off , int Len , int Pos ) : Seg ( Val , Off , Len ) , Pos ( Pos ) {


<s> Builder ( BasicBlock * B ) : IRBuilder < > ( B ) {


<s> if ( Work [ ThisW ] . size ( ) % <NUM_LIT> != <NUM_LIT> ) Work [ ThisW ] . push_back ( UndefValue :: get ( Ty ) ) ; for ( int i = <NUM_LIT> , e = Work [ ThisW ] . size ( ) ; i < e ; i += <NUM_LIT> ) { Value * Joined = Builder . CreateShuffleVector ( Work [ ThisW ] [ i ] , Work [ ThisW ] [ i + <NUM_LIT> ] , SMask ) ; Work [ OtherW ] . push_back ( Joined ) ; } std :: swap ( ThisW , OtherW ) ; } SMask . resize ( Vecs . size ( ) * getSizeOf ( Vecs . front ( ) -> getType ( ) ) ) ; std :: iota ( SMask . begin ( ) , SMask . end ( ) , <NUM_LIT> ) ; Value * Total = Work [ OtherW ] . front ( ) ; return Builder . CreateShuffleVector ( Total , SMask ) ;


<s> Value * Joined = Builder . CreateShuffleVector ( Work [ ThisW ] [ i ] , Work [ ThisW ] [ i + <NUM_LIT> ] , SMask ) ; Work [ OtherW ] . push_back ( Joined ) ; } std :: swap ( ThisW , OtherW ) ; } SMask . resize ( Vecs . size ( ) * getSizeOf ( Vecs . front ( ) -> getType ( ) ) ) ; std :: iota ( SMask . begin ( ) , SMask . end ( ) , <NUM_LIT> ) ; Value * Total = Work [ OtherW ] . front ( ) ; return Builder . CreateShuffleVector ( Total , SMask ) ;


<s> for ( Instruction & I : Block ) { auto AI = this -> getAddrInfo ( I ) ; if ( ! AI ) continue ; auto F = findBaseAndOffset ( * AI ) ; Instruction * GroupInst ; if ( Instruction * BI = F . first ) { AI -> Offset = F . second ; GroupInst = BI ; } else { WorkStack . push_back ( * AI ) ; GroupInst = AI -> Inst ; } AddrGroups [ GroupInst ] . push_back ( * AI ) ; } for ( DomTreeNode * C : DomN -> children ( ) ) Visit ( C , Visit ) ; while ( ! WorkStack . empty ( ) && WorkStack . back ( ) . Inst -> getParent ( ) == & Block ) WorkStack . pop_back ( ) ; } ; traverseBlock ( HVC . DT . getRootNode ( ) , traverseBlock ) ; assert ( WorkStack . empty ( ) ) ; erase_if ( AddrGroups , [ ] ( auto & G ) { return G . second . size ( ) == <NUM_LIT> ; } ) ; erase_if ( AddrGroups , [ & ] ( auto & G ) { return ! llvm :: any_of ( G . second , [ & ] ( auto & I ) { return HVC . HST . isTypeForHVX ( I . ValTy ) ; } ) ; } ) ;


<s> auto AlignVectors :: createAdjustedPointer ( IRBuilder < > & Builder , Value * Ptr , Type * ValTy , int Adjust ) const -> Value * { auto * PtrTy = cast < PointerType > ( Ptr -> getType ( ) ) ; if ( ! PtrTy -> isOpaque ( ) ) { Type * ElemTy = PtrTy -> getElementType ( ) ; int ElemSize = HVC . getAllocSizeOf ( ElemTy ) ; if ( Adjust % ElemSize == <NUM_LIT> && Adjust != <NUM_LIT> ) { Value * Tmp0 = Builder . CreateGEP ( ElemTy , Ptr , HVC . getConstInt ( Adjust / ElemSize ) ) ; return Builder . CreatePointerCast ( Tmp0 , ValTy -> getPointerTo ( ) ) ;


<s> if ( Mask == ConstantInt :: getTrue ( Mask -> getType ( ) ) ) return Builder . CreateAlignedLoad ( ValTy , Ptr , Align ( Alignment ) ) ;

return Builder . CreateMaskedLoad ( ValTy , Ptr , Align ( Alignment ) , Mask , PassThru ) ;


<s> auto AlignVectors :: createAlignedLoad ( IRBuilder < > & Builder , Type * ValTy , Value * Ptr , int Alignment , Value * Mask , Value * PassThru ) const -> Value * { assert ( ! HVC . isUndef ( Mask ) ) ; if ( HVC . isZero ( Mask ) ) return PassThru ;


<s> if ( HVC . isZero ( Mask ) || HVC . isUndef ( Val ) || HVC . isUndef ( Mask ) ) return UndefValue :: get ( Val -> getType ( ) ) ;

if ( Mask == ConstantInt :: getTrue ( Mask -> getType ( ) ) ) return Builder . CreateAlignedStore ( Val , Ptr , Align ( Alignment ) ) ;


<s> if ( SrcTy == DestTy ) return Val ; if ( HST . isTypeForHVX ( SrcTy , true ) ) { if ( cast < VectorType > ( SrcTy ) -> getElementType ( ) == BoolTy ) { <STR_LIT> TC = HwLen == <NUM_LIT> ? <STR_LIT> : <STR_LIT> ; Function * FI = <STR_LIT> ( F . getParent ( ) , TC , { DestTy , Val -> getType ( ) } ) ; return Builder . CreateCall ( FI , { Val } ) ; } return Builder . CreateBitCast ( Val , DestTy ) ; } llvm_unreachable ( <STR_LIT> ) ; } ; SmallVector < Value * , <NUM_LIT> > IntOps ; for ( Value * A : Args ) IntOps . push_back ( getCast ( Builder , A , getTypeForIntrin ( A -> getType ( ) ) ) ) ; Function * FI = <STR_LIT> ( F . getParent ( ) , IntID ) ; Value * Call = Builder . CreateCall ( FI , IntOps ) ; Type * CallTy = Call -> getType ( ) ; if ( CallTy == RetTy ) return Call ; assert ( HST . isTypeForHVX ( CallTy , true ) ) ; if ( cast < VectorType > ( CallTy ) -> getElementType ( ) == BoolTy ) return getCast ( Builder , Call , RetTy ) ;

return Builder . CreateBitCast ( Call , RetTy ) ;


<s> for ( Value * Op : D -> operands ( ) ) { if ( auto * I = dyn_cast < Instruction > ( Op ) ) { if ( I -> getParent ( ) == Parent && Base -> comesBefore ( I ) ) WorkQ . push_back ( I ) ; } } } return Deps ; } ; auto tryAddTo = [ & ] ( const AddrInfo & Info , MoveGroup & Move ) { assert ( ! Move . Main . empty ( ) && <STR_LIT> ) ; if ( Move . IsHvx != isHvx ( Info ) ) return false ; Instruction * Base = Move . Main . front ( ) ; if ( Base -> getParent ( ) != Info . Inst -> getParent ( ) ) return false ; auto isSafeToMoveToBase = [ & ] ( const Instruction * I ) { return HVC . isSafeToMoveBeforeInBB ( * I , Base -> getIterator ( ) ) ; } ; DepList Deps = getUpwardDeps ( Info . Inst , Base ) ; if ( ! llvm :: all_of ( Deps , isSafeToMoveToBase ) ) return false ; Deps . erase ( Info . Inst ) ; auto inAddrMap = [ & ] ( Instruction * I ) { return AddrGroups . count ( I ) > <NUM_LIT> ; } ; if ( llvm :: any_of ( Deps , inAddrMap ) ) return false ; Move . Main . push_back ( Info . Inst ) ; llvm :: append_range ( Move . Deps , Deps ) ;


<s> } } return Deps ; } ; auto tryAddTo = [ & ] ( const AddrInfo & Info , MoveGroup & Move ) { assert ( ! Move . Main . empty ( ) && <STR_LIT> ) ; if ( Move . IsHvx != isHvx ( Info ) ) return false ; Instruction * Base = Move . Main . front ( ) ; if ( Base -> getParent ( ) != Info . Inst -> getParent ( ) ) return false ; auto isSafeToMoveToBase = [ & ] ( const Instruction * I ) { return HVC . isSafeToMoveBeforeInBB ( * I , Base -> getIterator ( ) ) ; } ; DepList Deps = getUpwardDeps ( Info . Inst , Base ) ; if ( ! llvm :: all_of ( Deps , isSafeToMoveToBase ) ) return false ; Deps . erase ( Info . Inst ) ; auto inAddrMap = [ & ] ( Instruction * I ) { return AddrGroups . count ( I ) > <NUM_LIT> ; } ; if ( llvm :: any_of ( Deps , inAddrMap ) ) return false ;

Move . Main . push_back ( Info . Inst ) ;


<s> const AddrInfo & Info = * I ; if ( ! Info . Inst -> mayWriteToMemory ( ) ) continue ; if ( StoreGroups . empty ( ) || ! tryAddTo ( Info , StoreGroups . back ( ) ) ) StoreGroups . emplace_back ( Info , Group . front ( ) . Inst , isHvx ( Info ) , false ) ; }

erase_if ( StoreGroups , [ ] ( const MoveGroup & G ) { return G . Main . size ( ) <= <NUM_LIT> ; } ) ;


<s> void erase_if ( T && container , Pred p ) {

llvm :: erase_if ( std :: forward < T > ( container ) , p ) ;


<s> auto AlignVectors :: getAddrInfo ( Instruction & In ) const -> Optional < AddrInfo > { if ( auto * L = isCandidate < LoadInst > ( & In ) ) return AddrInfo ( HVC , L , L -> getPointerOperand ( ) , L -> getType ( ) , L -> getAlign ( ) ) ; if ( auto * S = isCandidate < StoreInst > ( & In ) ) return AddrInfo ( HVC , S , S -> getPointerOperand ( ) , S -> getValueOperand ( ) -> getType ( ) , S -> getAlign ( ) ) ; if ( auto * II = isCandidate < IntrinsicInst > ( & In ) ) { <STR_LIT> ID = II -> getIntrinsicID ( ) ;


<s> auto AlignVectors :: getAlignFromValue ( const Value * V ) const -> Align {

const auto * C = dyn_cast < ConstantInt > ( V ) ;


<s> const auto * C = dyn_cast < ConstantInt > ( V ) ; assert ( C && <STR_LIT> ) ;


<s> return DL . getTypeAllocSize ( const_cast < Type * > ( Ty ) ) . getFixedValue ( ) ;


<s> AU . addRequired < AssumptionCacheTracker > ( ) ; AU . addRequired < DominatorTreeWrapperPass > ( ) ; AU . addRequired < TargetLibraryInfoWrapperPass > ( ) ; AU . addRequired < TargetPassConfig > ( ) ;

FunctionPass :: getAnalysisUsage ( AU ) ;


<s> AU . addRequired < AssumptionCacheTracker > ( ) ; AU . addRequired < DominatorTreeWrapperPass > ( ) ; AU . addRequired < TargetLibraryInfoWrapperPass > ( ) ;

AU . addRequired < TargetPassConfig > ( ) ;


<s> assert ( ElemCount >= <NUM_LIT> ) ; IntegerType * BoolTy = Type :: getInt1Ty ( F . getContext ( ) ) ;

if ( ElemCount == <NUM_LIT> ) return BoolTy ;


<s> auto <STR_LIT> ( int ElemCount ) const -> Type * { assert ( ElemCount >= <NUM_LIT> ) ; IntegerType * BoolTy = Type :: getInt1Ty ( F . getContext ( ) ) ; if ( ElemCount == <NUM_LIT> ) return BoolTy ; return VectorType :: get ( BoolTy , ElemCount , false ) ;


<s> IntegerType * ByteTy = Type :: getInt8Ty ( F . getContext ( ) ) ; if ( ElemCount == <NUM_LIT> ) return ByteTy ;

return VectorType :: get ( ByteTy , ElemCount , false ) ;


<s> auto <STR_LIT> ( int ElemCount ) const -> Type * { assert ( ElemCount >= <NUM_LIT> ) ; IntegerType * ByteTy = Type :: getInt8Ty ( F . getContext ( ) ) ; if ( ElemCount == <NUM_LIT> ) return ByteTy ; return VectorType :: get ( ByteTy , ElemCount , false ) ;


<s> auto <STR_LIT> ( int Val ) const -> ConstantInt * { return ConstantInt :: getSigned ( getIntTy ( ) , Val ) ;


<s> return ConstantInt :: getSigned ( getIntTy ( ) , Val ) ;


<s> assert ( <NUM_LIT> <= Start && Start < Length ) ; SmallVector < int , <NUM_LIT> > SMask ( Length ) ; std :: iota ( SMask . begin ( ) , SMask . end ( ) , Start ) ;

return Builder . CreateShuffleVector ( Lo , Hi , SMask ) ;


<s> if ( auto * VecTy = dyn_cast < VectorType > ( Ty ) ) return ConstantVector :: getSplat ( VecTy -> getElementCount ( ) , Minus1 ) ;


<s> T * getIfUnordered ( T * MaybeT ) { return MaybeT && MaybeT -> isUnordered ( ) ? MaybeT : nullptr ;


<s> return Type :: getInt32Ty ( F . getContext ( ) ) ;


<s> auto <STR_LIT> ( const Value * Val ) const -> Optional < APInt > { if ( auto * CI = dyn_cast < ConstantInt > ( Val ) ) return CI -> getValue ( ) ; return None ;


<s> } Type * ValTy = getPayload ( Val ) -> getType ( ) ; if ( auto * VecTy = dyn_cast < VectorType > ( ValTy ) ) {

int ElemCount = VecTy -> getElementCount ( ) . getFixedValue ( ) ;


<s> return II -> getArgOperand ( <NUM_LIT> ) ; case <STR_LIT> : return II -> getArgOperand ( <NUM_LIT> ) ; } } Type * ValTy = getPayload ( Val ) -> getType ( ) ;

if ( auto * VecTy = dyn_cast < VectorType > ( ValTy ) ) {


<s> if ( auto * VecTy = dyn_cast < VectorType > ( Ty ) ) return ConstantVector :: getSplat ( VecTy -> getElementCount ( ) , Zero ) ; return Zero ;


<s> if ( auto * VecTy = dyn_cast < VectorType > ( Ty ) ) return ConstantVector :: getSplat ( VecTy -> getElementCount ( ) , Zero ) ;


<s> StringRef getPassName ( ) const override { return <STR_LIT> ;


<s> auto AlignVectors :: getPassThrough ( Value * Val ) const -> Value * { if ( auto * II = dyn_cast < IntrinsicInst > ( Val ) ) { if ( II -> getIntrinsicID ( ) == <STR_LIT> ) return II -> getArgOperand ( <NUM_LIT> ) ; } return UndefValue :: get ( getPayload ( Val ) -> getType ( ) ) ;


<s> if ( auto * In = dyn_cast < Instruction > ( Val ) ) { <STR_LIT> ID = <NUM_LIT> ; if ( auto * II = dyn_cast < IntrinsicInst > ( In ) ) ID = II -> getIntrinsicID ( ) ; if ( isa < StoreInst > ( In ) || ID == <STR_LIT> ) return In -> getOperand ( <NUM_LIT> ) ; } return Val ;


<s> if ( isa < StoreInst > ( In ) || ID == <STR_LIT> ) return In -> getOperand ( <NUM_LIT> ) ; } return Val ;


<s> auto <STR_LIT> ( const Type * Ty ) const -> int {

return DL . getTypeStoreSize ( const_cast < Type * > ( Ty ) ) . getFixedValue ( ) ;


<s> return DL . getABITypeAlign ( Ty ) . value ( ) ;


<s> <ISA_LIT>VectorCombine ( Function & F_ , AliasAnalysis & AA_ , AssumptionCache & AC_ , DominatorTree & DT_ , TargetLibraryInfo & TLI_ , const TargetMachine & TM_ ) : F ( F_ ) , DL ( F . getParent ( ) -> getDataLayout ( ) ) , AA ( AA_ ) , AC ( AC_ ) , DT ( DT_ ) , TLI ( TLI_ ) , HST ( static_cast < const <ISA_LIT>Subtarget & > ( * TM_ . getSubtargetImpl ( F ) ) ) {


<s> <ISA_LIT>VectorCombineLegacy ( ) : FunctionPass ( ID ) {


<s> assert ( <NUM_LIT> <= Start && Start + Length <= SrcLen ) ; assert ( <NUM_LIT> <= Where && Where + Length <= DstLen ) ; int P2Len = PowerOf2Ceil ( SrcLen | DstLen ) ; auto * Undef = UndefValue :: get ( getByteTy ( ) ) ; Value * P2Src = vresize ( Builder , Src , P2Len , Undef ) ; Value * P2Dst = vresize ( Builder , Dst , P2Len , Undef ) ; SmallVector < int , <NUM_LIT> > SMask ( P2Len ) ; for ( int i = <NUM_LIT> ; i != P2Len ; ++ i ) { SMask [ i ] = ( Where <= i && i < Where + Length ) ? P2Len + Start + ( i - Where ) : i ;


<s> if ( auto * VecTy = dyn_cast < VectorType > ( Ty ) ) return VecTy -> getElementType ( ) == getByteTy ( ) ;


<s> T * isCandidate ( Instruction * In ) { return dyn_cast < T > ( In ) ;


<s> return HVC . HST . isTypeForHVX ( AI . ValTy ) ;


<s> return MemoryLocation :: getOrNone ( & I ) ; } ; const BasicBlock & Block = * In . getParent ( ) ; assert ( Block . begin ( ) == To || Block . end ( ) == To || To -> getParent ( ) == & Block ) ; if ( isa < PHINode > ( In ) || ( To != Block . end ( ) && isa < PHINode > ( * To ) ) ) return false ; if ( ! mayBeMemoryDependent ( In ) ) return true ; bool MayWrite = In . mayWriteToMemory ( ) ; auto MaybeLoc = getLocOrNone ( In ) ; auto From = In . getIterator ( ) ; if ( From == To ) return true ; bool MoveUp = ( To != Block . end ( ) && To -> comesBefore ( & In ) ) ; auto Range = MoveUp ? std :: make_pair ( To , From ) : std :: make_pair ( std :: next ( From ) , To ) ; for ( auto It = Range . first ; It != Range . second ; ++ It ) { const Instruction & I = * It ; if ( llvm :: is_contained ( Ignore , & I ) ) continue ; if ( auto * II = dyn_cast < IntrinsicInst > ( & I ) ) { if ( II -> getIntrinsicID ( ) == <STR_LIT> ) continue ; } if ( I . mayThrow ( ) ) return false ; if ( auto * CB = dyn_cast < CallBase > ( & I ) ) { if ( ! CB -> hasFnAttr ( Attribute :: WillReturn ) ) return false ; if ( ! CB -> hasFnAttr ( Attribute :: NoSync ) ) return false ; } if ( I . mayReadOrWriteMemory ( ) ) { auto MaybeLocI = getLocOrNone ( I ) ; if ( MayWrite || I . mayWriteToMemory ( ) ) { if ( ! MaybeLoc || ! MaybeLocI ) return false ; if ( ! AA . isNoAlias ( * MaybeLoc , * MaybeLocI ) ) return false ; }


<s> auto <STR_LIT> ( Type * Ty ) const -> bool { if ( ! isByteVecTy ( Ty ) ) return false ; int Size = getSizeOf ( Ty ) ; if ( HST . isTypeForHVX ( Ty ) ) return Size == static_cast < int > ( HST . getVectorLength ( ) ) ; return Size == <NUM_LIT> || Size == <NUM_LIT> ;


if ( HST . isTypeForHVX ( Ty ) ) return Size == static_cast < int > ( HST . getVectorLength ( ) ) ;


<s> auto <STR_LIT> ( const Value * Val ) const -> bool { return isa < UndefValue > ( Val ) ;


<s> auto <STR_LIT> ( const Value * Val ) const -> bool { if ( auto * C = dyn_cast < Constant > ( Val ) ) return C -> isZeroValue ( ) ;


<s> if ( Move . IsLoad ) { for ( Instruction * D : Move . Deps ) D -> moveBefore ( Where ) ; ArrayRef < Instruction * > Main ( Move . Main ) ; for ( Instruction * M : Main . drop_front ( <NUM_LIT> ) ) { M -> moveAfter ( Where ) ; Where = M ; } } else {


<s> AI . Inst , IsHvx ( Hvx ) , IsLoad ( Load ) {


<s> } ; for ( int i = ( DoAlign ? - <NUM_LIT> : <NUM_LIT> ) ; i != NumSectors + DoAlign ; ++ i ) { ByteSpan VSection = VSpan . section ( i * ScLen , ScLen ) . shift ( - i * ScLen ) ; Value * AccumV = UndefValue :: get ( SecTy ) ; Value * AccumM = HVC . getNullValue ( SecTy ) ; for ( ByteSpan :: Block & S : VSection ) { Value * Pay = getPayload ( S . Seg . Val ) ; Value * Mask = HVC . rescale ( Builder , MakeVec ( Builder , getMask ( S . Seg . Val ) ) , Pay -> getType ( ) , HVC . getByteTy ( ) ) ; AccumM = HVC . insertb ( Builder , AccumM , HVC . vbytes ( Builder , Mask ) , S . Seg . Start , S . Seg . Size , S . Pos ) ; AccumV = HVC . insertb ( Builder , AccumV , HVC . vbytes ( Builder , Pay ) , S . Seg . Start , S . Seg . Size , S . Pos ) ; } ASpanV . Blocks . emplace_back ( AccumV , ScLen , i * ScLen ) ; ASpanM . Blocks . emplace_back ( AccumM , ScLen , i * ScLen ) ; } if ( DoAlign ) { for ( int j = <NUM_LIT> ; j != NumSectors + <NUM_LIT> ; ++ j ) { ASpanV [ j - <NUM_LIT> ] . Seg . Val = HVC . vlalignb ( Builder , ASpanV [ j - <NUM_LIT> ] . Seg . Val , ASpanV [ j ] . Seg . Val , AlignVal ) ; ASpanM [ j - <NUM_LIT> ] . Seg . Val = HVC . vlalignb ( Builder , ASpanM [ j - <NUM_LIT> ] . Seg . Val , ASpanM [ j ] . Seg . Val , AlignVal ) ; } } for ( int i = <NUM_LIT> ; i != NumSectors + DoAlign ; ++ i ) { Value * Ptr = createAdjustedPointer ( Builder , AlignAddr , SecTy , i * ScLen ) ; Value * Val = ASpanV [ i ] . Seg . Val ; Value * Mask = ASpanM [ i ] . Seg . Val ; if ( ! HVC . isUndef ( Val ) && ! HVC . isZero ( Mask ) ) { Value * Store = createAlignedStore ( Builder , Val , Ptr , ScLen , HVC . vlsb ( Builder , Mask ) ) ; int Start = ( i - DoAlign ) * ScLen ; int Width = ( <NUM_LIT> + DoAlign ) * ScLen ; propagateMetadata ( cast < Instruction > ( Store ) , VSpan . section ( Start , Width ) . values ( ) ) ; }


<s> int FromSize = getSizeOf ( FromSTy ) ; int ToSize = getSizeOf ( ToSTy ) ; assert ( FromSize % ToSize == <NUM_LIT> || ToSize % FromSize == <NUM_LIT> ) ; auto * MaskTy = cast < VectorType > ( Mask -> getType ( ) ) ; int FromCount = MaskTy -> getElementCount ( ) . getFixedValue ( ) ; int ToCount = ( FromCount * FromSize ) / ToSize ;

assert ( ( FromCount * FromSize ) % ToSize == <NUM_LIT> ) ;


<s> auto <STR_LIT> ( IRBuilder < > & Builder , Value * Mask , Type * FromTy , Type * ToTy ) const -> Value * { assert ( isa < VectorType > ( Mask -> getType ( ) ) ) ; Type * FromSTy = FromTy -> getScalarType ( ) ; Type * ToSTy = ToTy -> getScalarType ( ) ; if ( FromSTy == ToSTy ) return Mask ; int FromSize = getSizeOf ( FromSTy ) ; int ToSize = getSizeOf ( ToSTy ) ;

assert ( FromSize % ToSize == <NUM_LIT> || ToSize % FromSize == <NUM_LIT> ) ;


<s> if ( ! HST . useHVXOps ( ) ) return false ;

bool Changed = AlignVectors ( * this ) . run ( ) ;


<s> bool Changed = AlignVectors ( * this ) . run ( ) ; return Changed ;


<s> if ( skipFunction ( F ) ) return false ; AliasAnalysis & AA = getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; AssumptionCache & AC = getAnalysis < AssumptionCacheTracker > ( ) . getAssumptionCache ( F ) ;

DominatorTree & DT = getAnalysis < DominatorTreeWrapperPass > ( ) . getDomTree ( ) ;


<s> Segment ( Value * Val , int Begin , int Len ) : Val ( Val ) , Start ( Begin ) , Size ( Len ) {


<s> auto <STR_LIT> ( IRBuilder < > & Builder , Value * Val ) const -> Value * { Type * ScalarTy = Val -> getType ( ) -> getScalarType ( ) ; if ( ScalarTy == getByteTy ( ) ) return Val ;

if ( ScalarTy != getBoolTy ( ) ) return Builder . CreateBitCast ( Val , getByteTy ( getSizeOf ( Val ) ) ) ;


<s> Value * Pair = concat ( Builder , { Lo , Hi } ) ; Value * Shift = Builder . CreateLShr ( Builder . CreateShl ( Pair , Amt ) , <NUM_LIT> ) ; Value * Trunc = Builder . CreateTrunc ( Shift , Type :: getInt32Ty ( F . getContext ( ) ) ) ; return Builder . CreateBitCast ( Trunc , Hi -> getType ( ) ) ; } if ( VecLen == <NUM_LIT> ) { Value * Sub = Builder . CreateSub ( getConstInt ( VecLen ) , Amt ) ; return vralignb ( Builder , Lo , Hi , Sub ) ; }


<s> auto <STR_LIT> ( IRBuilder < > & Builder , Value * Val ) const -> Value * { Type * ScalarTy = Val -> getType ( ) -> getScalarType ( ) ; if ( ScalarTy == getBoolTy ( ) ) return Val ; Value * Bytes = vbytes ( Builder , Val ) ;

if ( auto * VecTy = dyn_cast < VectorType > ( Bytes -> getType ( ) ) ) return Builder . CreateTrunc ( Bytes , getBoolTy ( getSizeOf ( VecTy ) ) ) ;


<s> auto <STR_LIT> ( IRBuilder < > & Builder , Value * Val ) const -> Value * { Type * ScalarTy = Val -> getType ( ) -> getScalarType ( ) ;

if ( ScalarTy == getBoolTy ( ) ) return Val ;


<s> Value * Shift = Builder . CreateLShr ( Pair , Amt ) ; Value * Trunc = Builder . CreateTrunc ( Shift , Type :: getInt32Ty ( F . getContext ( ) ) ) ; return Builder . CreateBitCast ( Trunc , Lo -> getType ( ) ) ; } if ( VecLen == <NUM_LIT> ) { Type * Int64Ty = Type :: getInt64Ty ( F . getContext ( ) ) ; Value * Lo64 = Builder . CreateBitCast ( Lo , Int64Ty ) ; Value * Hi64 = Builder . CreateBitCast ( Hi , Int64Ty ) ; Function * FI = <STR_LIT> ( F . getParent ( ) , <STR_LIT> ) ; Value * Call = Builder . CreateCall ( FI , { Hi64 , Lo64 , Amt } ) ; return Builder . CreateBitCast ( Call , Lo -> getType ( ) ) ; } llvm_unreachable ( <STR_LIT> ) ;


<s> assert ( ValTy -> getElementType ( ) == Pad -> getType ( ) ) ; int CurSize = ValTy -> getElementCount ( ) . getFixedValue ( ) ; if ( CurSize == NewSize ) return Val ; if ( CurSize > NewSize ) return getElementRange ( Builder , Val , Val , <NUM_LIT> , NewSize ) ; SmallVector < int , <NUM_LIT> > SMask ( NewSize ) ;

std :: iota ( SMask . begin ( ) , SMask . begin ( ) + CurSize , <NUM_LIT> ) ;


<s> assert ( isa < VectorType > ( Val -> getType ( ) ) ) ; auto * ValTy = cast < VectorType > ( Val -> getType ( ) ) ; assert ( ValTy -> getElementType ( ) == Pad -> getType ( ) ) ; int CurSize = ValTy -> getElementCount ( ) . getFixedValue ( ) ; if ( CurSize == NewSize ) return Val ;

if ( CurSize > NewSize ) return getElementRange ( Builder , Val , Val , <NUM_LIT> , NewSize ) ;


<s> for ( Instruction * I : llvm :: reverse ( ToErase ) ) I -> eraseFromParent ( ) ;


<s> if ( II && ( II -> getIntrinsicID ( ) == <STR_LIT> || II -> getIntrinsicID ( ) == <STR_LIT> ) ) {

DEBUG ( dbgs ( ) << <STR_LIT> << * II << <STR_LIT> ) ;


<s> findLoopCarriedDeps ( ) ; findValueToReuse ( ) ; if ( ReuseCandidate . isDefined ( ) ) { reuseValue ( ) ; Changed = true ; Continue = true ; } llvm :: for_each ( Dependences , std :: default_delete < DepChain > ( ) ) ; } while ( Continue ) ;


<s> if ( User -> getParent ( ) != BB ) continue ; if ( ReplacedInsts . count ( User ) ) { DEBUG ( dbgs ( ) << * User << <STR_LIT> ) ; continue ; } if ( isa < PHINode > ( User ) ) continue ; if ( User -> mayHaveSideEffects ( ) ) continue ; if ( ! canReplace ( User ) ) continue ; PNUsers . push_back ( User ) ; } DEBUG ( dbgs ( ) << PNUsers . size ( ) << <STR_LIT> ) ; for ( Instruction * I : PNUsers ) { for ( auto UI = BEInst -> use_begin ( ) , E = BEInst -> use_end ( ) ; UI != E ; ++ UI ) { Use & U = * UI ; Instruction * BEUser = cast < Instruction > ( U . getUser ( ) ) ; if ( BEUser -> getParent ( ) != BB ) continue ; if ( ! isEquivalentOperation ( I , BEUser ) ) continue ; int NumOperands = I -> getNumOperands ( ) ; for ( int OpNo = <NUM_LIT> ; OpNo < NumOperands ; ++ OpNo ) {


<s> DEBUG ( dbgs ( ) << ReuseCandidate ) ; Instruction * Inst2Replace = ReuseCandidate . Inst2Replace ; Instruction * BEInst = ReuseCandidate . BackedgeInst ; int NumOperands = Inst2Replace -> getNumOperands ( ) ; std :: map < Instruction * , DepChain * > DepChains ; int Iterations = - <NUM_LIT> ; BasicBlock * LoopPH = CurLoop -> getLoopPreheader ( ) ; for ( int i = <NUM_LIT> ; i < NumOperands ; ++ i ) { Instruction * I = dyn_cast < Instruction > ( Inst2Replace -> getOperand ( i ) ) ; if ( ! I ) continue ; else { Instruction * J = cast < Instruction > ( BEInst -> getOperand ( i ) ) ; DepChain * D = getDepChainBtwn ( I , J ) ; assert ( D && <STR_LIT> ) ; if ( Iterations == - <NUM_LIT> ) Iterations = D -> iterations ( ) ; assert ( Iterations == D -> iterations ( ) && <STR_LIT> ) ; DepChains [ I ] = D ; } } DEBUG ( dbgs ( ) << <STR_LIT> ) ; SmallVector < Instruction * , <NUM_LIT> > InstsInPreheader ; for ( int i = <NUM_LIT> ; i < Iterations ; ++ i ) { Instruction * InstInPreheader = Inst2Replace -> clone ( ) ; SmallVector < Value * , <NUM_LIT> > Ops ; for ( int j = <NUM_LIT> ; j < NumOperands ; ++ j ) { Instruction * I = dyn_cast < Instruction > ( Inst2Replace -> getOperand ( j ) ) ; if ( ! I ) continue ; DepChain & D = * DepChains [ I ] ; Value * ValInPreheader = findValueInBlock ( D [ i ] , LoopPH ) ; InstInPreheader -> setOperand ( j , ValInPreheader ) ; } InstsInPreheader . push_back ( InstInPreheader ) ; InstInPreheader -> setName ( Inst2Replace -> getName ( ) + <STR_LIT> ) ;

InstInPreheader -> insertBefore ( LoopPH -> getTerminator ( ) ) ;


<s> continue ; } PHINode * PN = cast < PHINode > ( D -> front ( ) ) ; Instruction * BEInst = D -> back ( ) ; int Iters = D -> iterations ( ) ; BasicBlock * BB = PN -> getParent ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * PN << <STR_LIT> ) ; SmallVector < Instruction * , <NUM_LIT> > PNUsers ; for ( auto UI = PN -> use_begin ( ) , E = PN -> use_end ( ) ; UI != E ; ++ UI ) { Use & U = * UI ; Instruction * User = cast < Instruction > ( U . getUser ( ) ) ; if ( User -> getParent ( ) != BB ) continue ; if ( ReplacedInsts . count ( User ) ) { LLVM_DEBUG ( dbgs ( ) << * User << <STR_LIT> ) ; continue ; } if ( isa < PHINode > ( User ) ) continue ; if ( User -> mayHaveSideEffects ( ) ) continue ; if ( ! canReplace ( User ) ) continue ;


<s> void <ISA_LIT>VectorLoopCarriedReuse :: findValueToReuse ( ) { for ( auto * D : Dependences ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ( D -> front ( ) ) << <STR_LIT> ) ; if ( D -> iterations ( ) > <ISA_LIT>VLCRIterationLim ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ; } PHINode * PN = cast < PHINode > ( D -> front ( ) ) ; Instruction * BEInst = D -> back ( ) ; int Iters = D -> iterations ( ) ; BasicBlock * BB = PN -> getParent ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * PN << <STR_LIT> ) ; SmallVector < Instruction * , <NUM_LIT> > PNUsers ; for ( auto UI = PN -> use_begin ( ) , E = PN -> use_end ( ) ; UI != E ; ++ UI ) { Use & U = * UI ; Instruction * User = cast < Instruction > ( U . getUser ( ) ) ; if ( User -> getParent ( ) != BB ) continue ; if ( ReplacedInsts . count ( User ) ) { LLVM_DEBUG ( dbgs ( ) << * User << <STR_LIT> ) ; continue ; } if ( isa < PHINode > ( User ) ) continue ; if ( User -> mayHaveSideEffects ( ) ) continue ; if ( ! canReplace ( User ) ) continue ; PNUsers . push_back ( User ) ; } LLVM_DEBUG ( dbgs ( ) << PNUsers . size ( ) << <STR_LIT> ) ; for ( Instruction * I : PNUsers ) { for ( auto UI = BEInst -> use_begin ( ) , E = BEInst -> use_end ( ) ; UI != E ; ++ UI ) { Use & U = * UI ; Instruction * BEUser = cast < Instruction > ( U . getUser ( ) ) ; if ( BEUser -> getParent ( ) != BB ) continue ; if ( ! isEquivalentOperation ( I , BEUser ) ) continue ; int NumOperands = I -> getNumOperands ( ) ; for ( int OpNo = <NUM_LIT> ; OpNo < NumOperands ; ++ OpNo ) { Value * Op = I -> getOperand ( OpNo ) ; Value * BEOp = BEUser -> getOperand ( OpNo ) ; Instruction * OpInst = dyn_cast < Instruction > ( Op ) ; if ( ! OpInst ) { if ( Op == BEOp ) continue ; BEUser = nullptr ; break ; } Instruction * BEOpInst = dyn_cast < Instruction > ( BEOp ) ; if ( ! isDepChainBtwn ( OpInst , BEOpInst , Iters ) ) { BEUser = nullptr ; break ; } } if ( BEUser ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; ReuseCandidate . Inst2Replace = I ;

ReuseCandidate . BackedgeInst = BEUser ;


<s> const IntrinsicInst * II = dyn_cast < IntrinsicInst > ( I ) ; if ( II && ( II -> getIntrinsicID ( ) == <STR_LIT> || II -> getIntrinsicID ( ) == <STR_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * II << <STR_LIT> ) ; return false ; } return true ;


<s> PNUsers . push_back ( User ) ; } LLVM_DEBUG ( dbgs ( ) << PNUsers . size ( ) << <STR_LIT> ) ; for ( Instruction * I : PNUsers ) { for ( auto UI = BEInst -> use_begin ( ) , E = BEInst -> use_end ( ) ; UI != E ; ++ UI ) { Use & U = * UI ; Instruction * BEUser = cast < Instruction > ( U . getUser ( ) ) ; if ( BEUser -> getParent ( ) != BB ) continue ; if ( ! isEquivalentOperation ( I , BEUser ) ) continue ; int NumOperands = I -> getNumOperands ( ) ; for ( int OpNo = <NUM_LIT> ; OpNo < NumOperands ; ++ OpNo ) { Value * Op = I -> getOperand ( OpNo ) ; Instruction * OpInst = dyn_cast < Instruction > ( Op ) ; if ( ! OpInst ) continue ; Value * BEOp = BEUser -> getOperand ( OpNo ) ; Instruction * BEOpInst = dyn_cast < Instruction > ( BEOp ) ; if ( ! isDepChainBtwn ( OpInst , BEOpInst , Iters ) ) { BEUser = nullptr ; break ; } } if ( BEUser ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; ReuseCandidate . Inst2Replace = I ; ReuseCandidate . BackedgeInst = BEUser ; return ;


<s> for ( auto * D : Dependences ) { if ( D -> front ( ) == I1 && D -> back ( ) == I2 ) return D ; }


<s> bool <ISA_LIT>VectorLoopCarriedReuse :: isDepChainBtwn ( Instruction * I1 , Instruction * I2 , int Iters ) { for ( auto * D : Dependences ) {

if ( D -> front ( ) == I1 && D -> back ( ) == I2 && D -> iterations ( ) == Iters ) return true ;


<s> void reset ( ) { Inst2Replace = nullptr ; BackedgeInst = nullptr ;


<s> Instruction * Inst2Replace = ReuseCandidate . Inst2Replace ; Instruction * BEInst = ReuseCandidate . BackedgeInst ; int NumOperands = Inst2Replace -> getNumOperands ( ) ; std :: map < Instruction * , DepChain * > DepChains ; int Iterations = - <NUM_LIT> ; BasicBlock * LoopPH = CurLoop -> getLoopPreheader ( ) ; for ( int i = <NUM_LIT> ; i < NumOperands ; ++ i ) { Instruction * I = dyn_cast < Instruction > ( Inst2Replace -> getOperand ( i ) ) ; if ( ! I ) continue ; else { Instruction * J = cast < Instruction > ( BEInst -> getOperand ( i ) ) ; DepChain * D = getDepChainBtwn ( I , J ) ; assert ( D && <STR_LIT> ) ; if ( Iterations == - <NUM_LIT> ) Iterations = D -> iterations ( ) ; assert ( Iterations == D -> iterations ( ) && <STR_LIT> ) ; DepChains [ I ] = D ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; SmallVector < Instruction * , <NUM_LIT> > InstsInPreheader ; for ( int i = <NUM_LIT> ; i < Iterations ; ++ i ) { Instruction * InstInPreheader = Inst2Replace -> clone ( ) ; SmallVector < Value * , <NUM_LIT> > Ops ; for ( int j = <NUM_LIT> ; j < NumOperands ; ++ j ) { Instruction * I = dyn_cast < Instruction > ( Inst2Replace -> getOperand ( j ) ) ; if ( ! I ) continue ; DepChain & D = * DepChains [ I ] ; Value * ValInPreheader = findValueInBlock ( D [ i ] , LoopPH ) ;

InstInPreheader -> setOperand ( j , ValInPreheader ) ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override {


<s> AU . addPreservedID ( LCSSAID ) ; AU . setPreservesCFG ( ) ;


<s> <ISA_LIT>VectorLoopCarriedReusePass ( ) {


<s> if ( ! CurLoop -> getLoopPreheader ( ) ) return false ; if ( ! CurLoop -> getSubLoops ( ) . empty ( ) ) return false ; if ( CurLoop -> getNumBlocks ( ) != <NUM_LIT> ) return false ; return doVLCR ( ) ;


<s> if ( CurLoop -> getNumBlocks ( ) != <NUM_LIT> ) return false ;


<ISA_LIT>VectorLoopCarriedReuse Vlcr ( L ) ;


<s> <ISA_LIT>VectorLoopCarriedReuse Vlcr ( L ) ;


<s> DEBUG ( dbgs ( ) << <STR_LIT> << * CurLoop -> getHeader ( ) << <STR_LIT> ) ; do { Dependences . clear ( ) ; Continue = false ; findLoopCarriedDeps ( ) ; findValueToReuse ( ) ; if ( ReuseCandidate . isDefined ( ) ) { reuseValue ( ) ; Changed = true ; Continue = true ; } std :: for_each ( Dependences . begin ( ) , Dependences . end ( ) , std :: default_delete < DepChain > ( ) ) ; } while ( Continue ) ;


<s> void <ISA_LIT>VectorLoopCarriedReuse :: findLoopCarriedDeps ( ) { BasicBlock * BB = CurLoop -> getHeader ( ) ; for ( auto I = BB -> begin ( ) , E = BB -> end ( ) ; I != E && isa < PHINode > ( I ) ; ++ I ) { auto * PN = cast < PHINode > ( I ) ; if ( ! isa < VectorType > ( PN -> getType ( ) ) ) continue ; DepChain * D = new DepChain ( ) ; findDepChainFromPHI ( PN , * D ) ; if ( D -> size ( ) != <NUM_LIT> ) Dependences . insert ( D ) ; else delete D ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Dependences . size ( ) << <STR_LIT> ) ; LLVM_DEBUG ( for ( const DepChain * D : Dependences ) dbgs ( ) << * D << <STR_LIT> ; ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * II << <STR_LIT> ) ; return false ; default : return true ; }


<s> D . clear ( ) ; return ; } Value * BEVal = PN -> getIncomingValueForBlock ( BB ) ; Instruction * BEInst = dyn_cast < Instruction > ( BEVal ) ; assert ( BEInst && <STR_LIT> ) ; Value * PreHdrVal = PN -> getIncomingValueForBlock ( CurLoop -> getLoopPreheader ( ) ) ; if ( ! PreHdrVal || ! isa < Instruction > ( PreHdrVal ) ) { D . clear ( ) ; return ; } D . push_back ( PN ) ; findDepChainFromPHI ( BEInst , D ) ;


<s> Value * ValueInBlock = PN -> getIncomingValueForBlock ( BB ) ; return ValueInBlock ;


<s> Value * <ISA_LIT>VectorLoopCarriedReuse :: findValueInBlock ( Value * Op , BasicBlock * BB ) { PHINode * PN = dyn_cast < PHINode > ( Op ) ; assert ( PN ) ; Value * ValueInBlock = PN -> getIncomingValueForBlock ( BB ) ;


<s> int Iters = D -> iterations ( ) ; BasicBlock * BB = PN -> getParent ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * PN << <STR_LIT> ) ; SmallVector < Instruction * , <NUM_LIT> > PNUsers ; for ( auto UI = PN -> use_begin ( ) , E = PN -> use_end ( ) ; UI != E ; ++ UI ) { Use & U = * UI ; Instruction * User = cast < Instruction > ( U . getUser ( ) ) ; if ( User -> getParent ( ) != BB ) continue ; if ( ReplacedInsts . count ( User ) ) { LLVM_DEBUG ( dbgs ( ) << * User << <STR_LIT> ) ; continue ; } if ( isa < PHINode > ( User ) ) continue ; if ( User -> mayHaveSideEffects ( ) ) continue ; if ( ! canReplace ( User ) ) continue ; PNUsers . push_back ( User ) ; } LLVM_DEBUG ( dbgs ( ) << PNUsers . size ( ) << <STR_LIT> ) ; for ( Instruction * I : PNUsers ) { for ( auto UI = BEInst -> use_begin ( ) , E = BEInst -> use_end ( ) ; UI != E ; ++ UI ) { Use & U = * UI ; Instruction * BEUser = cast < Instruction > ( U . getUser ( ) ) ; if ( BEUser -> getParent ( ) != BB ) continue ; if ( ! isEquivalentOperation ( I , BEUser ) ) continue ; int NumOperands = I -> getNumOperands ( ) ; std :: map < Instruction * , DepChain * > DepChains ; CallInst * C1 = dyn_cast < CallInst > ( I ) ; if ( ( I && I -> isCommutative ( ) ) || ( C1 && isCallInstCommutative ( C1 ) ) ) { bool Found = false ; for ( int OpNo = <NUM_LIT> ; OpNo < NumOperands ; ++ OpNo ) { Value * Op = I -> getOperand ( OpNo ) ; Instruction * OpInst = dyn_cast < Instruction > ( Op ) ; Found = false ; for ( int T = <NUM_LIT> ; T < NumOperands ; ++ T ) { Value * BEOp = BEUser -> getOperand ( T ) ; Instruction * BEOpInst = dyn_cast < Instruction > ( BEOp ) ; if ( ! OpInst && ! BEOpInst ) {


<s> Instruction * front ( ) const { return Chain . front ( ) ;


AU . addRequired < LoopInfoWrapperPass > ( ) ;


<s> if ( D -> front ( ) == I1 && D -> back ( ) == I2 && D -> iterations ( ) == Iters ) return D ;


<s> PassRegistry * PR = PassRegistry :: getPassRegistry ( ) ;


<s> bool isDefined ( ) { return Inst2Replace != nullptr ;


<s> if ( CallInst * C2 = dyn_cast < CallInst > ( I2 ) ) { if ( C1 -> getCalledFunction ( ) != C2 -> getCalledFunction ( ) ) return false ; } } if ( I1 -> getType ( ) -> isVectorTy ( ) && I2 -> getType ( ) -> isVectorTy ( ) ) { unsigned NumOperands = I1 -> getNumOperands ( ) ; for ( unsigned i = <NUM_LIT> ; i < NumOperands ; ++ i ) { ConstantInt * C1 = dyn_cast < ConstantInt > ( I1 -> getOperand ( i ) ) ; ConstantInt * C2 = dyn_cast < ConstantInt > ( I2 -> getOperand ( i ) ) ; if ( ! C1 ) continue ; assert ( C2 ) ; if ( C1 -> getSExtValue ( ) != C2 -> getSExtValue ( ) ) return false ; } } return true ;


<s> if ( Other . size ( ) != size ( ) ) return false ; ChainOfDependences & OtherChain = Other . getChain ( ) ; for ( int i = <NUM_LIT> ; i < size ( ) ; ++ i ) { if ( Chain [ i ] != OtherChain [ i ] ) return false ; } return true ;


<s> void reset ( ) { Inst2Replace = nullptr ; BackedgeInst = nullptr ;


<s> BasicBlock * LoopPH = CurLoop -> getLoopPreheader ( ) ; assert ( ! DepChains . empty ( ) && <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; SmallVector < Instruction * , <NUM_LIT> > InstsInPreheader ; for ( int i = <NUM_LIT> ; i < Iterations ; ++ i ) { Instruction * InstInPreheader = Inst2Replace -> clone ( ) ; SmallVector < Value * , <NUM_LIT> > Ops ; for ( int j = <NUM_LIT> ; j < NumOperands ; ++ j ) { Instruction * I = dyn_cast < Instruction > ( Inst2Replace -> getOperand ( j ) ) ; if ( ! I ) continue ; DepChain & D = * DepChains [ I ] ; Value * ValInPreheader = findValueInBlock ( D [ i ] , LoopPH ) ; InstInPreheader -> setOperand ( j , ValInPreheader ) ; } InstsInPreheader . push_back ( InstInPreheader ) ; InstInPreheader -> setName ( Inst2Replace -> getName ( ) + <STR_LIT> ) ; InstInPreheader -> insertBefore ( LoopPH -> getTerminator ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * InstInPreheader << <STR_LIT> << LoopPH -> getName ( ) << <STR_LIT> ) ; } BasicBlock * BB = BEInst -> getParent ( ) ; IRBuilder < > IRB ( BB ) ; IRB . SetInsertPoint ( BB -> getFirstNonPHI ( ) ) ; Value * BEVal = BEInst ; PHINode * NewPhi ; for ( int i = Iterations - <NUM_LIT> ; i >= <NUM_LIT> ; -- i ) { Instruction * InstInPreheader = InstsInPreheader [ i ] ;

NewPhi = IRB . CreatePHI ( InstInPreheader -> getType ( ) , <NUM_LIT> ) ;


<s> if ( ! L -> getLoopPreheader ( ) ) return false ;

if ( ! L -> getSubLoops ( ) . empty ( ) ) return false ;


<s> bool <ISA_LIT>VectorLoopCarriedReuse :: runOnLoop ( Loop * L , LPPassManager & LPM ) { if ( skipLoop ( L ) ) return false ; if ( ! L -> getLoopPreheader ( ) ) return false ; if ( ! L -> getSubLoops ( ) . empty ( ) ) return false ; if ( L -> getNumBlocks ( ) != <NUM_LIT> ) return false ;