<s> } if ( GEP -> getNumOperands ( ) != <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return nullptr ; } Offsets = GEP -> getOperand ( <NUM_LIT> ) ; assert ( cast < FixedVectorType > ( Ty ) -> getNumElements ( ) == cast < FixedVectorType > ( Offsets -> getType ( ) ) -> getNumElements ( ) ) ; if ( Offsets -> getType ( ) -> getScalarSizeInBits ( ) != <NUM_LIT> ) return nullptr ; if ( ZExtInst * ZextOffs = dyn_cast < ZExtInst > ( Offsets ) ) Offsets = ZextOffs -> getOperand ( <NUM_LIT> ) ; else if ( ! ( cast < FixedVectorType > ( Offsets -> getType ( ) ) -> getNumElements ( ) == <NUM_LIT> && Offsets -> getType ( ) -> getScalarSizeInBits ( ) == <NUM_LIT> ) ) return nullptr ; if ( Ty != Offsets -> getType (


<s> if ( ZExtInst * ZextOffs = dyn_cast < ZExtInst > ( Offsets ) ) Offsets = ZextOffs -> getOperand ( <NUM_LIT> ) ; else if ( ! ( cast < FixedVectorType > ( Offsets -> getType ( ) ) -> getNumElements ( ) == <NUM_LIT> && Offsets -> getType ( ) -> getScalarSizeInBits ( ) == <NUM_LIT> ) ) return nullptr ; if ( Ty != Offsets -> getType ( ) ) { if ( ( Ty -> getScalarSizeInBits ( ) < Offsets -> getType ( ) -> getScalarSizeInBits ( ) ) ) { LLVM_DEBUG ( dbgs ( ) <<


<s> Value * PreTrunc = Trunc -> getOperand ( <NUM_LIT> ) ; Type * PreTruncTy = PreTrunc -> getType ( ) ; if ( PreTruncTy -> getPrimitiveSizeInBits ( ) == <NUM_LIT> ) { Input = PreTrunc ; InputTy = PreTruncTy ; } } if ( InputTy -> getPrimitiveSizeInBits ( ) != <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return nullptr ; } GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( Ptr ) ; Value * Offsets ; Value * BasePtr = checkGEP ( Offsets , InputTy , GEP , Builder ) ; if ( ! BasePtr ) return nullptr ; Value * Store = tryCreateIncrementingGatScat ( I , BasePtr , Offsets , GEP , Builder ) ; if ( Store ) return Store ; int Scale = computeScale ( BasePtr -> getType ( ) -> getPointerElementType ( ) -> getPrimitiveSizeInBits ( ) , MemoryTy -> getScalarSizeInBits ( ) ) ; if ( Scale == - <NUM_LIT> )


<s> IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ; Instruction * Root = I ; Value * Load = tryCreateMaskedGatherOffset ( I , Ptr , Root , Builder ) ; if ( ! Load ) Load = tryCreateMaskedGatherBase ( I , Ptr , Builder ) ; if ( ! Load ) return nullptr ; if ( ! isa < UndefValue > ( PassThru ) && ! match ( PassThru , m_Zero ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; Load = Builder . CreateSelect ( Mask , Load ,


<s> using namespace PatternMatch ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; auto * Ty = cast < FixedVectorType > ( I -> getType ( ) ) ; Value * Ptr = I -> getArgOperand ( <NUM_LIT> ) ; unsigned Alignment = cast < ConstantInt > ( I -> getArgOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; Value * PassThru = I -> getArgOperand ( <NUM_LIT> ) ; if ( ! isLegalTypeAndAlignment ( Ty -> getNumElements ( ) , Ty -> getScalarSizeInBits ( ) , Alignment ) ) return nullptr ; lookThroughBitcast ( Ptr ) ; assert ( Ptr -> getType ( ) -> isVectorTy ( ) && <STR_LIT> ) ; IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ; Instruction * Root = I ; Value * Load = tryCreateMaskedGatherOffset ( I , Ptr , Root , Builder ) ; if ( ! Load ) Load = tryCreateMaskedGatherBase ( I , Ptr , Builder ) ; if ( ! Load ) return nullptr ; if ( ! isa < UndefValue > ( PassThru ) && ! match ( PassThru , m_Zero ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; Load = Builder . CreateSelect ( Mask , Load , PassThru ) ; } Root -> replaceAllUsesWith (


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Value * Input = I -> getArgOperand ( <NUM_LIT> ) ; Value * Ptr = I -> getArgOperand ( <NUM_LIT> ) ; unsigned Alignment = cast < ConstantInt > ( I -> getArgOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; auto * Ty = cast < FixedVectorType > ( Input -> getType ( ) ) ; if ( ! isLegalTypeAndAlignment ( Ty -> getNumElements ( ) , Ty -> getScalarSizeInBits ( ) , Alignment ) ) return nullptr ; lookThroughBitcast ( Ptr ) ; assert ( Ptr -> getType ( ) -> isVectorTy ( ) && <STR_LIT> ) ; IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ; Value * Store = tryCreateMaskedScatterOffset ( I , Ptr , Builder ) ; if ( ! Store ) Store = tryCreateMaskedScatterBase ( I , Ptr , Builder ) ; if ( ! Store ) return nullptr ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; I -> eraseFromParent


<s> auto & TM = TPC . getTM < TargetMachine > ( ) ; auto * ST = & TM . getSubtarget < <ISA_LIT>Subtarget > ( F ) ; if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; SmallVector < IntrinsicInst * , <NUM_LIT> > Gathers ; SmallVector < IntrinsicInst * , <NUM_LIT> > Scatters ; for ( BasicBlock & BB : F ) { for ( Instruction & I : BB ) { IntrinsicInst * II = dyn_cast < IntrinsicInst > ( & I ) ; if ( II && II -> getIntrinsicID ( ) == <STR_LIT> ) { Gathers . push_back ( II ) ; if ( isa < GetElementPtrInst > ( II -> getArgOperand ( <NUM_LIT> ) ) ) optimiseOffsets ( cast < Instruction > ( II -> getArgOperand ( <NUM_LIT> ) ) -> getOperand ( <NUM_LIT> ) , II -> getParent ( ) , LI ) ; } else if ( II && II -> getIntrinsicID ( ) == <STR_LIT> ) { Scatters . push_back ( II ) ; if ( isa < GetElementPtrInst > ( II -> getArgOperand ( <NUM_LIT> ) ) ) optimiseOffsets ( cast < Instruction > ( II -> getArgOperand ( <NUM_LIT> ) ) -> getOperand ( <NUM_LIT> ) , II -> getParent ( ) , LI ) ; } } } bool Changed = false ; for ( unsigned i = <NUM_LIT> ; i < Gathers . size ( ) ; i ++ ) { IntrinsicInst * I = Gathers [ i ] ; Value * L = lowerGather ( I ) ; if ( L == nullptr ) continue ; SimplifyInstructionsInBlock ( cast < Instruction > ( L ) -> getParent ( ) ) ; Changed = true ; } for ( unsigned i = <NUM_LIT> ; i < Scatters . size ( ) ; i ++ ) { IntrinsicInst * I = Scatters [ i ] ; Value * S = lowerScatter ( I ) ; if ( S == nullptr ) continue ; SimplifyInstructionsInBlock ( cast < Instruction > ( S ) -> getParent ( ) ) ; Changed = true ; } return


<s> Value * MVEGatherScatterLowering :: tryCreateMaskedScatterBase ( IntrinsicInst * I , Value * Ptr , IRBuilder < > & Builder , int64_t Increment ) { using namespace PatternMatch ; Value * Input = I -> getArgOperand ( <NUM_LIT> ) ; auto * Ty = cast < VectorType > ( Input -> getType ( ) ) ; if ( ! ( Ty -> getNumElements ( ) == <NUM_LIT> && Ty -> getScalarSizeInBits ( ) == <NUM_LIT> ) ) { return nullptr


<s> Value * MVEGatherScatterLowering :: tryCreateMaskedScatterBase ( IntrinsicInst * I , Value * Ptr , IRBuilder < > & Builder , int64_t Increment ) { using namespace PatternMatch ; Value * Input = I -> getArgOperand ( <NUM_LIT> ) ; auto * Ty = cast < VectorType > ( Input -> getType ( ) ) ; if ( ! ( Ty -> getNumElements ( ) == <NUM_LIT> && Ty -> getScalarSizeInBits ( ) == <NUM_LIT> ) ) { return nullptr ; } Value * Mask = I -> getArgOperand (


<s> static Value * CheckAndCreateOffsetAdd ( Value * X , Value * Y , Value * GEP , IRBuilder < > & Builder ) { auto FixSummands = [ & Builder ] ( FixedVectorType * & VT , Value * & NonVectorVal ) { ConstantInt * Const ; if ( ( Const = dyn_cast < ConstantInt > ( NonVectorVal ) ) && VT -> getElementType ( ) != NonVectorVal -> getType ( ) ) { unsigned TargetElemSize = VT -> getElementType ( ) -> getPrimitiveSizeInBits ( ) ; uint64_t N = Const -> getZExtValue ( ) ; if ( N < ( unsigned ) ( <NUM_LIT> << ( TargetElemSize - <NUM_LIT> ) ) ) { NonVectorVal = Builder . CreateVectorSplat ( VT -> getNumElements ( ) , Builder . getIntN ( TargetElemSize , N ) ) ; return ; } } NonVectorVal = Builder . CreateVectorSplat ( VT -> getNumElements ( ) , NonVectorVal ) ; } ; FixedVectorType * XElType = dyn_cast < FixedVectorType > ( X -> getType ( ) ) ; FixedVectorType * YElType = dyn_cast < FixedVectorType > ( Y -> getType ( ) ) ; if ( XElType && ! YElType ) { FixSummands ( XElType , Y ) ; YElType = cast < FixedVectorType > ( Y -> getType ( ) ) ; } else if ( YElType && ! XElType ) { FixSummands ( YElType , X ) ; XElType = cast < FixedVectorType > ( X -> getType ( ) ) ; } if ( XElType != YElType ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return nullptr ; } if ( XElType -> getElementType ( ) -> getScalarSizeInBits ( ) != <NUM_LIT> ) { Constant * ConstX = dyn_cast < Constant > ( X ) ; Constant * ConstY = dyn_cast < Constant > ( Y ) ; if ( ! ConstX || ! ConstY ) return nullptr ; unsigned TargetElemSize = <NUM_LIT> / XElType -> getNumElements ( ) ; for ( unsigned i = <NUM_LIT> ; i <

XElType -> getNumElements ( ) ; i ++ ) {


<s> Type * Ty = I -> getType ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( Ty -> getVectorNumElements ( ) != <NUM_LIT> || Ty -> getScalarSizeInBits ( ) != <NUM_LIT> ) return nullptr ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ty , Ptr -> getType ( ) } , { Ptr , Builder . getInt32 ( <NUM_LIT> ) } ) ; else return Builder . CreateIntrinsic ( <STR_LIT> , { Ty , Ptr -> getType ( ) , Mask -> getType ( ) } , { Ptr , Builder .


<s> Type * OffsType = VectorType :: getInteger ( cast < VectorType > ( Ty ) ) ; if ( OffsType != Offsets -> getType ( ) ) { if ( OffsType -> getScalarSizeInBits ( ) > Offsets -> getType ( ) -> getScalarSizeInBits ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Offsets = Builder . CreateZExt ( Offsets , OffsType , <STR_LIT> ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <<


<s> GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( Ptr ) ; if ( ! GEP ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return nullptr ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; Value * GEPPtr = GEP -> getPointerOperand ( ) ; if ( GEPPtr -> getType ( ) -> isVectorTy ( ) ) { return nullptr ; } if ( GEP -> getNumOperands ( ) != <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return nullptr ; } Offsets = GEP -> getOperand ( <NUM_LIT> ) ; assert ( cast < VectorType > ( Ty ) -> getNumElements ( ) == cast < VectorType > ( Offsets -> getType ( ) ) -> getNumElements ( ) ) ; if ( Offsets -> getType ( ) -> getScalarSizeInBits ( ) != <NUM_LIT> ) return nullptr ; if ( ZExtInst * ZextOffs = dyn_cast < ZExtInst > ( Offsets ) ) Offsets = ZextOffs -> getOperand ( <NUM_LIT> ) ; else if ( ! ( cast < VectorType > ( Offsets -> getType ( ) ) -> getNumElements ( ) == <NUM_LIT> && Offsets -> getType ( ) -> getScalarSizeInBits ( ) == <NUM_LIT> ) ) return nullptr ; if ( Ty != Offsets -> getType ( ) ) { if ( ( Ty -> getScalarSizeInBits ( ) < Offsets -> getType ( ) -> getScalarSizeInBits ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT>


<s> auto * BCTy = cast < VectorType > ( BitCast -> getType ( ) ) ; auto * BCSrcTy = cast < VectorType > ( BitCast -> getOperand ( <NUM_LIT> ) -> getType ( ) ) ; if ( BCTy -> getNumElements ( ) == BCSrcTy -> getNumElements ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Ptr = BitCast -> getOperand (


<s> if ( auto * BitCast = dyn_cast < BitCastInst > ( Ptr ) ) { auto * BCTy = cast < VectorType > ( BitCast -> getType ( ) ) ; auto * BCSrcTy = cast < VectorType > ( BitCast -> getOperand ( <NUM_LIT> ) -> getType ( ) ) ; if ( BCTy -> getNumElements ( ) == BCSrcTy -> getNumElements ( ) ) { LLVM_DEBUG ( dbgs ( ) <<


<s> Value * Ptr = I -> getArgOperand ( <NUM_LIT> ) ; unsigned Alignment = cast < ConstantInt > ( I -> getArgOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; auto * Ty = cast < VectorType > ( Input -> getType ( ) ) ; if ( ! isLegalTypeAndAlignment ( Ty -> getNumElements ( ) , Ty -> getScalarSizeInBits ( ) , Alignment ) ) return nullptr ; lookThroughBitcast ( Ptr ) ; assert ( Ptr -> getType ( ) -> isVectorTy ( ) && <STR_LIT> ) ; IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ; Value * Store = tryCreateMaskedScatterOffset ( I ,


<s> Value * MVEGatherScatterLowering :: tryCreateMaskedGatherBase ( IntrinsicInst * I , Value * Ptr , IRBuilder < > & Builder ) { using namespace PatternMatch ; auto * Ty = cast < VectorType > ( I -> getType ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( Ty -> getNumElements ( ) != <NUM_LIT> || Ty -> getScalarSizeInBits (


<s> if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ty , Ptr -> getType ( ) } , { Ptr , Builder . getInt32 ( <NUM_LIT> ) } ) ; else return Builder . CreateIntrinsic ( <STR_LIT> , { Ty , Ptr -> getType ( ) , Mask -> getType ( ) } , { Ptr

, Builder . getInt32 ( <NUM_LIT> ) , Mask } ) ;


<s> if ( ! ( Ty -> getNumElements ( ) == <NUM_LIT> && Ty -> getScalarSizeInBits ( ) == <NUM_LIT> ) ) { return nullptr ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ptr -> getType ( ) , Input -> getType ( ) } , { Ptr , Builder . getInt32 ( <NUM_LIT> ) , Input } ) ; else return Builder . CreateIntrinsic ( <STR_LIT> , { Ptr -> getType ( ) , Input -> getType ( ) , Mask -> getType ( ) } , { Ptr , Builder . getInt32 ( <NUM_LIT>


<s> unsigned OffsetsElemCount = cast < FixedVectorType > ( Offsets -> getType ( ) ) -> getNumElements ( ) ; assert ( Ty -> getNumElements ( ) == OffsetsElemCount ) ; ZExtInst * ZextOffs = dyn_cast < ZExtInst > ( Offsets ) ; if ( ZextOffs ) Offsets = ZextOffs -> getOperand ( <NUM_LIT> ) ; FixedVectorType * OffsetType = cast < FixedVectorType > ( Offsets -> getType ( ) ) ; if ( ! ZextOffs || cast < FixedVectorType > ( ZextOffs -> getDestTy ( ) ) -> getElementType ( ) -> getScalarSizeInBits

( ) != <NUM_LIT> ) if ( ! checkOffsetSize ( Offsets , OffsetsElemCount ) ) return nullptr ;


<s> Offsets = GEP -> getOperand ( <NUM_LIT> ) ; unsigned OffsetsElemCount = cast < FixedVectorType > ( Offsets -> getType ( ) ) -> getNumElements ( ) ; assert ( Ty -> getNumElements ( ) == OffsetsElemCount ) ; ZExtInst * ZextOffs = dyn_cast < ZExtInst > ( Offsets ) ; if ( ZextOffs ) Offsets = ZextOffs -> getOperand ( <NUM_LIT> ) ; FixedVectorType * OffsetType = cast < FixedVectorType > ( Offsets -> getType ( ) ) ; if ( ! ZextOffs || cast < FixedVectorType > ( ZextOffs -> getDestTy ( ) ) -> getElementType ( ) -> getScalarSizeInBits ( ) != <NUM_LIT> ) if ( ! checkOffsetSize ( Offsets , OffsetsElemCount ) ) return nullptr ; if ( Ty != Offsets -> getType ( ) ) { if ( ( Ty -> getElementType ( ) -> getScalarSizeInBits ( ) < OffsetType -> getElementType ( ) -> getScalarSizeInBits ( ) ) ) { Offsets = Builder . CreateTrunc (


<s> return false ; } bool Gatscat = true ; for ( User * U : I -> users ( ) ) { if ( ! isa < Instruction > ( U ) ) return false ; if ( isa < GetElementPtrInst > ( U ) || isGatherScatter ( dyn_cast < IntrinsicInst > ( U ) ) ) { return Gatscat ; } else { unsigned OpCode = cast < Instruction > ( U )


<s> void MVEGatherScatterLowering :: lookThroughBitcast ( Value * & Ptr ) { if ( auto * BitCast = dyn_cast < BitCastInst > ( Ptr ) ) { auto * BCTy = cast < FixedVectorType > ( BitCast -> getType ( ) ) ; auto * BCSrcTy = cast < FixedVectorType > ( BitCast -> getOperand ( <NUM_LIT> ) -> getType ( ) ) ; if ( BCTy -> getNumElements ( ) == BCSrcTy -> getNumElements ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Ptr = BitCast -> getOperand (


<s> if ( ! isLegalTypeAndAlignment ( Ty -> getNumElements ( ) , Ty -> getScalarSizeInBits ( ) , Alignment ) ) return nullptr ; lookThroughBitcast ( Ptr ) ; assert ( Ptr -> getType ( ) -> isVectorTy ( ) && <STR_LIT> ) ; IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ; Instruction * Root = I ; Value * Load = tryCreateMaskedGatherOffset ( I , Ptr , Root , Builder ) ; if ( ! Load ) Load = tryCreateMaskedGatherBase ( I , Ptr , Builder ) ; if ( ! Load ) return nullptr ; if ( ! isa < UndefValue > ( PassThru ) && ! match ( PassThru , m_Zero ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; Load = Builder . CreateSelect ( Mask , Load , PassThru ) ; } Root -> replaceAllUsesWith ( Load ) ; Root -> eraseFromParent (


<s> Align Alignment = cast < ConstantInt > ( I -> getArgOperand ( <NUM_LIT> ) ) -> getAlignValue ( ) ; auto * Ty = cast < FixedVectorType > ( Input -> getType ( ) ) ; if ( ! isLegalTypeAndAlignment ( Ty -> getNumElements ( ) , Ty -> getScalarSizeInBits ( ) , Alignment ) ) return nullptr ; lookThroughBitcast ( Ptr ) ; assert ( Ptr -> getType ( ) -> isVectorTy ( ) && <STR_LIT> ) ; IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ; Value * Store = tryCreateMaskedScatterOffset ( I , Ptr , Builder


<s> if ( ! isLegalTypeAndAlignment ( Ty -> getNumElements ( ) , Ty -> getScalarSizeInBits ( ) , Alignment ) ) return nullptr ; lookThroughBitcast ( Ptr ) ; assert ( Ptr -> getType ( ) -> isVectorTy ( ) && <STR_LIT> ) ; IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ; Value * Store = tryCreateMaskedScatterOffset ( I , Ptr , Builder ) ; if ( ! Store ) Store = tryCreateMaskedScatterBase ( I , Ptr ,


<s> void MVEGatherScatterLowering :: pushOutMul ( PHINode * & Phi , Value * IncrementPerRound , Value * OffsSecondOperand , unsigned LoopIncrement , IRBuilder < > & Builder ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Instruction * InsertionPoint = & cast < Instruction > ( Phi -> getIncomingBlock ( LoopIncrement == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ) -> back ( ) ) ; Value * StartIndex = BinaryOperator :: Create ( Instruction :: Mul , Phi -> getIncomingValue ( LoopIncrement == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ) , OffsSecondOperand , <STR_LIT> , InsertionPoint ) ; Instruction * Product = BinaryOperator :: Create ( Instruction :: Mul , IncrementPerRound , OffsSecondOperand ,


<s> if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; SmallVector < IntrinsicInst * , <NUM_LIT> > Gathers ; SmallVector < IntrinsicInst * , <NUM_LIT> > Scatters ; bool Changed = false ; for ( BasicBlock & BB : F ) { for ( Instruction & I : BB ) { IntrinsicInst * II = dyn_cast < IntrinsicInst > ( & I ) ; if ( II && II -> getIntrinsicID ( ) == <STR_LIT> && isa < FixedVectorType > ( II -> getType ( ) ) ) { Gathers . push_back ( II ) ; Changed |= optimiseAddress ( II -> getArgOperand ( <NUM_LIT> ) ,


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; DataLayout DT = I -> getParent ( ) -> getParent ( ) -> getParent ( ) -> getDataLayout ( ) ; int TypeScale = computeScale ( DT . getTypeSizeInBits ( GEP -> getOperand ( <NUM_LIT> ) -> getType ( ) ) , DT . getTypeSizeInBits ( GEP -> getType ( ) ) / cast < FixedVectorType > ( GEP -> getType ( ) ) -> getNumElements ( ) ) ; if ( TypeScale == - <NUM_LIT> ) return nullptr ; if ( GEP -> hasOneUse ( ) ) { Value * Load = tryCreateIncrementingWBGatScat ( I , BasePtr , Offsets , TypeScale , Builder ) ; if ( Load != nullptr ) return Load ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; std :: pair < Value * , int64_t > Add = getVarAndConst ( Offsets , TypeScale ) ; if ( Add . first == nullptr ) return nullptr ; Value * OffsetsIncoming = Add . first ; int64_t Immediate = Add . second ; Instruction * ScaledOffsets = BinaryOperator :: Create ( Instruction :: Shl , OffsetsIncoming , Builder . CreateVectorSplat ( Ty -> getNumElements ( ) , Builder . getInt32 ( TypeScale ) ) , <STR_LIT> , I ) ; OffsetsIncoming = BinaryOperator :: Create ( Instruction :: Add , ScaledOffsets , Builder . CreateVectorSplat ( Ty -> getNumElements ( ) , Builder . CreatePtrToInt ( BasePtr , cast < VectorType > ( ScaledOffsets -> getType ( ) ) -> getElementType ( ) ) ) , <STR_LIT> , I ) ; if ( I -> getIntrinsicID ( ) == <STR_LIT> ) return cast < IntrinsicInst > ( tryCreateMaskedGatherBase ( I , OffsetsIncoming , Builder , Immediate ) ) ; else return cast < IntrinsicInst > ( tryCreateMaskedScatterBase ( I


<s> if ( Add . first == nullptr ) return nullptr ; Value * OffsetsIncoming = Add . first ; int64_t Immediate = Add . second ; if ( OffsetsIncoming != Phi ) return nullptr ; Builder . SetInsertPoint ( & Phi -> getIncomingBlock ( <NUM_LIT> - IncrementIndex ) -> back ( ) ) ; unsigned NumElems = cast < FixedVectorType > ( OffsetsIncoming -> getType ( ) ) -> getNumElements ( ) ; Instruction * ScaledOffsets = BinaryOperator :: Create ( Instruction :: Shl , Phi -> getIncomingValue ( <NUM_LIT> - IncrementIndex ) , Builder . CreateVectorSplat ( NumElems , Builder . getInt32 ( TypeScale ) ) , <STR_LIT> , & Phi -> getIncomingBlock ( <NUM_LIT> - IncrementIndex ) -> back ( ) ) ; OffsetsIncoming = BinaryOperator :: Create ( Instruction :: Add , ScaledOffsets , Builder . CreateVectorSplat ( NumElems , Builder . CreatePtrToInt ( BasePtr , cast < VectorType > ( ScaledOffsets -> getType ( ) ) -> getElementType ( ) ) ) , <STR_LIT> , & Phi -> getIncomingBlock ( <NUM_LIT> - IncrementIndex ) -> back ( ) ) ; OffsetsIncoming = BinaryOperator :: Create ( Instruction :: Sub , OffsetsIncoming , Builder . CreateVectorSplat ( NumElems , Builder . getInt32 ( Immediate ) ) , <STR_LIT> , & Phi -> getIncomingBlock ( <NUM_LIT> - IncrementIndex ) -> back ( ) ) ; Phi -> setIncomingValue ( <NUM_LIT> - IncrementIndex , OffsetsIncoming ) ; Builder . SetInsertPoint ( I ) ; Value * EndResult ; Value * NewInduction ; if ( I -> getIntrinsicID ( ) == <STR_LIT> ) { Value * Load = tryCreateMaskedGatherBaseWB ( I , Phi , Builder , Immediate ) ; EndResult = Builder . CreateExtractValue ( Load , <NUM_LIT> , <STR_LIT> ) ; NewInduction = Builder . CreateExtractValue ( Load , <NUM_LIT> , <STR_LIT> ) ; } else { NewInduction = tryCreateMaskedScatterBaseWB ( I , Phi , Builder , Immediate ) ; EndResult = NewInduction ; } Instruction * AddInst = cast < Instruction > ( Offsets ) ; AddInst -> replaceAllUsesWith ( NewInduction ) ; AddInst -> eraseFromParent


<s> if ( Phi == nullptr || Phi -> getNumIncomingValues ( ) != <NUM_LIT> || Phi -> getParent ( ) != L -> getHeader ( ) || Phi -> getNumUses ( ) != <NUM_LIT> ) return nullptr ; unsigned IncrementIndex = Phi -> getIncomingBlock ( <NUM_LIT> ) == L -> getLoopLatch ( ) ? <NUM_LIT> : <NUM_LIT> ; Offsets = Phi -> getIncomingValue ( IncrementIndex ) ; std :: pair < Value * , int64_t > Add = getVarAndConst ( Offsets , TypeScale ) ; if ( Add . first == nullptr ) return nullptr ; Value * OffsetsIncoming = Add . first ; int64_t Immediate = Add . second ; if ( OffsetsIncoming != Phi ) return nullptr ; Builder . SetInsertPoint ( & Phi -> getIncomingBlock ( <NUM_LIT> - IncrementIndex ) -> back ( ) ) ; unsigned NumElems = cast < FixedVectorType > ( OffsetsIncoming -> getType ( ) ) -> getNumElements ( ) ; Instruction * ScaledOffsets = BinaryOperator :: Create ( Instruction :: Shl , Phi -> getIncomingValue ( <NUM_LIT> - IncrementIndex ) , Builder . CreateVectorSplat ( NumElems , Builder . getInt32 ( TypeScale ) ) , <STR_LIT> , & Phi -> getIncomingBlock ( <NUM_LIT> - IncrementIndex ) -> back ( ) ) ; OffsetsIncoming = BinaryOperator :: Create ( Instruction :: Add , ScaledOffsets , Builder . CreateVectorSplat ( NumElems , Builder . CreatePtrToInt ( BasePtr , cast < VectorType > ( ScaledOffsets -> getType ( ) ) -> getElementType ( ) ) ) , <STR_LIT> , & Phi -> getIncomingBlock ( <NUM_LIT> - IncrementIndex ) -> back ( ) ) ; OffsetsIncoming = BinaryOperator :: Create ( Instruction :: Sub , OffsetsIncoming , Builder . CreateVectorSplat ( NumElems , Builder . getInt32 ( Immediate ) ) , <STR_LIT> , & Phi -> getIncomingBlock ( <NUM_LIT> - IncrementIndex ) -> back ( ) ) ; Phi -> setIncomingValue ( <NUM_LIT> - IncrementIndex , OffsetsIncoming ) ; Builder . SetInsertPoint ( I ) ; Value * EndResult ; Value * NewInduction ; if ( I -> getIntrinsicID ( ) == <STR_LIT> ) { Value * Load = tryCreateMaskedGatherBaseWB ( I , Phi , Builder , Immediate ) ; EndResult = Builder . CreateExtractValue ( Load , <NUM_LIT> , <STR_LIT> ) ; NewInduction = Builder . CreateExtractValue ( Load , <NUM_LIT> , <STR_LIT> ) ; } else { NewInduction = tryCreateMaskedScatterBaseWB ( I , Phi , Builder , Immediate ) ; EndResult =


<s> if ( Ty -> getNumElements ( ) != <NUM_LIT> || Ty -> getScalarSizeInBits ( ) != <NUM_LIT> ) return nullptr ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ty , Ptr -> getType ( ) } ,

{ Ptr , Builder . getInt32 ( Increment ) } ) ;


<s> Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ty , Ptr -> getType ( ) } , { Ptr , Builder . getInt32 ( Increment ) } ) ; else return Builder . CreateIntrinsic ( <STR_LIT> , { Ty , Ptr -> getType ( ) , Mask -> getType ( ) } , { Ptr

, Builder . getInt32 ( Increment ) , Mask } ) ;


<s> Value * MVEGatherScatterLowering :: tryCreateMaskedGatherBaseWB ( IntrinsicInst * I , Value * Ptr , IRBuilder < > & Builder , int64_t Increment ) { using namespace PatternMatch ; auto * Ty = cast < FixedVectorType > ( I -> getType ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( Ty -> getNumElements ( ) != <NUM_LIT> || Ty

-> getScalarSizeInBits ( ) != <NUM_LIT> ) return nullptr ;


<s> unsigned Unsigned = <NUM_LIT> ; auto * Extend = Root ; if ( OriginalTy -> getPrimitiveSizeInBits ( ) < <NUM_LIT> ) { if ( ! I -> hasOneUse ( ) ) return nullptr ; Extend = cast < Instruction > ( * I -> users ( ) . begin ( ) ) ; if ( isa < SExtInst > ( Extend ) ) { Unsigned = <NUM_LIT> ; } else if ( ! isa < ZExtInst > ( Extend ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return nullptr ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; ResultTy = Extend -> getType ( ) ; if ( ResultTy -> getPrimitiveSizeInBits ( ) != <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return nullptr ; } } GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( Ptr ) ; Value * Offsets ; Value * BasePtr = checkGEP ( Offsets , cast < FixedVectorType > ( ResultTy ) , GEP , Builder ) ; if ( ! BasePtr ) return nullptr ; Value * Load = tryCreateIncrementingGatScat ( I , BasePtr , Offsets , GEP , Builder ) ; if ( Load ) return Load ; int Scale = computeScale ( BasePtr -> getType ( ) -> getPointerElementType ( ) -> getPrimitiveSizeInBits ( ) , OriginalTy -> getScalarSizeInBits ( ) ) ; if ( Scale == - <NUM_LIT> ) return nullptr ; Root = Extend


<s> } Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ptr -> getType ( ) , Input -> getType ( ) } , { Ptr , Builder . getInt32 ( Increment ) , Input } ) ; else return Builder . CreateIntrinsic ( <STR_LIT> , { Ptr -> getType ( ) , Input -> getType ( ) , Mask ->

getType ( ) } , { Ptr , Builder . getInt32 ( Increment ) , Input , Mask } ) ;


<s> if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ptr -> getType ( ) , Input -> getType ( ) } , { Ptr , Builder . getInt32 ( Increment ) , Input } ) ; else return Builder . CreateIntrinsic ( <STR_LIT> , { Ptr -> getType ( ) , Input -> getType ( ) , Mask -> getType ( ) } , { Ptr , Builder . getInt32 ( Increment ) ,


<s> auto * Ty = cast < FixedVectorType > ( Input -> getType ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( Ty -> getNumElements ( ) != <NUM_LIT> || Ty -> getScalarSizeInBits ( ) != <NUM_LIT> ) return nullptr ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ptr -> getType ( ) , Input -> getType ( ) } , { Ptr , Builder . getInt32 ( Increment ) , Input } ) ; else return Builder . CreateIntrinsic ( <STR_LIT> , { Ptr -> getType ( ) , Input -> getType ( ) , Mask -> getType (

) } , { Ptr , Builder . getInt32 ( Increment ) , Input , Mask } ) ;


<s> Pass * llvm :: createMVELaneInterleavingPass ( ) { return new MVELaneInterleaving (


<s> StringRef getPassName ( ) const override { return


<s> } } for ( auto * T : Truncs ) { if ( T -> hasOneUse ( ) && ! isa < StoreInst > ( * T -> user_begin ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * T << <STR_LIT> ) ; return true ; } } for ( auto * E : Exts ) { if ( ! E -> hasOneUse ( ) || cast < Instruction > ( * E -> user_begin ( ) ) -> getOpcode ( ) != Instruction :: Mul ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * E << <STR_LIT> ) ; return


<s> initializeMVELaneInterleavingPass ( * PassRegistry :: getPassRegistry ( )


<s> initializeMVELaneInterleavingPass ( * PassRegistry ::


<s> case <STR_LIT> : if ( ST -> hasMVEFloatOps ( ) ) continue ; break ; default : break ; } if ( IsMasked ( & I ) ) { auto * VecTy = getVectorType ( Int ) ; unsigned Lanes = VecTy -> getNumElements ( ) ; unsigned ElementWidth = VecTy -> getScalarSizeInBits ( ) ; unsigned MaxWidth = TTI -> getRegisterBitWidth ( true ) ; if ( Lanes * ElementWidth > MaxWidth || Lanes == MaxWidth ) return false ; MaskedInsts . push_back ( cast < IntrinsicInst > ( & I ) ) ; continue ; } for ( const Use & U : Int -> args ( ) ) { if ( isa < VectorType > ( U -> getType ( ) ) ) return false ; } } } if ( ! ActiveLaneMask ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } return ! MaskedInsts .


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } } unsigned SizeInBits = TripCount -> getType ( ) -> getScalarSizeInBits ( ) ; auto MaxMinusVW = APInt ( SizeInBits , ~ <NUM_LIT> ) - APInt ( SizeInBits , VectorWidth ) ; APInt UpperboundTC = SE -> getUnsignedRangeMax ( TC ) ; if ( UpperboundTC . ugt ( MaxMinusVW ) && ! ForceTailPredication ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; dbgs ( ) << UpperboundTC << <STR_LIT> << MaxMinusVW << <STR_LIT> ; ) ; return false ; } auto * ECPlusVWMinus1 = SE -> getAddExpr ( EC , SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth - <NUM_LIT> ) ) ) ; auto * Ceil = SE -> getUDivExpr ( ECPlusVWMinus1 , SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth ) ) ) ; ConstantRange RangeCeil = SE -> getUnsignedRange ( Ceil ) ; ConstantRange RangeTC = SE -> getUnsignedRange ( TC ) ; if ( ! RangeTC . isSingleElement ( ) ) { auto ZeroRange = ConstantRange ( APInt ( TripCount -> getType ( ) -> getScalarSizeInBits ( ) , <NUM_LIT> ) ) ; RangeTC = RangeTC . unionWith ( ZeroRange ) ; } if ( ! RangeTC . contains ( RangeCeil ) && ! ForceTailPredication ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto * IV = ActiveLaneMask -> getOperand ( <NUM_LIT> ) ; auto * IVExpr = SE -> getSCEV ( IV ) ; auto * AddExpr = dyn_cast < SCEVAddRecExpr > ( IVExpr ) ; if ( ! AddExpr ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; IVExpr -> dump ( ) ) ; return false ; } if ( AddExpr -> getLoop ( ) != L ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto * Step = dyn_cast < SCEVConstant > ( AddExpr -> getOperand ( <NUM_LIT> ) ) ; if ( ! Step ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; AddExpr -> getOperand ( <NUM_LIT> ) -> dump ( ) ) ; return false ; } auto StepValue = Step -> getValue ( ) -> getSExtValue


<s> } SmallPtrSet < Instruction * , <NUM_LIT> > Dead ; while ( ! MaybeDead . empty ( ) ) { auto * I = MaybeDead . front ( ) ; MaybeDead . remove ( I ) ; if ( I -> hasNUsesOrMore ( <NUM_LIT> ) ) continue ; for ( auto & U : I -> operands ( ) ) if ( auto * OpI = dyn_cast < Instruction > ( U ) ) MaybeDead . insert ( OpI ) ; I -> dropAllReferences ( ) ; Dead . insert ( I ) ; } for ( auto * I : Dead ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; I -> dump ( ) ) ; I -> eraseFromParent ( ) ; } for ( auto I : L -> blocks ( ) ) DeleteDeadPHIs ( I ) ; return


<s> if ( Const -> getAPInt ( ) != - VF -> getValue ( ) ) return nullptr ; } else return nullptr ; return dyn_cast < SCEVMulExpr > ( S -> getOperand ( <NUM_LIT> ) ) ; } ; auto VisitMul = [ & ] ( const SCEVMulExpr * S ) -> const SCEVUDivExpr * { if ( auto * Const = dyn_cast < SCEVConstant > ( S -> getOperand ( <NUM_LIT> ) ) ) { if ( Const -> getValue ( ) != VF ) return nullptr ; } else return nullptr ; return dyn_cast < SCEVUDivExpr > ( S -> getOperand ( <NUM_LIT> ) ) ; } ; auto VisitDiv = [ & ] ( const SCEVUDivExpr * S ) -> const SCEV * { if ( auto * Const = dyn_cast < SCEVConstant > ( S -> getRHS ( ) ) ) { if ( Const -> getValue ( ) != VF ) return nullptr ; } else return nullptr ; if ( auto * RoundUp = dyn_cast < SCEVAddExpr > ( S -> getLHS ( ) ) ) { if ( auto * Const = dyn_cast < SCEVConstant > ( RoundUp -> getOperand ( <NUM_LIT> ) ) ) { if ( Const -> getAPInt ( ) != ( VF -> getValue ( ) - <NUM_LIT> ) ) return nullptr ; } else return nullptr ; return RoundUp -> getOperand ( <NUM_LIT> ) ; } return nullptr ; } ; const SCEV * Elems = nullptr ; if ( auto * TC = dyn_cast < SCEVAddExpr > ( TripCountSE ) ) if ( auto * Div = dyn_cast < SCEVUDivExpr > ( TC -> getOperand ( <NUM_LIT> ) ) ) if ( auto * Add = dyn_cast < SCEVAddExpr > ( Div -> getLHS ( ) ) ) if ( auto * Mul = VisitAdd ( Add ) ) if ( auto * Div = VisitMul ( Mul ) ) if ( auto * Res = VisitDiv ( Div ) ) Elems = Res ; if ( ! Elems ) return false ; Instruction * InsertPt = L -> getLoopPreheader ( ) -> getTerminator ( ) ; if ( ! isSafeToExpandAt ( Elems , InsertPt , * SE ) ) return false ; auto DL = L -> getHeader ( ) -> getModule ( ) -> getDataLayout ( ) ; SCEVExpander Expander ( * SE , DL , <STR_LIT> ) ; TCP . NumElements = Expander . expandCodeFor ( Elems , Elems -> getType ( ) , InsertPt ) ; if ( ! MatchElemCountLoopSetup ( L , TCP . Shuffle , TCP . NumElements ) )


<s> static VectorType * getVectorType ( IntrinsicInst * I ) { unsigned TypeOp = I -> getIntrinsicID ( ) == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ; auto * PtrTy = cast < PointerType > ( I -> getOperand (


<s> void MVETailPredication :: InsertVCTPIntrinsic ( TripCountPattern & TCP , DenseMap < Instruction * , Instruction * > & NewPredicates ) { IRBuilder < > Builder ( L -> getHeader ( ) -> getFirstNonPHI ( ) ) ; Module * M = L -> getHeader ( ) -> getModule ( ) ; Type * Ty = IntegerType :: get ( M -> getContext ( ) , <NUM_LIT> ) ; PHINode * Processed = Builder . CreatePHI ( Ty , <NUM_LIT> ) ; Processed -> addIncoming ( TCP . NumElements , L -> getLoopPreheader ( ) ) ; Builder . SetInsertPoint ( cast < Instruction > ( TCP . Predicate ) ) ; ConstantInt * Factor = ConstantInt :: get ( cast < IntegerType > ( Ty ) , TCP . VecTy -> getNumElements ( ) ) ; <STR_LIT> VCTPID ; switch ( TCP . VecTy -> getNumElements ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; } Function * VCTP = <STR_LIT> ( M , VCTPID ) ; Value * TailPredicate = Builder . CreateCall ( VCTP , Processed ) ; TCP . Predicate -> replaceAllUsesWith


<s> VectorType * VecTy = getVectorType ( cast < IntrinsicInst > ( & I ) ) ; unsigned Lanes = VecTy -> getNumElements ( ) ; unsigned ElementWidth = VecTy -> getScalarSizeInBits ( ) ; unsigned MaxWidth = TTI -> getRegisterBitWidth ( true ) ; if ( Lanes * ElementWidth > MaxWidth || Lanes == MaxWidth ) return false ; MaskedInsts . push_back ( cast < IntrinsicInst > ( & I ) ) ; } else if ( auto * Int = dyn_cast < IntrinsicInst


<s> Constant * Const = nullptr ; if ( ! match ( TCP . Induction , m_Add ( m_Instruction ( BroadcastSplat ) , m_Constant ( Const ) ) ) && ! match ( TCP . Induction , m_Or ( m_Instruction ( BroadcastSplat ) , m_Constant ( Const ) ) ) ) return false ; if ( auto * CDS = dyn_cast < ConstantDataSequential > ( Const ) ) { for ( unsigned i = <NUM_LIT> ; i < CDS -> getNumElements ( ) ; ++ i ) { if ( CDS -> getElementAsInteger ( i ) != i ) return false ; } } else return false ; Instruction * Insert = nullptr ; if ( ! match ( BroadcastSplat , m_ShuffleVector ( m_Instruction ( Insert ) , m_Undef ( ) , m_ZeroMask ( ) ) ) ) return false ; Instruction * IV = nullptr ; if ( ! match ( Insert , m_InsertElement ( m_Undef ( ) , m_Instruction ( IV ) , m_Zero ( ) ) ) ) return false ; auto * Phi = dyn_cast < PHINode > ( IV ) ; if ( ! Phi ) return


<s> Instruction * Insert = nullptr ; if ( ! match ( Shuffle , m_ShuffleVector ( m_Instruction ( Insert ) , m_Undef ( ) , m_ZeroMask ( ) ) ) ) return false ; Instruction * BECount = nullptr ; if ( ! match ( Insert , m_InsertElement ( m_Undef ( ) , m_Instruction ( BECount ) , m_Zero ( ) ) ) ) return false ; Value * TripCount


<s> LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; TTI = & getAnalysis < TargetTransformInfoWrapperPass > ( ) . getTTI ( F ) ; SE = & getAnalysis < ScalarEvolutionWrapperPass > ( ) . getSE ( ) ; auto * TLIP = getAnalysisIfAvailable < TargetLibraryInfoWrapperPass > ( ) ; TLI = TLIP ? & TLIP -> getTLI ( * L -> getHeader ( ) -> getParent ( ) ) : nullptr ; DL = & L -> getHeader ( ) -> getModule ( ) -> getDataLayout ( ) ; this -> L = L ; if ( ! ST -> hasMVEIntegerOps ( ) || ! ST -> hasV8_1MMainlineOps ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } BasicBlock * Preheader = L -> getLoopPreheader ( ) ; if ( ! Preheader ) return false ; auto FindLoopIterations = [ ] ( BasicBlock * BB ) ->


<s> TripCountPattern ( Instruction * P , Value * TC , VectorType * VT ) : Predicate ( P ) , TripCount ( TC ) , VecTy (


<s> TripCountPattern ( Instruction * P , Value * TC , VectorType * VT ) : Predicate (

P ) , TripCount ( TC ) , VecTy ( VT ) { }


<s> break ; } } } } SmallPtrSet < Instruction * , <NUM_LIT> > Dead ; while ( ! MaybeDead . empty ( ) ) { auto * I = MaybeDead . front ( ) ; MaybeDead . remove ( I ) ; if ( I -> hasNUsesOrMore ( <NUM_LIT> ) ) continue ; for ( auto & U : I -> operands ( ) ) { if ( auto * OpI = dyn_cast < Instruction > ( U ) ) MaybeDead . insert ( OpI ) ; } I -> dropAllReferences ( ) ; Dead . insert ( I


<s> } return nullptr ; } ; const SCEV * Elems = nullptr ; if ( auto * TC = dyn_cast < SCEVAddExpr > ( TripCountSE ) ) if ( auto * Div = dyn_cast < SCEVUDivExpr > ( TC -> getOperand ( <NUM_LIT> ) ) ) if ( auto * Add = dyn_cast < SCEVAddExpr > ( Div -> getLHS ( ) ) ) if ( auto * Mul = VisitAdd ( Add ) ) if ( auto * Div = VisitMul ( Mul ) ) if ( auto * Res = VisitDiv ( Div ) ) Elems = Res ; if ( ! Elems ) return nullptr ; Instruction * InsertPt = L -> getLoopPreheader ( ) -> getTerminator ( ) ; if ( ! isSafeToExpandAt ( Elems , InsertPt , * SE ) ) return nullptr ; auto DL = L -> getHeader ( ) -> getModule ( ) -> getDataLayout ( ) ; SCEVExpander Expander ( * SE , DL , <STR_LIT> ) ; return Expander . expandCodeFor ( Elems , Elems -> getType ( ) , InsertPt


<s> auto * PtrTy = cast < PointerType > ( I -> getOperand ( TypeOp ) -> getType ( ) ) ; return cast < VectorType > ( PtrTy -> getElementType


<s> unsigned MaxWidth = TTI -> getRegisterBitWidth ( true ) ; if ( Lanes * ElementWidth > MaxWidth || Lanes == MaxWidth ) return false ; MaskedInsts . push_back ( cast < IntrinsicInst > ( & I ) ) ; } else if ( auto * Int = dyn_cast < IntrinsicInst > ( & I ) ) { for ( auto & U : Int -> args ( ) ) { if ( isa < VectorType > ( U -> getType ( ) ) )


<s> Instruction * Induction = nullptr ; if ( ! match ( I , m_ICmp ( Pred , m_Instruction ( Induction ) , m_Instruction ( Shuffle ) ) ) || Pred != ICmpInst :: ICMP_ULE || ! L -> isLoopInvariant ( Shuffle ) ) return false ; Instruction * Insert = nullptr ; if ( ! match ( Shuffle , m_ShuffleVector ( m_Instruction ( Insert ) , m_Undef ( ) , m_Zero ( ) ) ) ) return false ; Instruction * BECount = nullptr ; if ( ! match ( Insert , m_InsertElement ( m_Undef ( ) , m_Instruction ( BECount ) , m_Zero ( ) ) ) ) return false ; Value * TripCount = nullptr ; if ( ! match ( BECount , m_Add ( m_Value ( TripCount ) , m_AllOnes ( ) ) ) ) return false ; if ( TripCount != NumElements ) return false ; Instruction * BroadcastSplat = nullptr ; Constant * Const = nullptr ; if ( ! match ( Induction , m_Add ( m_Instruction ( BroadcastSplat ) , m_Constant ( Const ) ) ) ) return false ; if ( auto * CDS = dyn_cast < ConstantDataSequential > ( Const ) ) { for ( unsigned i = <NUM_LIT> ; i < CDS -> getNumElements ( ) ; ++ i ) { if ( CDS -> getElementAsInteger ( i ) != i ) return false ; } } else return false ; if ( ! match ( BroadcastSplat , m_ShuffleVector ( m_Instruction ( Insert ) , m_Undef ( ) , m_Zero ( ) ) ) ) return false ; Instruction * IV


<s> Instruction * Induction = nullptr ; if ( ! match ( I , m_ICmp ( Pred , m_Instruction ( Induction ) , m_Instruction ( Shuffle ) ) ) || Pred != ICmpInst :: ICMP_ULE || ! L -> isLoopInvariant ( Shuffle ) ) return false ; Instruction * Insert = nullptr ; if ( ! match ( Shuffle , m_ShuffleVector ( m_Instruction ( Insert ) , m_Undef ( ) , m_Zero ( ) ) ) ) return false ; Instruction * BECount = nullptr ; if ( ! match ( Insert , m_InsertElement ( m_Undef ( ) , m_Instruction ( BECount ) , m_Zero ( ) ) ) ) return false ; Value * TripCount = nullptr ; if ( ! match ( BECount , m_Add ( m_Value ( TripCount ) , m_AllOnes ( ) ) ) ) return false ; if ( TripCount != NumElements ) return false ; Instruction * BroadcastSplat = nullptr ; Constant * Const = nullptr ; if ( ! match ( Induction , m_Add ( m_Instruction ( BroadcastSplat ) , m_Constant ( Const ) ) ) ) return false ; if ( auto * CDS = dyn_cast < ConstantDataSequential > ( Const ) ) { for ( unsigned i = <NUM_LIT> ; i < CDS -> getNumElements ( ) ; ++ i ) { if ( CDS -> getElementAsInteger ( i ) != i ) return false ; } } else return false ; if ( ! match ( BroadcastSplat , m_ShuffleVector ( m_Instruction ( Insert ) , m_Undef ( ) , m_Zero ( ) ) ) ) return false ; Instruction * IV = nullptr ; if ( ! match ( Insert , m_InsertElement ( m_Undef ( ) , m_Instruction ( IV ) , m_Zero


<s> this -> L = L ; if ( ! ST -> hasMVEIntegerOps ( ) || ! ST -> hasV8_1MMainlineOps ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } BasicBlock * Preheader = L -> getLoopPreheader ( ) ; if ( ! Preheader ) return false ; auto FindLoopIterations = [ ] ( BasicBlock * BB ) -> IntrinsicInst * { for ( auto & I : * BB ) { auto * Call = dyn_cast < IntrinsicInst > ( & I ) ; if ( ! Call ) continue ; <STR_LIT> ID = Call -> getIntrinsicID ( ) ; if ( ID == <STR_LIT> || ID == <STR_LIT> ) return cast < IntrinsicInst > ( & I ) ; } return nullptr ; } ; IntrinsicInst * Setup = FindLoopIterations ( Preheader ) ; if ( ! Setup ) { if ( ! Preheader -> getSinglePredecessor ( )


<s> unsigned VectorWidth = cast < FixedVectorType > ( ActiveLaneMask -> getType ( ) ) -> getNumElements ( ) ; Builder . SetInsertPoint ( L -> getHeader ( ) -> getFirstNonPHI ( ) ) ; PHINode * Processed = Builder . CreatePHI ( Ty , <NUM_LIT> ) ; Processed -> addIncoming ( ActiveLaneMask -> getOperand ( <NUM_LIT> ) , L -> getLoopPreheader ( ) ) ; Builder . SetInsertPoint ( ActiveLaneMask ) ; ConstantInt * Factor = ConstantInt :: get ( cast < IntegerType > ( Ty ) , VectorWidth ) ; <STR_LIT> VCTPID


<s> dbgs ( ) << <STR_LIT> ; EC -> dump ( ) ; dbgs ( ) << <STR_LIT> << VectorWidth << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; Ceil -> dump ( ) ; ) ; bool Zero = SE -> getMinusSCEV ( SE -> getBackedgeTakenCount ( L ) , SE -> getUDivExpr ( SE -> getAddExpr ( SE -> getMulExpr ( Ceil , VW ) , SE -> getNegativeSCEV ( VW ) ) , VW ) ) -> isZero ( ) ; if ( ! Zero ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } } auto * IV = ActiveLaneMask -> getOperand ( <NUM_LIT> ) ; auto * IVExpr = SE -> getSCEV ( IV ) ; auto * AddExpr = dyn_cast < SCEVAddRecExpr > ( IVExpr ) ; if ( ! AddExpr ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; IVExpr -> dump ( ) ) ; return false ; } if ( AddExpr -> getLoop ( ) != L ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto * Base = dyn_cast < SCEVConstant > ( AddExpr -> getOperand ( <NUM_LIT> ) ) ; if ( ! Base || ! Base -> isZero ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto * Step = dyn_cast < SCEVConstant > ( AddExpr -> getOperand ( <NUM_LIT> ) ) ; if ( ! Step ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; AddExpr -> getOperand ( <NUM_LIT> ) -> dump ( ) ) ; return false ; } auto StepValue = Step -> getValue ( ) -> getSExtValue ( ) ; if ( VectorWidth == StepValue ) return true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << StepValue << <STR_LIT> << VectorWidth <<


<s> BasicBlock * Preheader = L -> getLoopPreheader ( ) ; if ( ! Preheader ) return false ; auto FindLoopIterations = [ ] ( BasicBlock * BB ) -> IntrinsicInst * { for ( auto & I : * BB ) { auto * Call = dyn_cast < IntrinsicInst > ( & I ) ; if ( ! Call ) continue ; <STR_LIT> ID = Call -> getIntrinsicID ( ) ; if ( ID == <STR_LIT> || ID == <STR_LIT> ) return cast < IntrinsicInst > ( & I ) ; } return nullptr ; } ; IntrinsicInst * Setup = FindLoopIterations


<s> if ( ! match ( Insert , m_InsertElement ( m_Undef ( ) , m_Instruction ( BECount ) , m_Zero ( ) ) ) ) return false ; Value * TripCount = nullptr ; if ( ! match ( BECount , m_Add ( m_Value ( TripCount ) , m_AllOnes ( ) ) ) ) return false ; if ( TripCount != NumElements || ! L -> isLoopInvariant ( BECount ) ) return false ; return


<s> void MVETailPredication :: RematerializeIterCount ( ) { SmallVector < WeakTrackingVH , <NUM_LIT> > DeadInsts ; SCEVExpander Rewriter ( * SE , * DL , <STR_LIT> ) ; ReplaceExitVal ReplaceExitValue


<s> auto * ST = & TM . getSubtarget < <ISA_LIT>Subtarget > ( F ) ; DT = & getAnalysis < DominatorTreeWrapperPass > ( ) . getDomTree ( ) ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; TTI = & getAnalysis < TargetTransformInfoWrapperPass > ( ) . getTTI ( F ) ; SE = & getAnalysis < ScalarEvolutionWrapperPass > ( ) . getSE ( ) ; auto * TLIP = getAnalysisIfAvailable < TargetLibraryInfoWrapperPass > ( ) ; TLI = TLIP ? & TLIP -> getTLI ( * L -> getHeader ( ) -> getParent ( ) ) : nullptr ; DL = & L -> getHeader ( ) -> getModule ( ) -> getDataLayout ( ) ; this -> L = L ; if ( ! ST -> hasMVEIntegerOps ( ) || ! ST -> hasV8_1MMainlineOps ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } BasicBlock * Preheader = L -> getLoopPreheader ( ) ; if ( ! Preheader ) return false ; auto FindLoopIterations = [ ] ( BasicBlock * BB ) -> IntrinsicInst * { for ( auto & I : * BB ) { auto * Call = dyn_cast < IntrinsicInst > ( & I ) ; if ( ! Call ) continue ; <STR_LIT> ID = Call -> getIntrinsicID ( ) ; if ( ID == <STR_LIT> || ID == <STR_LIT> ) return cast < IntrinsicInst > ( & I ) ; } return nullptr ; } ; IntrinsicInst * Setup = FindLoopIterations ( Preheader ) ; if ( ! Setup ) { if ( ! Preheader -> getSinglePredecessor ( ) ) return false ; Setup = FindLoopIterations ( Preheader -> getSinglePredecessor ( ) ) ; if ( ! Setup ) return false ; } IntrinsicInst * Decrement = nullptr ; for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( IsDecrement ( I ) ) { Decrement = cast < IntrinsicInst > ( & I ) ; break ; } } } if ( ! Decrement ) return false ; ClonedVCTPInExitBlock = false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * L << * Setup << <STR_LIT> << * Decrement << <STR_LIT> ) ; if ( TryConvert ( Setup -> getArgOperand ( <NUM_LIT> ) ) ) { if ( ClonedVCTPInExitBlock ) RematerializeIterCount ( ) ; return true ; } return false


<s> auto * Predicate = dyn_cast < Instruction > ( I -> getArgOperand ( PredOp ) ) ; if ( ! Predicate || Predicates . count ( Predicate ) ) continue ; TripCountPattern TCP ( Predicate , TripCount , getVectorType ( I ) ) ; if ( ! ( ComputeConstElements ( TCP ) || ComputeRuntimeElements ( TCP ) ) ) continue ; if ( ! isTailPredicate ( TCP ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; Predicates . insert ( Predicate ) ; InsertVCTPIntrinsic ( TCP , NewPredicates ) ; } if ( ! NewPredicates . size (


<s> IRBuilder < > Builder ( L -> getLoopPreheader ( ) -> getTerminator ( ) ) ; Module * M = L -> getHeader ( ) -> getModule ( ) ; Type * Ty = IntegerType :: get ( M -> getContext ( ) , <NUM_LIT> ) ; unsigned VectorWidth = cast < FixedVectorType > ( ActiveLaneMask -> getType ( ) ) -> getNumElements ( ) ; Builder . SetInsertPoint ( L -> getHeader ( ) -> getFirstNonPHI ( ) ) ; PHINode * Processed = Builder . CreatePHI ( Ty , <NUM_LIT> ) ; Processed -> addIncoming ( ActiveLaneMask -> getOperand ( <NUM_LIT> ) , L -> getLoopPreheader ( ) ) ; Builder . SetInsertPoint ( ActiveLaneMask ) ; ConstantInt * Factor = ConstantInt :: get ( cast < IntegerType > ( Ty ) , VectorWidth ) ; <STR_LIT> VCTPID ; switch ( VectorWidth ) { default : llvm_unreachable ( <STR_LIT> ) ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; } Function * VCTP = <STR_LIT> ( M , VCTPID ) ; Value * VCTPCall = Builder . CreateCall ( VCTP , Processed


<s> case <STR_LIT> : case <STR_LIT> : continue ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( ST -> hasMVEFloatOps ( ) ) continue ; LLVM_FALLTHROUGH ; default : break ; } if ( IsMasked ( & I ) ) { auto * VecTy = getVectorType ( Int ) ; unsigned Lanes = VecTy -> getNumElements ( ) ; unsigned ElementWidth = VecTy -> getScalarSizeInBits ( ) ; unsigned MaxWidth = TTI -> getRegisterBitWidth ( true ) ; if ( Lanes * ElementWidth > MaxWidth || Lanes == MaxWidth ) return false ; MaskedInsts . push_back ( cast < IntrinsicInst > ( & I ) ) ; continue ; } for ( const Use & U : Int -> args ( ) ) { if ( isa < VectorType > ( U -> getType ( ) ) ) return false ; } } } if ( ! ActiveLaneMask ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return


<s> uint64_t UpperboundTC = SE -> getSignedRange ( TC ) . getUpper ( ) . getZExtValue ( ) ; if ( UpperboundTC > MaxMinusVW && ! ForceTailPredication ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; dbgs ( ) << UpperboundTC << <STR_LIT> << MaxMinusVW << <STR_LIT> ; ) ; return false ; } auto * One = SE -> getOne ( TripCount -> getType ( ) ) ; auto * ElementCount = SE -> getAddExpr ( BTC , One ) ; auto * ECPlusVWMinus1 = SE -> getAddExpr ( ElementCount , SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth - <NUM_LIT> ) ) ) ; auto * Ceil = SE -> getUDivExpr ( ECPlusVWMinus1 , SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth ) ) ) ; ConstantRange RangeCeil = SE -> getSignedRange ( Ceil ) ; ConstantRange RangeTC = SE -> getSignedRange ( TC ) ; if ( ! RangeTC . isSingleElement ( ) ) { auto ZeroRange = ConstantRange ( APInt ( TripCount -> getType ( ) -> getScalarSizeInBits ( ) , <NUM_LIT> ) ) ; RangeTC = RangeTC . unionWith ( ZeroRange ) ; } if ( ! RangeTC . contains ( RangeCeil ) && ! ForceTailPredication ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto * IV = ActiveLaneMask -> getOperand ( <NUM_LIT> ) ; auto * IVExpr = SE -> getSCEV ( IV ) ; auto * AddExpr = dyn_cast < SCEVAddRecExpr > ( IVExpr ) ; if ( ! AddExpr ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; IVExpr -> dump ( ) ) ; return false ; } if ( AddExpr -> getLoop ( ) != L ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto * Step = dyn_cast < SCEVConstant > ( AddExpr -> getOperand ( <NUM_LIT> ) ) ; if ( ! Step ) { LLVM_DEBUG ( dbgs ( ) <<


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ActiveLaneMask << <STR_LIT> ) ; auto * VecTy = getVectorType ( I ) ; if ( ! IsSafeActiveMask ( ActiveLaneMask , TripCount , VecTy ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; InsertVCTPIntrinsic ( ActiveLaneMask , TripCount , VecTy ) ; } Cleanup ( Predicates , L ) ; return


<s> BasicBlock * Preheader = L -> getLoopPreheader ( ) ; if ( ! Preheader ) return false ; auto FindLoopIterations = [ ] ( BasicBlock * BB ) -> IntrinsicInst * { for ( auto & I : * BB ) { auto * Call = dyn_cast < IntrinsicInst > ( & I ) ; if ( ! Call ) continue ; <STR_LIT> ID = Call -> getIntrinsicID ( ) ; if ( ID == <STR_LIT> || ID == <STR_LIT> ) return cast < IntrinsicInst > ( & I ) ; } return nullptr ; } ; IntrinsicInst * Setup = FindLoopIterations ( Preheader ) ; if ( !


<s> auto & TM = TPC . getTM < TargetMachine > ( ) ; ST = & TM . getSubtarget < <ISA_LIT>Subtarget > ( F ) ; TTI = & getAnalysis < TargetTransformInfoWrapperPass > ( ) . getTTI ( F ) ; SE = & getAnalysis < ScalarEvolutionWrapperPass > ( ) . getSE ( ) ; this -> L = L ; if ( ! ST -> hasMVEIntegerOps ( ) || ! ST -> hasV8_1MMainlineOps ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } BasicBlock * Preheader = L -> getLoopPreheader ( ) ; if ( ! Preheader ) return false ; auto FindLoopIterations = [ ] ( BasicBlock * BB ) -> IntrinsicInst * { for ( auto & I : * BB ) { auto * Call = dyn_cast < IntrinsicInst >


<s> AU . addRequired < LoopInfoWrapperPass > ( ) ; AU . addRequired < TargetPassConfig > ( ) ; AU . addRequired < TargetTransformInfoWrapperPass >


<s> if ( ID == <STR_LIT> || ID == <STR_LIT> ) VecTy = dyn_cast < FixedVectorType > ( I -> getType ( ) -> getContainedType ( <NUM_LIT> ) ) ; else VecTy = dyn_cast < FixedVectorType > ( I -> getType ( ) ) ; } else if ( ID == <STR_LIT> ) { VecTy = dyn_cast < FixedVectorType > ( I -> getOperand ( <NUM_LIT> ) -> getType ( ) ) ; } else { VecTy = dyn_cast < FixedVectorType > ( I -> getOperand ( <NUM_LIT> ) -> getType ( ) ) ; } assert ( VecTy && <STR_LIT> ) ; return VecTy


<s> static bool IsMasked ( Instruction * I ) { auto * Call = dyn_cast < IntrinsicInst > ( I


<s> if ( ST -> hasMVEFloatOps ( ) ) continue ; break ; default : break ; } if ( IsMasked ( & I ) ) { auto * VecTy = getVectorType ( Int ) ; unsigned Lanes = VecTy -> getNumElements ( ) ; unsigned ElementWidth = VecTy -> getScalarSizeInBits ( ) ; unsigned MaxWidth = TTI -> getRegisterBitWidth ( true ) ; if ( Lanes * ElementWidth > MaxWidth || Lanes == MaxWidth ) return false ; MaskedInsts . push_back ( cast < IntrinsicInst > ( & I ) ) ; continue ; } for ( const Use & U : Int -> args ( ) ) { if ( isa < VectorType > ( U -> getType ( ) ) ) return false ; } } } if ( !


<s> bool ForceTailPredication = EnableTailPredication == TailPredication :: ForceEnabledNoReductions || EnableTailPredication == TailPredication :: ForceEnabled ; Value * ElemCount = ActiveLaneMask -> getOperand ( <NUM_LIT> ) ; auto * EC = SE -> getSCEV ( ElemCount ) ; auto * TC = SE -> getSCEV ( TripCount ) ; int VectorWidth = VecTy -> getNumElements ( ) ; ConstantInt * ConstElemCount = nullptr ; if ( ! SE -> isLoopInvariant ( EC , L ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( ( ConstElemCount = dyn_cast < ConstantInt > ( ElemCount ) ) ) { ConstantInt * TC = dyn_cast < ConstantInt > ( TripCount ) ; if ( ! TC ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } uint64_t TC1 = TC -> getZExtValue ( ) * VectorWidth ; uint64_t TC2 = ConstElemCount -> getZExtValue ( ) + <NUM_LIT> ; if ( TC1 != TC2 ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << TC1 << <STR_LIT> << TC2 << <STR_LIT> ) ; return false ; } } else if ( ! ForceTailPredication ) { auto * VW = SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth ) ) ; auto * ECPlusVWMinus1 = SE -> getAddExpr ( EC , SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth - <NUM_LIT> ) ) ) ; auto * Ceil = SE -> getUDivExpr ( ECPlusVWMinus1 , VW ) ; ( void ) TC ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; TC -> dump ( ) ; dbgs ( ) << <STR_LIT> ; EC -> dump (


<s> bool MVETailPredication :: runOnLoop ( Loop * L , LPPassManager & ) { if ( skipLoop ( L ) || ! EnableTailPredication ) return false ; MaskedInsts . clear ( ) ; Function & F = * L -> getHeader ( ) -> getParent ( ) ; auto & TPC = getAnalysis < TargetPassConfig > ( ) ; auto & TM = TPC . getTM < TargetMachine > ( ) ; ST = & TM . getSubtarget < <ISA_LIT>Subtarget > ( F ) ; TTI = & getAnalysis < TargetTransformInfoWrapperPass > ( ) . getTTI ( F ) ; SE = & getAnalysis < ScalarEvolutionWrapperPass > ( ) . getSE ( ) ; this -> L = L ; if ( ! ST -> hasMVEIntegerOps ( ) || ! ST -> hasV8_1MMainlineOps ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } BasicBlock * Preheader = L -> getLoopPreheader ( ) ; if ( ! Preheader ) return false ; auto FindLoopIterations = [ ] ( BasicBlock * BB ) -> IntrinsicInst * { for ( auto & I : * BB ) { auto * Call = dyn_cast < IntrinsicInst > ( & I ) ; if ( ! Call ) continue ; <STR_LIT> ID = Call -> getIntrinsicID ( ) ; if ( ID == <STR_LIT> || ID == <STR_LIT> ) return cast < IntrinsicInst > ( & I ) ; } return nullptr ; } ; IntrinsicInst * Setup = FindLoopIterations ( Preheader ) ; if ( ! Setup ) { if ( ! Preheader -> getSinglePredecessor ( ) ) return false ; Setup = FindLoopIterations ( Preheader -> getSinglePredecessor ( ) ) ; if ( ! Setup ) return false ; } IntrinsicInst * Decrement


<s> bool MVETailPredication :: TryConvert ( Value * TripCount ) { if ( ! IsPredicatedVectorLoop ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; SetVector < Instruction * > Predicates ; auto getPredicateOp = [ ] ( IntrinsicInst * I ) { unsigned IntrinsicID = I -> getIntrinsicID ( ) ; if ( IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> ) return <NUM_LIT> ; return ( IntrinsicID == <STR_LIT> || isGather ( I ) ) ? <NUM_LIT> : <NUM_LIT> ; } ; for ( auto * I : MaskedInsts ) { Value * PredOp = I -> getArgOperand ( getPredicateOp ( I ) ) ; auto * Predicate = dyn_cast < Instruction > ( PredOp ) ; if ( ! Predicate || Predicates . count ( Predicate ) ) continue ; auto * ActiveLaneMask = dyn_cast < IntrinsicInst > ( Predicate ) ; if ( ! ActiveLaneMask || ActiveLaneMask -> getIntrinsicID ( ) != <STR_LIT> ) continue ; Predicates . insert ( Predicate ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << *


<s> if ( ! match ( BroadcastSplat , m_Shuffle ( m_Instruction ( Insert ) , m_Undef ( ) , m_ZeroMask ( ) ) ) ) return false ; Instruction * IV = nullptr ; if ( ! match ( Insert , m_InsertElt ( m_Undef ( ) , m_Instruction ( IV ) , m_Zero ( ) ) ) ) return false ; auto * Phi = dyn_cast < PHINode > ( IV ) ; if ( ! Phi ) return false ; Value * OnEntry = Phi -> getIncomingValueForBlock ( L -> getLoopPreheader ( ) ) ; if ( ! match ( OnEntry , m_Zero ( ) ) ) return false ; Value * InLoop = Phi -> getIncomingValueForBlock ( L -> getLoopLatch ( ) ) ; unsigned Lanes = cast < FixedVectorType > ( Insert -> getType ( ) ) ->


<s> } else return false ; Instruction * Insert = nullptr ; if ( ! match ( BroadcastSplat , m_Shuffle ( m_Instruction ( Insert ) , m_Undef ( ) , m_ZeroMask ( ) ) ) ) return false ; Instruction * IV = nullptr ; if ( ! match ( Insert , m_InsertElt ( m_Undef ( ) , m_Instruction ( IV ) , m_Zero ( ) ) ) ) return false ; auto * Phi = dyn_cast < PHINode > ( IV ) ; if ( ! Phi ) return false ; Value * OnEntry = Phi -> getIncomingValueForBlock ( L -> getLoopPreheader ( ) ) ; if ( ! match ( OnEntry , m_Zero ( ) ) ) return false ; Value * InLoop = Phi -> getIncomingValueForBlock ( L -> getLoopLatch ( ) ) ; unsigned Lanes = cast < FixedVectorType > ( Insert -> getType ( ) ) -> getNumElements ( ) ; Instruction * LHS


<s> Value * TripCount = nullptr ; if ( ! match ( BECount , m_Add ( m_Value ( TripCount ) , m_AllOnes ( ) ) ) ) return false ; if ( TripCount != NumElements || ! L -> isLoopInvariant ( BECount ) ) return false ; return


<s> Instruction * Insert = nullptr ; if ( ! match ( Shuffle , m_Shuffle ( m_Instruction ( Insert ) , m_Undef ( ) , m_ZeroMask ( ) ) ) ) return false ; Instruction * BECount = nullptr ; if ( ! match ( Insert , m_InsertElt ( m_Undef ( ) , m_Instruction ( BECount


<s> TripCountPattern ( Instruction * P , Value * TC , FixedVectorType * VT ) : Predicate ( P )


<s> TripCountPattern ( Instruction * P , Value * TC , FixedVectorType * VT ) : Predicate

( P ) , TripCount ( TC ) , VecTy ( VT ) { }


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; TC -> dump ( ) ; dbgs ( ) << <STR_LIT> ; EC -> dump ( ) ; dbgs ( ) << <STR_LIT> << VectorWidth << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; Ceil -> dump ( ) ; ) ; bool Zero = SE -> getMinusSCEV ( SE -> getBackedgeTakenCount ( L ) , SE -> getUDivExpr ( SE -> getAddExpr ( SE -> getMulExpr ( Ceil , VW ) , SE -> getNegativeSCEV ( VW ) ) , VW ) ) -> isZero ( ) ; if ( ! Zero ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } } auto * IV = ActiveLaneMask -> getOperand ( <NUM_LIT> ) ; auto * IVExpr = SE -> getSCEV ( IV ) ; auto * AddExpr = dyn_cast < SCEVAddRecExpr > ( IVExpr ) ; if ( ! AddExpr ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; IVExpr -> dump ( ) ) ; return false ; } if ( AddExpr -> getLoop ( ) != L ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto * Base = dyn_cast < SCEVConstant > ( AddExpr -> getOperand ( <NUM_LIT> ) ) ; if ( ! Base || ! Base -> isZero ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto * Step = dyn_cast < SCEVConstant > ( AddExpr -> getOperand ( <NUM_LIT> ) ) ; if ( ! Step ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; AddExpr -> getOperand ( <NUM_LIT> ) -> dump ( ) ) ; return false ; } auto StepValue = Step -> getValue ( ) -> getSExtValue ( ) ; if ( VectorWidth == StepValue ) return true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << StepValue << <STR_LIT> << VectorWidth << <STR_LIT> ) ; return


<s> DenseMap < Instruction * , Instruction * > NewPredicates ; for ( auto * I : MaskedInsts ) { <STR_LIT> ID = I -> getIntrinsicID ( ) ; unsigned PredOp = ID == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ; auto * Predicate = dyn_cast < Instruction > ( I -> getArgOperand ( PredOp ) ) ; if ( ! Predicate || Predicates . count ( Predicate ) ) continue ; VectorType * VecTy = getVectorType ( I ) ; Value * NumElements = ComputeElements ( TripCount , VecTy ) ; if ( ! NumElements ) continue ; if ( ! isTailPredicate ( Predicate , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; Predicates . insert ( Predicate ) ; IRBuilder < > Builder ( L -> getHeader ( ) -> getFirstNonPHI ( ) ) ; PHINode * Processed = Builder . CreatePHI ( Ty , <NUM_LIT> ) ; Processed -> addIncoming ( NumElements , L -> getLoopPreheader ( ) ) ; Builder . SetInsertPoint ( cast < Instruction > ( Predicate ) ) ; ConstantInt * Factor = ConstantInt :: get ( cast < IntegerType > ( Ty ) , VecTy -> getNumElements ( ) ) ; <STR_LIT> VCTPID ; switch ( VecTy -> getNumElements ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; } Function * VCTP = <STR_LIT> ( M , VCTPID ) ; Value * TailPredicate = Builder . CreateCall ( VCTP , Processed ) ; Predicate -> replaceAllUsesWith ( TailPredicate ) ; NewPredicates [ Predicate ] = cast < Instruction > ( TailPredicate ) ; Value * Remaining = Builder . CreateSub ( Processed , Factor ) ; Processed -> addIncoming ( Remaining , L -> getLoopLatch ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Processed << <STR_LIT> << <STR_LIT> << * TailPredicate << <STR_LIT> ) ; } Cleanup ( NewPredicates , Predicates , L ) ; return


<s> continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; Predicates . insert ( Predicate ) ; IRBuilder < > Builder ( L -> getHeader ( ) -> getFirstNonPHI ( ) ) ; PHINode * Processed = Builder . CreatePHI ( Ty , <NUM_LIT> ) ; Processed -> addIncoming ( NumElements , L -> getLoopPreheader ( ) ) ; Builder . SetInsertPoint ( cast < Instruction > ( Predicate ) ) ; ConstantInt * Factor = ConstantInt :: get ( cast < IntegerType > ( Ty ) , VecTy -> getNumElements ( ) ) ; <STR_LIT> VCTPID ; switch ( VecTy -> getNumElements ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; } Function * VCTP = <STR_LIT> ( M , VCTPID ) ; Value * TailPredicate = Builder . CreateCall ( VCTP , Processed ) ; Predicate -> replaceAllUsesWith ( TailPredicate ) ; NewPredicates [ Predicate ] = cast < Instruction > ( TailPredicate ) ; Value * Remaining = Builder . CreateSub ( Processed , Factor ) ; Processed -> addIncoming ( Remaining , L ->


<s> Type * Ty = IntegerType :: get ( M -> getContext ( ) , <NUM_LIT> ) ; PHINode * Processed = Builder . CreatePHI ( Ty , <NUM_LIT> ) ; Processed -> addIncoming ( NumElements , L -> getLoopPreheader ( ) ) ; Builder . SetInsertPoint ( cast < Instruction > ( Predicate ) ) ; ConstantInt * Factor = ConstantInt :: get ( cast < IntegerType > ( Ty ) , VecTy -> getNumElements ( ) ) ; <STR_LIT> VCTPID ; switch ( VecTy -> getNumElements ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; } Function * VCTP = <STR_LIT> ( M , VCTPID ) ; Value * TailPredicate = Builder . CreateCall ( VCTP , Processed ) ; Predicate -> replaceAllUsesWith ( TailPredicate ) ; NewPredicates [ Predicate ] = cast < Instruction > ( TailPredicate


<s> Instruction * BroadcastSplat = nullptr ; Constant * Const = nullptr ; if ( ! match ( Induction , m_Add ( m_Instruction ( BroadcastSplat ) , m_Constant ( Const ) ) ) ) return false ; if ( auto * CDS = dyn_cast < ConstantDataSequential > ( Const ) ) { for ( unsigned i = <NUM_LIT> ; i < CDS -> getNumElements ( ) ; ++ i ) { if ( CDS -> getElementAsInteger ( i ) != i ) return false ; } } else return false ; if ( ! match ( BroadcastSplat , m_ShuffleVector ( m_Instruction ( Insert ) , m_Undef ( ) , m_Zero ( ) ) ) ) return false ; Instruction * IV = nullptr ; if ( ! match ( Insert , m_InsertElement ( m_Undef ( ) , m_Instruction

( IV ) , m_Zero ( ) ) ) ) return false ;


<s> return false ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; SetVector < Instruction * > Predicates ; DenseMap < Instruction * , Instruction * > NewPredicates ; for ( auto * I : MaskedInsts ) { <STR_LIT> ID = I -> getIntrinsicID ( ) ; unsigned PredOp = ID == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ; auto * Predicate = dyn_cast < Instruction > ( I -> getArgOperand ( PredOp ) ) ; if ( ! Predicate || Predicates . count ( Predicate ) ) continue ; VectorType * VecTy = getVectorType ( I ) ; Value * NumElements = ComputeElements ( TripCount , VecTy ) ; if ( ! NumElements ) continue ; if ( ! isTailPredicate ( Predicate , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; Predicates . insert ( Predicate ) ; InsertVCTPIntrinsic ( Predicate , NewPredicates , VecTy ,


<s> DenseMap < Instruction * , Instruction * > NewPredicates ; for ( auto * I : MaskedInsts ) { <STR_LIT> ID = I -> getIntrinsicID ( ) ; unsigned PredOp = ID == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ; auto * Predicate = dyn_cast < Instruction > ( I -> getArgOperand ( PredOp ) ) ; if ( ! Predicate || Predicates . count ( Predicate ) ) continue ; VectorType * VecTy = getVectorType ( I ) ; Value * NumElements = ComputeElements ( TripCount , VecTy ) ; if ( ! NumElements ) continue ; if ( ! isTailPredicate ( Predicate , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << *


<s> if ( ! isTailPredicate ( Predicate , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; Predicates . insert ( Predicate ) ; InsertVCTPIntrinsic ( Predicate , NewPredicates , VecTy , NumElements ) ; } ClonedVCTPInExitBlock = Cleanup ( NewPredicates ,


<s> bool MVETailPredication :: IsPredicatedVectorLoop ( ) { for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( IsMasked ( & I ) ) { VectorType * VecTy = getVectorType ( cast < IntrinsicInst > ( & I ) ) ; unsigned Lanes = VecTy -> getNumElements ( ) ; unsigned ElementWidth = VecTy -> getScalarSizeInBits


<s> Instruction * Induction = nullptr ; if ( ! match ( V , m_ICmp ( Pred , m_Instruction ( Induction ) , m_Instruction ( Shuffle ) ) ) || Pred != ICmpInst :: ICMP_ULE || ! L -> isLoopInvariant ( Shuffle ) ) return false ; Instruction * Insert = nullptr ; if ( ! match ( Shuffle , m_ShuffleVector ( m_Instruction ( Insert ) , m_Undef ( ) , m_Zero ( ) ) ) ) return false ; Instruction * BECount = nullptr ; if ( ! match ( Insert , m_InsertElement ( m_Undef ( ) , m_Instruction ( BECount ) , m_Zero ( ) ) ) ) return false ; Value * TripCount = nullptr ; if ( ! match ( BECount , m_Add ( m_Value ( TripCount ) , m_AllOnes ( ) ) ) ) return false ; if ( TripCount != NumElements ) return false ; Instruction * BroadcastSplat = nullptr ; Constant * Const = nullptr ; if ( ! match ( Induction , m_Add ( m_Instruction ( BroadcastSplat ) , m_Constant ( Const ) ) ) ) return false ; if ( auto * CDS = dyn_cast < ConstantDataSequential > ( Const ) ) { for ( unsigned i = <NUM_LIT> ; i < CDS -> getNumElements ( ) ; ++ i ) { if ( CDS -> getElementAsInteger ( i ) != i ) return false ; } } else return false ; if ( ! match ( BroadcastSplat , m_ShuffleVector ( m_Instruction ( Insert ) , m_Undef ( ) , m_Zero ( ) ) ) ) return false ; Instruction * IV = nullptr ; if ( ! match ( Insert , m_InsertElement ( m_Undef ( ) , m_Instruction ( IV ) , m_Zero ( ) ) ) ) return false ; auto * Phi = dyn_cast < PHINode > ( IV ) ; if ( ! Phi ) return false ; Value * OnEntry = Phi -> getIncomingValueForBlock ( L -> getLoopPreheader ( ) ) ; if ( ! match ( OnEntry , m_Zero ( ) ) ) return false ; Value * InLoop = Phi -> getIncomingValueForBlock ( L -> getLoopLatch ( ) ) ; unsigned Lanes = cast < VectorType > ( Insert -> getType ( ) ) -> getNumElements ( ) ; Instruction * LHS = nullptr ; if ( ! match ( InLoop , m_Add ( m_Instruction ( LHS ) , m_SpecificInt ( Lanes ) ) ) ) return false ; return LHS ==


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; Predicates . insert ( Predicate ) ; IRBuilder < > Builder ( L -> getHeader ( ) -> getFirstNonPHI ( ) ) ; PHINode * Processed = Builder . CreatePHI ( Ty , <NUM_LIT> ) ; Processed -> addIncoming ( NumElements , L -> getLoopPreheader ( ) ) ; Builder . SetInsertPoint ( cast < Instruction > ( Predicate ) ) ; ConstantInt * Factor = ConstantInt :: get ( cast < IntegerType > ( Ty ) , VecTy -> getNumElements ( ) ) ; <STR_LIT> VCTPID ; switch ( VecTy -> getNumElements ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ;


<s> ConstantInt * ConstElemCount = nullptr ; if ( ! SE -> isLoopInvariant ( EC , L ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( ( ConstElemCount = dyn_cast < ConstantInt > ( ElemCount ) ) ) { ConstantInt * TC = dyn_cast < ConstantInt > ( TripCount ) ; if ( ! TC ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } uint64_t TC1 = TC -> getZExtValue ( ) ; uint64_t TC2 = ( ConstElemCount -> getZExtValue ( ) + VectorWidth - <NUM_LIT> ) / VectorWidth ; if ( TC1 != TC2 ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << TC1 << <STR_LIT> << TC2 << <STR_LIT> ) ; return false ; } } else if ( ! ForceTailPredication ) { auto * VW = SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth ) ) ; auto * ECPlusVWMinus1 = SE -> getAddExpr ( EC , SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth - <NUM_LIT> ) ) ) ; auto * Ceil = SE -> getUDivExpr ( ECPlusVWMinus1 , VW ) ; ( void ) TC ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; TC -> dump


<s> } } } SmallPtrSet < Instruction * , <NUM_LIT> > Dead ; while ( ! MaybeDead . empty ( ) ) { auto * I = MaybeDead . front ( ) ; MaybeDead . remove ( I ) ; if ( I -> hasNUsesOrMore ( <NUM_LIT> ) ) continue ; for ( auto & U : I -> operands ( ) ) if ( auto * OpI = dyn_cast < Instruction > ( U ) ) MaybeDead . insert ( OpI ) ; Dead . insert ( I ) ; } for ( auto * I : Dead ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; I -> dump ( ) ) ; I -> eraseFromParent


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } bool ClonedVCTPInExitBlock = false ; for ( auto & Pair : NewPredicates ) { Instruction * OldPred = Pair . first ; Instruction * NewPred = Pair . second ; for ( auto & I : * Exit ) { if ( I . isSameOperationAs ( OldPred ) ) { Instruction * PredClone = NewPred -> clone ( ) ; PredClone -> insertBefore ( & I ) ; I . replaceAllUsesWith ( PredClone ) ; MaybeDead . insert ( & I ) ; ClonedVCTPInExitBlock = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; I . dump ( ) ; dbgs ( ) << <STR_LIT> ; PredClone -> dump ( ) ) ; break ; } } } SmallPtrSet < Instruction * , <NUM_LIT> > Dead ; while ( ! MaybeDead . empty ( ) ) { auto * I = MaybeDead . front ( ) ; MaybeDead . remove ( I ) ; if ( I -> hasNUsesOrMore ( <NUM_LIT> ) )


<s> Pass * llvm :: createMVETailPredicationPass ( ) { return new MVETailPredication


<s> Pass * llvm :: createMVETailPredicationPass ( )


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < ScalarEvolutionWrapperPass > ( ) ; AU . addRequired < LoopInfoWrapperPass > ( ) ; AU . addRequired < TargetPassConfig >


<s> auto * PtrTy = cast < PointerType > ( I -> getOperand ( TypeOp ) -> getType ( ) ) ; auto * VecTy = cast < FixedVectorType > ( PtrTy -> getElementType ( ) ) ; assert ( VecTy && <STR_LIT> ) ; return VecTy


<s> auto * PtrTy = cast < PointerType > ( I -> getOperand ( TypeOp ) -> getType ( ) ) ; auto * VecTy = cast < FixedVectorType > ( PtrTy


<s> Type * Ty = IntegerType :: get ( M -> getContext ( ) , <NUM_LIT> ) ; Value * BTC = ActiveLaneMask -> getOperand ( <NUM_LIT> ) ; Builder . SetInsertPoint ( L -> getLoopPreheader ( ) -> getTerminator ( ) ) ; Value * NumElements = getNumElements ( L -> getLoopPreheader ( ) , BTC ) ; Builder . SetInsertPoint ( L -> getHeader ( ) -> getFirstNonPHI ( ) ) ; PHINode * Processed = Builder . CreatePHI ( Ty , <NUM_LIT> ) ; Processed -> addIncoming ( NumElements , L -> getLoopPreheader ( ) ) ; Builder . SetInsertPoint ( ActiveLaneMask ) ; ConstantInt * Factor = ConstantInt :: get ( cast < IntegerType > ( Ty ) , VecTy -> getNumElements ( ) ) ; <STR_LIT> VCTPID ; switch ( VecTy -> getNumElements ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; } Function * VCTP = <STR_LIT> ( M , VCTPID ) ; Value * VCTPCall = Builder . CreateCall ( VCTP , Processed


<s> static bool IsDecrement ( Instruction & I ) { auto * Call = dyn_cast < IntrinsicInst > ( & I ) ; if ( ! Call ) return false


<s> if ( IsMasked ( & I ) ) { FixedVectorType * VecTy = getVectorType ( cast < IntrinsicInst > ( & I ) ) ; unsigned Lanes = VecTy -> getNumElements ( ) ; unsigned ElementWidth = VecTy -> getScalarSizeInBits ( ) ; unsigned MaxWidth = TTI -> getRegisterBitWidth ( true ) ; if ( Lanes * ElementWidth > MaxWidth || Lanes == MaxWidth ) return false ; MaskedInsts . push_back ( cast < IntrinsicInst > ( & I ) ) ; } else if ( auto * Int = dyn_cast < IntrinsicInst > ( & I ) ) { if ( Int -> getIntrinsicID ( ) == <STR_LIT> ) continue ; for ( auto & U : Int ->


<s> return false ; } auto * One = SE -> getOne ( TripCount -> getType ( ) ) ; auto * ElementCount = SE -> getAddExpr ( BTC , One ) ; auto * ECPlusVWMinus1 = SE -> getAddExpr ( ElementCount , SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth - <NUM_LIT> ) ) ) ; auto * Ceil = SE -> getUDivExpr ( ECPlusVWMinus1 , SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth ) ) ) ; ConstantRange RangeCeil = SE -> getSignedRange ( Ceil ) ; ConstantRange RangeTC = SE -> getSignedRange ( TC ) ; if ( ! RangeTC . isSingleElement ( ) ) { auto ZeroRange = ConstantRange ( APInt ( TripCount -> getType ( ) -> getScalarSizeInBits ( ) , <NUM_LIT> ) ) ; RangeTC = RangeTC . unionWith ( ZeroRange ) ; } if ( ! RangeTC . contains ( RangeCeil ) && ! ForceTailPredication ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto * IV = ActiveLaneMask -> getOperand ( <NUM_LIT> ) ; auto * IVExpr = SE -> getSCEV ( IV ) ; auto * AddExpr = dyn_cast < SCEVAddRecExpr > ( IVExpr ) ; if ( ! AddExpr ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; IVExpr -> dump ( ) ) ; return false


<s> auto * BTC = SE -> getSCEV ( BackedgeTakenCount ) ; if ( ! llvm :: cannotBeMaxInLoop ( BTC , L , * SE , false ) && ! ForceTailPredication ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; BTC -> dump ( ) ) ; return false ; } auto * TC = SE -> getSCEV ( TripCount ) ; unsigned SizeInBits = TripCount -> getType ( ) -> getScalarSizeInBits ( ) ; auto Diff = APInt ( SizeInBits , ~ <NUM_LIT> ) - APInt ( SizeInBits , VectorWidth ) ; uint64_t MaxMinusVW = Diff . getZExtValue ( ) ; uint64_t UpperboundTC = SE -> getSignedRange ( TC ) . getUpper ( ) . getZExtValue ( ) ; if ( UpperboundTC > MaxMinusVW && ! ForceTailPredication ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; dbgs ( ) << UpperboundTC << <STR_LIT> << MaxMinusVW << <STR_LIT> ; ) ; return false ; } auto * One = SE -> getOne ( TripCount -> getType ( ) ) ; auto * ElementCount = SE -> getAddExpr ( BTC , One ) ; auto * ECPlusVWMinus1 = SE -> getAddExpr ( ElementCount , SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth - <NUM_LIT> ) ) ) ; auto * Ceil = SE -> getUDivExpr ( ECPlusVWMinus1 , SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) ,


<s> IRBuilder < > Builder ( ActiveLaneMask -> getParent ( ) -> getFirstNonPHI ( ) ) ; Value * Index = ActiveLaneMask -> getOperand ( <NUM_LIT> ) ; Value * SplatIndex = Builder . CreateVectorSplat ( VectorWidth , Index , <STR_LIT> ) ; SmallVector < Constant * , <NUM_LIT> > Indices ; for ( int i = <NUM_LIT> ; i < VectorWidth ; ++ i ) Indices . push_back ( ConstantInt :: get ( Index -> getType ( ) , i ) ) ; Constant * CV = ConstantVector :: get ( Indices ) ; Value * Induction = Builder . CreateAdd ( SplatIndex , CV , <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * SplatIndex << <STR_LIT> ; dbgs ( ) << <STR_LIT> << * Induction << <STR_LIT> ) ; auto * Preheader = L -> getLoopPreheader ( ) ; auto * BTC = ActiveLaneMask -> getOperand ( <NUM_LIT> ) ; Value * SplatBTC = nullptr ; if ( auto * C = dyn_cast < ConstantInt > ( BTC ) ) { Builder . SetInsertPoint ( Preheader -> getTerminator ( ) ) ; SplatBTC = Builder . CreateVectorSplat ( VectorWidth , C ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * SplatBTC << <STR_LIT> ) ; } else { Instruction * InsertElem ; for ( auto & V : * Preheader ) { InsertElem = dyn_cast < InsertElementInst > ( & V ) ; if ( ! InsertElem ) continue ; ConstantInt * CI = dyn_cast < ConstantInt > ( InsertElem -> getOperand ( <NUM_LIT> ) ) ; if ( ! CI ) continue ; if ( InsertElem -> getOperand ( <NUM_LIT> ) != BTC || CI -> getSExtValue ( ) != <NUM_LIT> ) continue ; if ( ( SplatBTC = dyn_cast < ShuffleVectorInst > ( * InsertElem -> users ( ) . begin ( ) ) ) ) break ; } } if ( ! SplatBTC ) { Builder . SetInsertPoint ( Preheader -> getTerminator ( ) ) ; Value * Undef = UndefValue :: get ( FixedVectorType :: get ( BTC -> getType ( ) ,


<s> for ( auto & I : * BB ) { auto * Call = dyn_cast < IntrinsicInst > ( & I ) ; if ( ! Call ) continue ; <STR_LIT> ID = Call -> getIntrinsicID ( ) ; if ( ID == <STR_LIT> || ID == <STR_LIT> ) return cast < IntrinsicInst > ( & I ) ; } return nullptr ; } ; IntrinsicInst * Setup = FindLoopIterations ( Preheader ) ; if ( ! Setup ) { if ( ! Preheader -> getSinglePredecessor ( ) ) return false ; Setup = FindLoopIterations ( Preheader -> getSinglePredecessor ( ) ) ; if ( ! Setup ) return false ; } IntrinsicInst * Decrement = nullptr ; for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( IsDecrement ( I ) ) { Decrement = cast < IntrinsicInst > ( & I ) ; break ; } } } if ( ! Decrement ) return false ; ClonedVCTPInExitBlock = false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * L << * Setup << <STR_LIT> << * Decrement << <STR_LIT> ) ; if ( TryConvert ( Setup -> getArgOperand (


<s> DT = & getAnalysis < DominatorTreeWrapperPass > ( ) . getDomTree ( ) ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; TTI = & getAnalysis < TargetTransformInfoWrapperPass > ( ) . getTTI ( F ) ; SE = & getAnalysis < ScalarEvolutionWrapperPass > ( ) . getSE ( ) ; auto * TLIP = getAnalysisIfAvailable < TargetLibraryInfoWrapperPass > ( ) ; TLI = TLIP ? & TLIP -> getTLI ( * L -> getHeader ( ) -> getParent ( ) ) : nullptr ; DL = & L -> getHeader ( ) -> getModule ( ) -> getDataLayout ( ) ; this -> L = L ; ActiveLaneMask = nullptr ; if ( ! ST -> hasMVEIntegerOps ( ) || ! ST -> hasV8_1MMainlineOps ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } BasicBlock * Preheader = L -> getLoopPreheader ( ) ; if ( ! Preheader ) return false ; auto FindLoopIterations = [ ] ( BasicBlock * BB ) -> IntrinsicInst * { for ( auto & I : * BB ) { auto * Call = dyn_cast < IntrinsicInst > ( & I ) ; if ( ! Call ) continue ; <STR_LIT> ID = Call -> getIntrinsicID ( ) ; if ( ID == <STR_LIT> || ID == <STR_LIT> ) return cast < IntrinsicInst > ( & I ) ; } return nullptr ; } ; IntrinsicInst * Setup = FindLoopIterations ( Preheader ) ; if ( ! Setup ) { if ( ! Preheader -> getSinglePredecessor ( ) ) return false ; Setup = FindLoopIterations ( Preheader ->


<s> if ( ! Predicate || Predicates . count ( Predicate ) ) continue ; ActiveLaneMask = dyn_cast < IntrinsicInst > ( Predicate ) ; if ( ! ActiveLaneMask || ActiveLaneMask -> getIntrinsicID ( ) != <STR_LIT> ) continue ; Predicates . insert ( Predicate ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ActiveLaneMask << <STR_LIT> ) ; VecTy = getVectorType ( I ) ; if ( ! IsSafeActiveMask ( TripCount , VecTy ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; InsertVCTPIntrinsic ( ActiveLaneMask , TripCount


<s> for ( auto * I : MaskedInsts ) { unsigned PredOp = I -> getIntrinsicID ( ) == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ; auto * Predicate = dyn_cast < Instruction > ( I -> getArgOperand ( PredOp ) ) ; if ( ! Predicate || Predicates . count ( Predicate ) ) continue ; ActiveLaneMask = dyn_cast < IntrinsicInst > ( Predicate ) ; if ( ! ActiveLaneMask || ActiveLaneMask -> getIntrinsicID ( ) != <STR_LIT>


<s> return MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || MI


<s> MachineBasicBlock * MBB = MI -> getParent ( ) ; assert ( MI -> getOpcode ( ) == <STR_LIT> && <STR_LIT> ) ; if ( UseCmp ) { MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <NUM_LIT> ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT> ) ; } else { MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <NUM_LIT>


<s> MachineBasicBlock * MBB = MI -> getParent ( ) ; assert ( MI -> getOpcode ( ) == <STR_LIT> && <STR_LIT> ) ; if ( UseCmp ) { MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <NUM_LIT> ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT> ) ; } else { MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . add ( MI -> getOperand


<s> static inline bool isLoopStart ( MachineInstr & MI ) { return MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) ==

<STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ;


<s> MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT> ) ; MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( BrOpc ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT> ) ; MI -> eraseFromParent


<s> assert ( isWhileLoopStart ( MI ) && <STR_LIT> ) ; unsigned Op = MI . getOpcode ( )


<s> static inline bool isDoLoopStart ( const MachineInstr & MI


<s> static inline bool isLoopStart ( const MachineInstr & MI ) { return isDoLoopStart ( MI ) || isWhileLoopStart


<s> inline void RevertDoLoopStart ( MachineInstr * MI , const TargetInstrInfo * TII ) { MachineBasicBlock * MBB = MI -> getParent ( ) ; BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . add ( MI ->

getOperand ( <NUM_LIT> ) ) . add ( MI -> getOperand ( <NUM_LIT> ) ) . add ( predOps ( <STR_LIT> ) ) ;


<s> MIB . addReg ( <STR_LIT> ) ; } MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( BrOpc ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg (


<s> MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <NUM_LIT> ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT> , RegState :: Define ) ; } MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( BrOpc ) ) ; MIB . addMBB ( getWhileLoopStartTargetBB ( * MI ) ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg


<s> if ( Visited . count ( MBB ) ) continue ; for ( MachineInstr & MI : * MBB ) { if ( ! IsInvalidTPInstruction ( MI ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MachineInstrBuilder MIB = BuildMI ( * PreHeader , PreHeader -> getFirstTerminator ( ) , LoopStart -> getDebugLoc (


<s> MachineInstr * MVETPAndVPTOptimisations :: CheckForLRUseInPredecessors ( MachineBasicBlock * PreHeader , MachineInstr * LoopStart ) { SmallVector < MachineBasicBlock * > Worklist ; SmallPtrSet < MachineBasicBlock * , <NUM_LIT> > Visited ; Worklist . push_back ( PreHeader ) ; Visited . insert ( LoopStart -> getParent ( ) ) ; while ( ! Worklist . empty ( ) ) { MachineBasicBlock * MBB = Worklist . pop_back_val ( ) ; if ( Visited . count ( MBB ) ) continue ; for ( MachineInstr & MI : * MBB ) { if ( ! IsInvalidTPInstruction ( MI ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MachineInstrBuilder MIB = BuildMI ( * PreHeader , PreHeader -> getFirstTerminator ( ) , LoopStart -> getDebugLoc ( ) , TII


<s> MachineInstr * LoopEnd , * LoopPhi , * LoopStart , * LoopDec ; if ( ! findLoopComponents ( ML , MRI , LoopStart , LoopPhi , LoopDec , LoopEnd ) ) return false ; if ( LoopDec != LoopEnd || LoopStart -> getOpcode ( ) != <STR_LIT> ) return false ; SmallVector < MachineInstr * , <NUM_LIT> > VCTPs ; for ( MachineBasicBlock * BB : ML -> blocks ( ) ) for ( MachineInstr & MI : * BB ) if ( isVCTP ( & MI ) ) VCTPs . push_back ( & MI ) ; if ( VCTPs . empty ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MachineInstr * FirstVCTP = * VCTPs . begin ( ) ; for ( MachineInstr * VCTP : VCTPs ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * VCTP ) ; if ( VCTP -> getOpcode ( ) != FirstVCTP -> getOpcode ( ) || VCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) != FirstVCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } } Register CountReg = FirstVCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! CountReg . isVirtual ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MachineInstr * Phi = LookThroughCOPY ( MRI -> getVRegDef ( CountReg ) , MRI ) ; if ( ! Phi || Phi -> getOpcode ( ) != TargetOpcode :: PHI || Phi -> getNumOperands ( ) != <NUM_LIT> || ( Phi -> getOperand ( <NUM_LIT> ) . getMBB ( ) != ML -> getLoopLatch ( ) && Phi -> getOperand ( <NUM_LIT> ) . getMBB ( ) != ML -> getLoopLatch ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } CountReg = Phi -> getOperand ( <NUM_LIT> ) . getMBB ( ) == ML -> getLoopLatch ( ) ? Phi -> getOperand ( <NUM_LIT> ) . getReg ( ) : Phi -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineBasicBlock * MBB = LoopStart -> getParent ( ) ; MachineBasicBlock :: iterator InsertPt = MBB -> getFirstTerminator ( ) ; for ( MachineInstr & Use : MRI -> use_instructions ( LoopStart -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) if ( ( InsertPt != MBB -> end ( ) && ! DT -> dominates ( & * InsertPt , & Use ) ) || ! DT -> dominates ( ML -> getHeader ( ) , Use . getParent ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MachineInstrBuilder MI = BuildMI ( * MBB , InsertPt , LoopStart -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . add

( LoopStart -> getOperand ( <NUM_LIT> ) ) . add ( LoopStart -> getOperand ( <NUM_LIT> ) ) . addReg ( CountReg ) ;


<s> if ( llvm :: is_contained ( ExpectedUsers , & MI ) ) continue ; if ( MI . getOpcode ( ) != TargetOpcode :: COPY || ! MI . getOperand ( <NUM_LIT> ) . getReg ( ) . isVirtual ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; return false ; } Worklist . push_back ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Copies . push_back ( & MI ) ; } } return true ; } ; if ( ! CheckUsers ( PhiReg , { LoopDec } , MRI ) || ! CheckUsers ( DecReg , { LoopPhi , LoopEnd } , MRI ) || ! CheckUsers ( StartReg , { LoopPhi } , MRI ) ) { if ( LoopStart -> getOpcode ( ) == <STR_LIT> ) { RevertWhileLoopStartLR ( LoopStart , TII ) ; RevertLoopDec ( LoopDec , TII ) ; RevertLoopEnd ( LoopEnd , TII ) ; return true ; } return false ; } MRI -> constrainRegClass ( StartReg , & <STR_LIT> ) ; MRI -> constrainRegClass ( PhiReg , & <STR_LIT> ) ; MRI -> constrainRegClass ( DecReg , & <STR_LIT> ) ; if ( LoopPhi -> getOperand ( <NUM_LIT> ) . getMBB ( ) == ML -> getLoopLatch ( ) ) { LoopPhi -> getOperand ( <NUM_LIT> ) . setReg ( StartReg ) ; LoopPhi -> getOperand ( <NUM_LIT> ) . setReg ( DecReg ) ; } else { LoopPhi -> getOperand ( <NUM_LIT> ) . setReg ( StartReg ) ; LoopPhi -> getOperand ( <NUM_LIT> ) . setReg ( DecReg ) ; } SmallVector < MachineOperand , <NUM_LIT> > Cond ; MachineBasicBlock * TBB = nullptr , * FBB = nullptr ; if ( ! TII -> analyzeBranch ( * LoopEnd -> getParent ( ) , TBB , FBB , Cond ) && ! FBB ) { MachineFunction :: iterator MBBI = ++ LoopEnd -> getParent


<s> const <ISA_LIT>Subtarget & STI = Fn . getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( ! STI . isThumb2 ( ) || ! STI . hasLOB ( ) ) return false ; TII = static_cast < const Thumb2InstrInfo * > ( STI . getInstrInfo ( ) ) ; MRI = & Fn . getRegInfo ( ) ; MachineLoopInfo * MLI = & getAnalysis < MachineLoopInfo > ( ) ; MachineDominatorTree * DT = & getAnalysis < MachineDominatorTree > ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << Fn . getName ( ) << '\n' ) ; bool Modified = false ; for ( MachineLoop * ML : MLI -> getBase ( ) . getLoopsInPreorder ( ) ) { Modified |= LowerWhileLoopStart ( ML ) ; Modified |= MergeLoopEnd ( ML ) ; Modified |= ConvertTailPredLoop ( ML , DT ) ; } for ( MachineBasicBlock & MBB : Fn ) { Modified |= HintDoLoopStartReg ( MBB ) ; Modified |= ReplaceConstByVPNOTs ( MBB , DT ) ; Modified |= ReplaceVCMPsByVPNOTs (


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << Fn . getName ( ) << '\n' ) ; bool Modified = false ; for ( MachineLoop * ML : MLI -> getBase ( ) . getLoopsInPreorder ( ) ) { Modified |= LowerWhileLoopStart ( ML ) ; Modified |= MergeLoopEnd ( ML ) ; Modified |= ConvertTailPredLoop ( ML , DT ) ; } for ( MachineBasicBlock & MBB : Fn ) { Modified |= HintDoLoopStartReg ( MBB ) ; Modified |= ReplaceConstByVPNOTs ( MBB , DT ) ; Modified |= ReplaceVCMPsByVPNOTs ( MBB ) ; Modified |= ReduceOldVCCRValueUses ( MBB ) ; Modified |= ConvertVPSEL ( MBB ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case


<s> static bool CanHaveSwappedOperands ( unsigned Opcode ) { switch ( Opcode ) { default : return true ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case


<s> while ( ! Worklist . empty ( ) ) { MachineBasicBlock * MBB = Worklist . pop_back_val ( ) ; if ( Visited . count ( MBB ) ) continue ; for ( MachineInstr & MI : * MBB ) { if ( ! IsInvalidTPInstruction ( MI ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MachineInstrBuilder MIB = BuildMI ( * PreHeader , PreHeader -> getFirstTerminator ( ) , LoopStart -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . add ( LoopStart -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( LoopStart -> getOperand ( <NUM_LIT> ) ) ; LoopStart -> getOperand ( <NUM_LIT> ) . setIsKill ( false ) ; RevertWhileLoopStartLR ( LoopStart , TII , <STR_LIT> , true ) ; return MIB ; } Visited . insert ( MBB )


<s> Worklist . push_back ( PreHeader ) ; Visited . insert ( LoopStart -> getParent ( ) ) ; while ( ! Worklist . empty ( ) ) { MachineBasicBlock * MBB = Worklist . pop_back_val ( ) ; if ( Visited . count ( MBB ) ) continue ; for ( MachineInstr & MI : * MBB ) { if ( ! IsInvalidTPInstruction ( MI ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MachineInstrBuilder MIB = BuildMI ( * PreHeader , PreHeader -> getFirstTerminator ( ) , LoopStart -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . add ( LoopStart -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( LoopStart -> getOperand ( <NUM_LIT> ) ) ; LoopStart -> getOperand ( <NUM_LIT> ) . setIsKill ( false ) ; RevertWhileLoopStartLR ( LoopStart , TII , <STR_LIT> , true ) ; return MIB ; } Visited . insert ( MBB ) ; for ( auto * Pred : MBB -> predecessors ( ) ) Worklist . push_back ( Pred ) ; } return


<s> SmallVector < MachineInstr * , <NUM_LIT> > DeadInstructions ; for ( MachineInstr & MI : MBB . instrs ( ) ) { if ( isVCTP ( & MI ) ) { HasVCTP = true ; continue ; } if ( ! HasVCTP || MI . getOpcode ( ) != <STR_LIT> ) continue ; MachineInstrBuilder MIBuilder = BuildMI ( MBB , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( <STR_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; ( void ) MIBuilder ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; dbgs ( ) << <STR_LIT> ; MIBuilder . getInstr ( ) -> dump ( ) ) ; DeadInstructions . push_back (


<s> bool HasVCTP = false ; SmallVector < MachineInstr * , <NUM_LIT> > DeadInstructions ; for ( MachineInstr & MI : MBB . instrs ( ) ) { if ( isVCTP ( & MI ) ) { HasVCTP = true ; continue ; } if ( ! HasVCTP || MI . getOpcode ( ) != <STR_LIT> ) continue ; MachineInstrBuilder MIBuilder = BuildMI ( MBB , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( <STR_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; ( void ) MIBuilder ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; dbgs ( ) << <STR_LIT> ; MIBuilder . getInstr ( ) -> dump ( ) ) ; DeadInstructions . push_back ( & MI ) ; } for ( MachineInstr * DeadInstruction : DeadInstructions ) DeadInstruction -> eraseFromParent ( ) ; return ! DeadInstructions . empty (


<s> FunctionPass * llvm :: createMVETPAndVPTOptimisationsPass ( ) { return new MVETPAndVPTOptimisations (


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < MachineLoopInfo > ( ) ; AU . addPreserved < MachineLoopInfo > ( ) ; AU . addRequired < MachineDominatorTree > (


<s> AU . addRequired < MachineDominatorTree > ( ) ; AU . addPreserved <


<s> static <STR_LIT> GetCondCode ( MachineInstr & Instr ) { assert ( IsVCMP ( Instr . getOpcode ( ) ) && <STR_LIT> ) ; return <STR_LIT> ( Instr . getOperand ( <NUM_LIT> ) . getImm


<s> return MI . isCall ( ) || isLoopStart


<s> static bool IsVPNOTEquivalent ( MachineInstr & Cond , MachineInstr & Prev ) { assert ( IsVCMP ( Cond . getOpcode ( ) ) && IsVCMP ( Prev . getOpcode ( ) ) ) ; if ( Cond . getOpcode ( ) != Prev . getOpcode ( ) ) return false ; MachineOperand & CondOP1 = Cond . getOperand ( <NUM_LIT> ) , & CondOP2 = Cond


<s> static bool IsWritingToVCCR ( MachineInstr & Instr ) { if ( Instr . getNumOperands ( ) == <NUM_LIT> ) return false ; MachineOperand & Dst = Instr . getOperand ( <NUM_LIT> ) ; if ( ! Dst . isReg ( ) ) return false ; Register DstReg = Dst . getReg ( ) ; if ( ! DstReg . isVirtual ( )


<s> static MachineInstr * LookThroughCOPY ( MachineInstr * MI , MachineRegisterInfo * MRI ) { while ( MI && MI -> getOpcode ( ) == TargetOpcode :: COPY && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) . isVirtual ( ) ) MI = MRI -> getVRegDef ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; return


<s> static MachineInstr * LookThroughCOPY ( MachineInstr * MI , MachineRegisterInfo * MRI ) { while ( MI && MI -> getOpcode ( ) == TargetOpcode :: COPY && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) . isVirtual ( ) ) MI = MRI -> getVRegDef ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; return MI


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << ML -> getHeader ( ) -> getName ( ) << <STR_LIT> ) ; MachineInstr * LoopEnd , * LoopPhi , * LoopStart , * LoopDec ; if ( ! findLoopComponents ( ML , MRI , LoopStart , LoopPhi , LoopDec , LoopEnd ) ) return false ; if ( LoopStart -> getOpcode ( ) != <STR_LIT> ) return false ; Register LR = LoopStart -> getOperand ( <NUM_LIT> ) . getReg ( ) ; auto WLSIt = find_if ( MRI -> use_nodbg_instructions ( LR ) , [ ] ( auto & MI ) { return MI . getOpcode ( ) == <STR_LIT> ; } ) ; if ( ! MergeEndDec || WLSIt == MRI -> use_instr_nodbg_end ( ) ) { RevertWhileLoopSetup ( LoopStart , TII ) ; RevertLoopDec ( LoopStart , TII ) ; RevertLoopEnd ( LoopStart , TII ) ; return true ; } MachineInstrBuilder MI = BuildMI ( * WLSIt -> getParent ( ) , * WLSIt , WLSIt -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , LR ) . add ( LoopStart -> getOperand ( <NUM_LIT> ) ) . add ( WLSIt -> getOperand ( <NUM_LIT> ) ) ; ( void ) MI ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI . getInstr ( ) ) ; WLSIt -> eraseFromParent


<s> if ( MachineOperand * MO = Iter -> findRegisterUseOperand ( VPNOTOperand , true ) ) { VPNOTOperandKiller = MO ; } if ( Iter -> findRegisterUseOperandIdx ( Reg ) != - <NUM_LIT> ) { MustMove = true ; continue ; } if ( Iter -> findRegisterUseOperandIdx ( VPNOTResult ) == - <NUM_LIT> ) continue ; HasUser = true ; if ( ! MustMove ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; VPNOT . dump ( ) ; dbgs ( ) << <STR_LIT> ; Iter -> dump ( ) ) ; MBB . splice ( Iter , & MBB , VPNOT .


<s> -- CmpMI ; if ( CmpMI -> modifiesRegister ( <STR_LIT> , TRI ) ) break ; if ( CmpMI -> readsRegister ( <STR_LIT> , TRI ) ) break ; } if ( CmpMI == MI ) return nullptr ; NewOpcode = VCMPOpcodeToVPT ( CmpMI -> getOpcode ( ) ) ; if ( NewOpcode == <NUM_LIT> )


<s> static <STR_LIT> GetInitialBlockMask ( unsigned BlockSize ) { switch ( BlockSize ) { case <NUM_LIT> : return <STR_LIT> :: T ; case <NUM_LIT> : return <STR_LIT> :: TT ; case <NUM_LIT> : return <STR_LIT> :: TTT ; case <NUM_LIT>


<s> static <STR_LIT> GetInitialBlockMask ( unsigned BlockSize ) { switch ( BlockSize ) { case <NUM_LIT> : return <STR_LIT> ::


<s> <STR_LIT> BlockMask = CreateVPTBlock ( MBIter , EndIter , DeadInstructions ) ; MachineInstrBuilder MIBuilder ; unsigned NewOpcode ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << ( unsigned ) BlockMask << <STR_LIT> ) ; if ( MachineInstr * VCMP = findVCMPToFoldIntoVPST ( MI , TRI , NewOpcode ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; VCMP -> dump ( ) ) ; MIBuilder = BuildMI ( Block , MI , DL , TII -> get ( NewOpcode ) ) ; MIBuilder . addImm ( ( uint64_t ) BlockMask ) ; MIBuilder . add ( VCMP -> getOperand ( <NUM_LIT> ) ) ; MIBuilder . add ( VCMP -> getOperand ( <NUM_LIT> ) ) ; MIBuilder . add ( VCMP -> getOperand ( <NUM_LIT> ) ) ; for ( MachineInstr & MII : make_range ( VCMP -> getIterator ( ) , MI -> getIterator ( ) ) ) { MII . clearRegisterKills ( VCMP -> getOperand ( <NUM_LIT> ) . getReg ( ) , TRI ) ; MII . clearRegisterKills ( VCMP -> getOperand ( <NUM_LIT> ) . getReg ( ) , TRI ) ; } VCMP -> eraseFromParent ( ) ; } else { MIBuilder = BuildMI ( Block , MI , DL , TII -> get ( <STR_LIT> ) ) ; MIBuilder . addImm ( ( uint64_t ) BlockMask ) ; } for ( MachineInstr * DeadMI : DeadInstructions ) DeadMI -> eraseFromParent (