<s> unsigned IncrementIndex = Phi -> getIncomingBlock ( <NUM_LIT> ) == L -> getLoopLatch ( ) ? <NUM_LIT> : <NUM_LIT> ; Offsets = Phi -> getIncomingValue ( IncrementIndex ) ; std :: pair < Value * , int64_t > Add = getVarAndConst ( Offsets , TypeScale ) ; if ( Add . first == nullptr ) return nullptr ; Value * OffsetsIncoming = Add . first ; int64_t Immediate = Add . second ; if ( OffsetsIncoming != Phi ) return nullptr ; Builder . SetInsertPoint ( & Phi -> getIncomingBlock ( <NUM_LIT> - IncrementIndex ) -> back ( ) ) ; unsigned NumElems = cast < FixedVectorType > ( OffsetsIncoming -> getType ( ) ) -> getNumElements ( ) ; Instruction * ScaledOffsets = BinaryOperator :: Create ( Instruction :: Shl , Phi -> getIncomingValue ( <NUM_LIT> - IncrementIndex ) , Builder . CreateVectorSplat ( NumElems , Builder . getInt32 ( TypeScale ) ) , <STR_LIT> , & Phi -> getIncomingBlock ( <NUM_LIT> - IncrementIndex ) -> back ( ) ) ; OffsetsIncoming = BinaryOperator :: Create ( Instruction :: Add , ScaledOffsets , Builder . CreateVectorSplat ( NumElems , Builder . CreatePtrToInt ( BasePtr , cast < VectorType > ( ScaledOffsets -> getType ( ) ) -> getElementType ( ) ) ) , <STR_LIT> , & Phi -> getIncomingBlock ( <NUM_LIT> - IncrementIndex ) -> back ( ) ) ; OffsetsIncoming = BinaryOperator :: Create ( Instruction :: Sub , OffsetsIncoming , Builder . CreateVectorSplat ( NumElems , Builder . getInt32 ( Immediate ) ) , <STR_LIT> , & Phi -> getIncomingBlock ( <NUM_LIT> - IncrementIndex ) -> back ( ) ) ; Phi -> setIncomingValue ( <NUM_LIT> - IncrementIndex , OffsetsIncoming ) ; Builder . SetInsertPoint ( I ) ; Instruction * EndResult ; Instruction * NewInduction ; if ( I -> getIntrinsicID ( ) == <STR_LIT> ) { Value * Load = tryCreateMaskedGatherBaseWB ( I , Phi , Builder , Immediate ) ; EndResult = ExtractValueInst :: Create ( Load , <NUM_LIT> , <STR_LIT> ) ; NewInduction = ExtractValueInst :: Create ( Load , <NUM_LIT> , <STR_LIT> ) ; Builder . Insert ( EndResult ) ; Builder . Insert ( NewInduction ) ; } else { EndResult = NewInduction = tryCreateMaskedScatterBaseWB ( I , Phi , Builder , Immediate ) ; }

Instruction * AddInst = cast < Instruction > ( Offsets ) ;


<s> auto * Ty = cast < FixedVectorType > ( I -> getType ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( Ty -> getNumElements ( ) != <NUM_LIT> || Ty -> getScalarSizeInBits ( ) != <NUM_LIT> ) return nullptr ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ;

if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ty , Ptr -> getType ( ) } , { Ptr , Builder . getInt32 ( Increment ) } ) ;


<s> Instruction * MVEGatherScatterLowering :: tryCreateMaskedGatherBase ( IntrinsicInst * I , Value * Ptr , IRBuilder < > & Builder , int64_t Increment ) { using namespace PatternMatch ; auto * Ty = cast < FixedVectorType > ( I -> getType ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( Ty -> getNumElements ( ) != <NUM_LIT> || Ty -> getScalarSizeInBits ( ) != <NUM_LIT> ) return nullptr ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ;


<s> Instruction * MVEGatherScatterLowering :: tryCreateMaskedGatherBaseWB ( IntrinsicInst * I , Value * Ptr , IRBuilder < > & Builder , int64_t Increment ) { using namespace PatternMatch ; auto * Ty = cast < FixedVectorType > ( I -> getType ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; if ( Ty -> getNumElements ( ) != <NUM_LIT> || Ty -> getScalarSizeInBits ( ) != <NUM_LIT> ) return nullptr ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ;


<s> ResultTy = User -> getType ( ) ; } } if ( ResultTy -> getPrimitiveSizeInBits ( ) < <NUM_LIT> && ResultTy -> isIntOrIntVectorTy ( ) ) { ResultTy = ResultTy -> getWithNewBitWidth ( <NUM_LIT> / cast < FixedVectorType > ( ResultTy ) -> getNumElements ( ) ) ; TruncResult = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ResultTy << <STR_LIT> ) ; } if ( ResultTy -> getPrimitiveSizeInBits ( ) != <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return nullptr ; } } Value * Offsets ; int Scale ; Value * BasePtr = decomposePtr ( Ptr , Offsets , Scale , cast < FixedVectorType > ( ResultTy ) , MemoryTy , Builder ) ; if ( ! BasePtr ) return nullptr ; Root = Extend ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; Instruction * Load = nullptr ; if ( ! match ( Mask , m_One ( ) ) ) Load = Builder . CreateIntrinsic ( <STR_LIT> , { ResultTy , BasePtr -> getType ( ) , Offsets -> getType ( ) , Mask -> getType ( ) } , { BasePtr , Offsets , Builder . getInt32 ( MemoryTy -> getScalarSizeInBits ( ) ) , Builder . getInt32 ( Scale ) , Builder . getInt32 ( Unsigned ) , Mask } ) ; else Load = Builder . CreateIntrinsic ( <STR_LIT> , { ResultTy , BasePtr -> getType ( ) , Offsets -> getType ( ) } , { BasePtr , Offsets , Builder . getInt32 ( MemoryTy -> getScalarSizeInBits ( ) ) , Builder . getInt32 ( Scale ) , Builder . getInt32 ( Unsigned ) } ) ; if ( TruncResult ) { Load = TruncInst :: Create ( Instruction :: Trunc , Load , MemoryTy ) ; Builder . Insert ( Load ) ; }


<s> if ( ! ( Ty -> getNumElements ( ) == <NUM_LIT> && Ty -> getScalarSizeInBits ( ) == <NUM_LIT> ) ) { return nullptr ; } Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ptr -> getType ( ) , Input -> getType ( ) } , { Ptr , Builder . getInt32 ( Increment ) , Input } ) ;

else return Builder . CreateIntrinsic ( <STR_LIT> , { Ptr -> getType ( ) , Input -> getType ( ) , Mask -> getType ( ) } , { Ptr , Builder . getInt32 ( Increment ) , Input , Mask } ) ;


<s> auto * Ty = cast < FixedVectorType > ( Input -> getType ( ) ) ; if ( ! ( Ty -> getNumElements ( ) == <NUM_LIT> && Ty -> getScalarSizeInBits ( ) == <NUM_LIT> ) ) { return nullptr ; } Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ;

if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ptr -> getType ( ) , Input -> getType ( ) } , { Ptr , Builder . getInt32 ( Increment ) , Input } ) ;


<s> if ( Ty -> getNumElements ( ) != <NUM_LIT> || Ty -> getScalarSizeInBits ( ) != <NUM_LIT> ) return nullptr ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ptr -> getType ( ) , Input -> getType ( ) } , { Ptr , Builder . getInt32 ( Increment ) , Input } ) ; else return Builder . CreateIntrinsic ( <STR_LIT> , { Ptr -> getType ( ) , Input -> getType ( ) , Mask -> getType ( ) } , { Ptr , Builder . getInt32 ( Increment ) , Input , Mask } ) ;


<s> Instruction * MVEGatherScatterLowering :: tryCreateMaskedScatterBaseWB ( IntrinsicInst * I , Value * Ptr , IRBuilder < > & Builder , int64_t Increment ) { using namespace PatternMatch ; Value * Input = I -> getArgOperand ( <NUM_LIT> ) ; auto * Ty = cast < FixedVectorType > ( Input -> getType ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; if ( Ty -> getNumElements ( ) != <NUM_LIT> || Ty -> getScalarSizeInBits ( ) != <NUM_LIT> ) return nullptr ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ;


<s> if ( ! BasePtr ) return nullptr ; if ( ExtendInput ) Input = Builder . CreateZExt ( Input , InputTy ) ; if ( ! match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { BasePtr -> getType ( ) , Offsets -> getType ( ) , Input -> getType ( ) , Mask -> getType ( ) } , { BasePtr , Offsets , Input , Builder . getInt32 ( MemoryTy -> getScalarSizeInBits ( ) ) , Builder . getInt32 ( Scale ) , Mask } ) ;

else return Builder . CreateIntrinsic ( <STR_LIT> , { BasePtr -> getType ( ) , Offsets -> getType ( ) , Input -> getType ( ) } , { BasePtr , Offsets , Input , Builder . getInt32 ( MemoryTy -> getScalarSizeInBits ( ) ) , Builder . getInt32 ( Scale ) } ) ;


<s> GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( Address ) ; if ( ! GEP ) return false ; bool Changed = false ; if ( GEP -> hasOneUse ( ) && dyn_cast < GetElementPtrInst > ( GEP -> getPointerOperand ( ) ) ) {

IRBuilder < > Builder ( GEP -> getContext ( ) ) ;


<s> void MVEGatherScatterLowering :: pushOutMul ( PHINode * & Phi , Value * IncrementPerRound , Value * OffsSecondOperand , unsigned LoopIncrement , IRBuilder < > & Builder ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ;

Instruction * InsertionPoint = & cast < Instruction > ( Phi -> getIncomingBlock ( LoopIncrement == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ) -> back ( ) ) ;


<s> LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; SmallVector < IntrinsicInst * , <NUM_LIT> > Gathers ; SmallVector < IntrinsicInst * , <NUM_LIT> > Scatters ; bool Changed = false ; for ( BasicBlock & BB : F ) { Changed |= SimplifyInstructionsInBlock ( & BB ) ; for ( Instruction & I : BB ) { IntrinsicInst * II = dyn_cast < IntrinsicInst > ( & I ) ; if ( II && II -> getIntrinsicID ( ) == <STR_LIT> && isa < FixedVectorType > ( II -> getType ( ) ) ) { Gathers . push_back ( II ) ; Changed |= optimiseAddress ( II -> getArgOperand ( <NUM_LIT> ) , II -> getParent ( ) , LI ) ; } else if ( II && II -> getIntrinsicID ( ) == <STR_LIT> && isa < FixedVectorType > ( II -> getArgOperand ( <NUM_LIT> ) -> getType ( ) ) ) { Scatters . push_back ( II ) ; Changed |= optimiseAddress ( II -> getArgOperand ( <NUM_LIT> ) , II -> getParent ( ) , LI ) ; } }


<s> FixedVectorType * Ty ; if ( I -> getIntrinsicID ( ) == <STR_LIT> ) Ty = cast < FixedVectorType > ( I -> getType ( ) ) ; else Ty = cast < FixedVectorType > ( I -> getArgOperand ( <NUM_LIT> ) -> getType ( ) ) ; if ( Ty -> getNumElements ( ) != <NUM_LIT> || Ty -> getScalarSizeInBits ( ) != <NUM_LIT> ) return nullptr ; Loop * L = LI -> getLoopFor ( I -> getParent ( ) ) ; if ( L == nullptr ) return nullptr ; GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( Ptr ) ; Value * Offsets ; Value * BasePtr = decomposeGEP ( Offsets , Ty , GEP , Builder ) ; if ( ! BasePtr ) return nullptr ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; DataLayout DT = I -> getParent ( ) -> getParent ( ) -> getParent ( ) -> getDataLayout ( ) ; int TypeScale = computeScale ( DT . getTypeSizeInBits ( GEP -> getOperand ( <NUM_LIT> ) -> getType ( ) ) , DT . getTypeSizeInBits ( GEP -> getType ( ) ) / cast < FixedVectorType > ( GEP -> getType ( ) ) -> getNumElements ( ) ) ;

if ( TypeScale == - <NUM_LIT> ) return nullptr ;


<s> if ( L == nullptr ) return nullptr ; GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( Ptr ) ; Value * Offsets ; Value * BasePtr = decomposeGEP ( Offsets , Ty , GEP , Builder ) ; if ( ! BasePtr ) return nullptr ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; DataLayout DT = I -> getParent ( ) -> getParent ( ) -> getParent ( ) -> getDataLayout ( ) ; int TypeScale = computeScale ( DT . getTypeSizeInBits ( GEP -> getOperand ( <NUM_LIT> ) -> getType ( ) ) , DT . getTypeSizeInBits ( GEP -> getType ( ) ) / cast < FixedVectorType > ( GEP -> getType ( ) ) -> getNumElements ( ) ) ; if ( TypeScale == - <NUM_LIT> ) return nullptr ; if ( GEP -> hasOneUse ( ) ) { if ( auto * Load = tryCreateIncrementingWBGatScat ( I , BasePtr , Offsets , TypeScale , Builder ) ) return Load ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; std :: pair < Value * , int64_t > Add = getVarAndConst ( Offsets , TypeScale ) ; if ( Add . first == nullptr ) return nullptr ; Value * OffsetsIncoming = Add . first ; int64_t Immediate = Add . second ; Instruction * ScaledOffsets = BinaryOperator :: Create ( Instruction :: Shl , OffsetsIncoming , Builder . CreateVectorSplat ( Ty -> getNumElements ( ) , Builder . getInt32 ( TypeScale ) ) , <STR_LIT> , I ) ; OffsetsIncoming = BinaryOperator :: Create ( Instruction :: Add , ScaledOffsets , Builder . CreateVectorSplat ( Ty -> getNumElements ( ) , Builder . CreatePtrToInt ( BasePtr , cast < VectorType > ( ScaledOffsets -> getType ( ) ) -> getElementType ( ) ) ) , <STR_LIT> , I ) ; if ( I -> getIntrinsicID ( ) == <STR_LIT> ) return tryCreateMaskedGatherBase ( I , OffsetsIncoming , Builder , Immediate ) ; else return tryCreateMaskedScatterBase ( I , OffsetsIncoming , Builder , Immediate ) ;


<s> if ( GEPPtr -> getType ( ) -> isVectorTy ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << <STR_LIT> ) ; return nullptr ; } if ( GEP -> getNumOperands ( ) != <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return nullptr ; } Offsets = GEP -> getOperand ( <NUM_LIT> ) ; if ( ZExtInst * ZextOffs = dyn_cast < ZExtInst > ( Offsets ) ) Offsets = ZextOffs -> getOperand ( <NUM_LIT> ) ; Type * OffsType = VectorType :: getInteger ( cast < VectorType > ( Ty ) ) ; if ( OffsType != Offsets -> getType ( ) ) { if ( OffsType -> getScalarSizeInBits ( ) > Offsets -> getType ( ) -> getScalarSizeInBits ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Offsets = Builder . CreateZExt ( Offsets , OffsType , <STR_LIT> ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ;


<s> AU . addRequired < TargetPassConfig > ( ) ; FunctionPass :: getAnalysisUsage ( AU ) ;


<s> AU . setPreservesCFG ( ) ; AU . addRequired < TargetPassConfig > ( ) ;

FunctionPass :: getAnalysisUsage ( AU ) ;


<s> if ( ( ( NumElements == <NUM_LIT> && ElemSize == <NUM_LIT> ) || ( NumElements == <NUM_LIT> && ElemSize == <NUM_LIT> ) || ( NumElements == <NUM_LIT> && ElemSize == <NUM_LIT> ) ) && ElemSize / <NUM_LIT> <= Alignment ) return true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return false ;


<s> if ( auto * BitCast = dyn_cast < BitCastInst > ( Ptr ) ) { Type * BCTy = BitCast -> getType ( ) ; Type * BCSrcTy = BitCast -> getOperand ( <NUM_LIT> ) -> getType ( ) ;

if ( BCTy -> getVectorNumElements ( ) == BCSrcTy -> getVectorNumElements ( ) ) {


<s> Type * BCSrcTy = BitCast -> getOperand ( <NUM_LIT> ) -> getType ( ) ;


<s> lookThroughBitcast ( Ptr ) ; assert ( Ptr -> getType ( ) -> isVectorTy ( ) && <STR_LIT> ) ; IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ; Value * Load = tryCreateMaskedGatherOffset ( I , Ptr , Builder ) ; if ( ! Load ) Load = tryCreateMaskedGatherBase ( I , Ptr , Builder ) ; if ( ! Load ) return false ; if ( ! isa < UndefValue > ( PassThru ) && ! match ( PassThru , m_Zero ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; Load = Builder . CreateSelect ( Mask , Load , PassThru ) ; }


<s> unsigned Alignment = cast < ConstantInt > ( I -> getArgOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; Value * PassThru = I -> getArgOperand ( <NUM_LIT> ) ; if ( ! isLegalTypeAndAlignment ( Ty -> getVectorNumElements ( ) , Ty -> getScalarSizeInBits ( ) , Alignment ) ) return false ; lookThroughBitcast ( Ptr ) ; assert ( Ptr -> getType ( ) -> isVectorTy ( ) && <STR_LIT> ) ; IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ; Value * Load = tryCreateMaskedGatherOffset ( I , Ptr , Builder ) ; if ( ! Load ) Load = tryCreateMaskedGatherBase ( I , Ptr , Builder ) ; if ( ! Load ) return false ; if ( ! isa < UndefValue > ( PassThru ) && ! match ( PassThru , m_Zero ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ;

Load = Builder . CreateSelect ( Mask , Load , PassThru ) ;


<s> auto * ST = & TM . getSubtarget < <ISA_LIT>Subtarget > ( F ) ; if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; SmallVector < IntrinsicInst * , <NUM_LIT> > Gathers ; for ( BasicBlock & BB : F ) { for ( Instruction & I : BB ) { IntrinsicInst * II = dyn_cast < IntrinsicInst > ( & I ) ; if ( II && II -> getIntrinsicID ( ) == <STR_LIT> ) Gathers . push_back ( II ) ; } } if ( Gathers . empty ( ) ) return false ; for ( IntrinsicInst * I : Gathers ) lowerGather ( I ) ;


<s> for ( BasicBlock & BB : F ) { for ( Instruction & I : BB ) { IntrinsicInst * II = dyn_cast < IntrinsicInst > ( & I ) ; if ( II && II -> getIntrinsicID ( ) == <STR_LIT> ) Gathers . push_back ( II ) ; } } if ( Gathers . empty ( ) ) return false ; for ( IntrinsicInst * I : Gathers ) lowerGather ( I ) ;


<s> if ( Ty -> getVectorNumElements ( ) != <NUM_LIT> ) return nullptr ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ty , Ptr -> getType ( ) } , { Ptr , Builder . getInt32 ( <NUM_LIT> ) } ) ; else return Builder . CreateIntrinsic ( <STR_LIT> , { Ty , Ptr -> getType ( ) , Mask -> getType ( ) } , { Ptr , Builder . getInt32 ( <NUM_LIT> ) , Mask } ) ;


<s> unsigned Scale ; int GEPElemSize = BasePtr -> getType ( ) -> getPointerElementType ( ) -> getPrimitiveSizeInBits ( ) ; int ResultElemSize = Ty -> getScalarSizeInBits ( ) ; if ( GEPElemSize == <NUM_LIT> && ResultElemSize == <NUM_LIT> ) { Scale = <NUM_LIT> ; } else if ( GEPElemSize == <NUM_LIT> && ResultElemSize == <NUM_LIT> ) { Scale = <NUM_LIT> ; } else if ( GEPElemSize == <NUM_LIT> ) { Scale = <NUM_LIT> ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return nullptr ; } Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ;

if ( ! match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ty , BasePtr -> getType ( ) , Offsets -> getType ( ) , Mask -> getType ( ) } , { BasePtr , Offsets , Builder . getInt32 ( Ty -> getScalarSizeInBits ( ) ) , Builder . getInt32 ( Scale ) , Builder . getInt32 ( <NUM_LIT> ) , Mask } ) ;


<s> } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return nullptr ; } Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; if ( ! match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ty , BasePtr -> getType ( ) , Offsets -> getType ( ) , Mask -> getType ( ) } , { BasePtr , Offsets , Builder . getInt32 ( Ty -> getScalarSizeInBits ( ) ) , Builder . getInt32 ( Scale ) , Builder . getInt32 ( <NUM_LIT> ) , Mask } ) ;

else return Builder . CreateIntrinsic ( <STR_LIT> , { Ty , BasePtr -> getType ( ) , Offsets -> getType ( ) } , { BasePtr , Offsets , Builder . getInt32 ( Ty -> getScalarSizeInBits ( ) ) , Builder . getInt32 ( Scale ) , Builder . getInt32 ( <NUM_LIT> ) } ) ;


<s> return nullptr ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; Value * GEPPtr = GEP -> getPointerOperand ( ) ; if ( GEPPtr -> getType ( ) -> isVectorTy ( ) ) { return nullptr ; } if ( GEP -> getNumOperands ( ) != <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return nullptr ; } Offsets = GEP -> getOperand ( <NUM_LIT> ) ; if ( ZExtInst * ZextOffs = dyn_cast < ZExtInst > ( Offsets ) ) Offsets = ZextOffs -> getOperand ( <NUM_LIT> ) ; Type * OffsType = VectorType :: getInteger ( cast < VectorType > ( Ty ) ) ; if ( OffsType != Offsets -> getType ( ) ) { if ( OffsType -> getScalarSizeInBits ( ) > Offsets -> getType ( ) -> getScalarSizeInBits ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Offsets = Builder . CreateZExt ( Offsets , OffsType , <STR_LIT> ) ; } else {


<s> assert ( Ptr -> getType ( ) -> isVectorTy ( ) && <STR_LIT> ) ; IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ; Instruction * Root = I ; Value * Load = tryCreateMaskedGatherOffset ( I , Ptr , Root , Builder ) ; if ( ! Load ) Load = tryCreateMaskedGatherBase ( I , Ptr , Builder ) ; if ( ! Load ) return false ; if ( ! isa < UndefValue > ( PassThru ) && ! match ( PassThru , m_Zero ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; Load = Builder . CreateSelect ( Mask , Load , PassThru ) ; } Root -> replaceAllUsesWith ( Load ) ; Root -> eraseFromParent ( ) ; if ( Root != I ) I -> eraseFromParent ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ;


<s> Value * Ptr = I -> getArgOperand ( <NUM_LIT> ) ; unsigned Alignment = cast < ConstantInt > ( I -> getArgOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; Type * Ty = Input -> getType ( ) ; if ( ! isLegalTypeAndAlignment ( Ty -> getVectorNumElements ( ) , Ty -> getScalarSizeInBits ( ) , Alignment ) ) return false ; lookThroughBitcast ( Ptr ) ; assert ( Ptr -> getType ( ) -> isVectorTy ( ) && <STR_LIT> ) ; IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ; Value * Store = tryCreateMaskedScatterOffset ( I , Ptr , Builder ) ; if ( ! Store ) Store = tryCreateMaskedScatterBase ( I , Ptr , Builder ) ; if ( ! Store ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; I -> replaceAllUsesWith ( Store ) ; I -> eraseFromParent ( ) ; return true ;


<s> if ( ! EnableMaskedGatherScatters ) return false ; auto & TPC = getAnalysis < TargetPassConfig > ( ) ; auto & TM = TPC . getTM < TargetMachine > ( ) ; auto * ST = & TM . getSubtarget < <ISA_LIT>Subtarget > ( F ) ; if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; SmallVector < IntrinsicInst * , <NUM_LIT> > Gathers ; SmallVector < IntrinsicInst * , <NUM_LIT> > Scatters ; for ( BasicBlock & BB : F ) { for ( Instruction & I : BB ) { IntrinsicInst * II = dyn_cast < IntrinsicInst > ( & I ) ; if ( II && II -> getIntrinsicID ( ) == <STR_LIT> ) Gathers . push_back ( II ) ; else if ( II && II -> getIntrinsicID ( ) == <STR_LIT> ) Scatters . push_back ( II ) ; }


<s> bool MVEGatherScatterLowering :: runOnFunction ( Function & F ) { if ( ! EnableMaskedGatherScatters ) return false ; auto & TPC = getAnalysis < TargetPassConfig > ( ) ; auto & TM = TPC . getTM < TargetMachine > ( ) ; auto * ST = & TM . getSubtarget < <ISA_LIT>Subtarget > ( F ) ; if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; SmallVector < IntrinsicInst * , <NUM_LIT> > Gathers ;

SmallVector < IntrinsicInst * , <NUM_LIT> > Scatters ;


<s> if ( Scale == - <NUM_LIT> ) return nullptr ; Root = Extend ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; if ( ! match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { ResultTy , BasePtr -> getType ( ) , Offsets -> getType ( ) , Mask -> getType ( ) } , { BasePtr , Offsets , Builder . getInt32 ( OriginalTy -> getScalarSizeInBits ( ) ) , Builder . getInt32 ( Scale ) , Builder . getInt32 ( Unsigned ) , Mask } ) ; else return Builder . CreateIntrinsic ( <STR_LIT> , { ResultTy , BasePtr -> getType ( ) , Offsets -> getType ( ) } , { BasePtr , Offsets , Builder . getInt32 ( OriginalTy -> getScalarSizeInBits ( ) ) , Builder . getInt32 ( Scale ) , Builder . getInt32 ( Unsigned ) } ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return nullptr ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; ResultTy = Extend -> getType ( ) ; if ( ResultTy -> getPrimitiveSizeInBits ( ) != <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return nullptr ; } } Value * Offsets ; Value * BasePtr = checkGEP ( Offsets , ResultTy , Ptr , Builder ) ; if ( ! BasePtr ) return nullptr ; int Scale = computeScale ( BasePtr -> getType ( ) -> getPointerElementType ( ) -> getPrimitiveSizeInBits ( ) , OriginalTy -> getScalarSizeInBits ( ) ) ; if ( Scale == - <NUM_LIT> ) return nullptr ; Root = Extend ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; if ( ! match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { ResultTy , BasePtr -> getType ( ) , Offsets -> getType ( ) , Mask -> getType ( ) } , { BasePtr , Offsets , Builder . getInt32 ( OriginalTy -> getScalarSizeInBits ( ) ) , Builder . getInt32 ( Scale ) , Builder . getInt32 ( Unsigned ) , Mask } ) ;

else return Builder . CreateIntrinsic ( <STR_LIT> , { ResultTy , BasePtr -> getType ( ) , Offsets -> getType ( ) } , { BasePtr , Offsets , Builder . getInt32 ( OriginalTy -> getScalarSizeInBits ( ) ) , Builder . getInt32 ( Scale ) , Builder . getInt32 ( Unsigned ) } ) ;


<s> using namespace PatternMatch ; Value * Input = I -> getArgOperand ( <NUM_LIT> ) ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; Type * Ty = Input -> getType ( ) ; if ( ! ( Ty -> getVectorNumElements ( ) == <NUM_LIT> && Ty -> getScalarSizeInBits ( ) == <NUM_LIT> ) ) { return nullptr ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ;

if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ptr -> getType ( ) , Input -> getType ( ) } , { Ptr , Builder . getInt32 ( <NUM_LIT> ) , Input } ) ;


<s> Value * Input = I -> getArgOperand ( <NUM_LIT> ) ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; Type * Ty = Input -> getType ( ) ; if ( ! ( Ty -> getVectorNumElements ( ) == <NUM_LIT> && Ty -> getScalarSizeInBits ( ) == <NUM_LIT> ) ) { return nullptr ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ptr -> getType ( ) , Input -> getType ( ) } , { Ptr , Builder . getInt32 ( <NUM_LIT> ) , Input } ) ;

else return Builder . CreateIntrinsic ( <STR_LIT> , { Ptr -> getType ( ) , Input -> getType ( ) , Mask -> getType ( ) } , { Ptr , Builder . getInt32 ( <NUM_LIT> ) , Input , Mask } ) ;


<s> Value * MVEGatherScatterLowering :: tryCreateMaskedScatterOffset ( IntrinsicInst * I , Value * Ptr , IRBuilder < > Builder ) { using namespace PatternMatch ; Value * Input = I -> getArgOperand ( <NUM_LIT> ) ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; Type * InputTy = Input -> getType ( ) ; Type * MemoryTy = InputTy ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; if ( TruncInst * Trunc = dyn_cast < TruncInst > ( Input ) ) { Value * PreTrunc = Trunc -> getOperand ( <NUM_LIT> ) ; Type * PreTruncTy = PreTrunc -> getType ( ) ; if ( PreTruncTy -> getPrimitiveSizeInBits ( ) == <NUM_LIT> ) { Input = PreTrunc ;


<s> Type * MemoryTy = InputTy ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; if ( TruncInst * Trunc = dyn_cast < TruncInst > ( Input ) ) { Value * PreTrunc = Trunc -> getOperand ( <NUM_LIT> ) ; Type * PreTruncTy = PreTrunc -> getType ( ) ; if ( PreTruncTy -> getPrimitiveSizeInBits ( ) == <NUM_LIT> ) { Input = PreTrunc ; InputTy = PreTruncTy ; } } if ( InputTy -> getPrimitiveSizeInBits ( ) != <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return nullptr ; } Value * Offsets ; Value * BasePtr = checkGEP ( Offsets , InputTy , Ptr , Builder ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return nullptr ; } Offsets = GEP -> getOperand ( <NUM_LIT> ) ; assert ( Ty -> getVectorNumElements ( ) == Offsets -> getType ( ) -> getVectorNumElements ( ) ) ; if ( Offsets -> getType ( ) -> getScalarSizeInBits ( ) != <NUM_LIT> ) return nullptr ; if ( ZExtInst * ZextOffs = dyn_cast < ZExtInst > ( Offsets ) ) Offsets = ZextOffs -> getOperand ( <NUM_LIT> ) ; else if ( ! ( Offsets -> getType ( ) -> getVectorNumElements ( ) == <NUM_LIT> && Offsets -> getType ( ) -> getScalarSizeInBits ( ) == <NUM_LIT> ) ) return nullptr ; if ( Ty != Offsets -> getType ( ) ) { if ( ( Ty -> getScalarSizeInBits ( ) < Offsets -> getType ( ) -> getScalarSizeInBits ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ;


<s> bool isGatherScatter ( IntrinsicInst * IntInst ) { if ( IntInst == nullptr ) return false ; unsigned IntrinsicID = IntInst -> getIntrinsicID ( ) ; return ( IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> ) ;


<s> if ( ( ( NumElements == <NUM_LIT> && ( ElemSize == <NUM_LIT> || ElemSize == <NUM_LIT> || ElemSize == <NUM_LIT> ) ) || ( NumElements == <NUM_LIT> && ( ElemSize == <NUM_LIT> || ElemSize == <NUM_LIT> ) ) || ( NumElements == <NUM_LIT> && ElemSize == <NUM_LIT> ) ) && ElemSize / <NUM_LIT> <= Alignment ) return true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ;


<s> bool MVEGatherScatterLowering :: isLegalTypeAndAlignment ( unsigned NumElements , unsigned ElemSize , unsigned Alignment ) {

if ( ( ( NumElements == <NUM_LIT> && ( ElemSize == <NUM_LIT> || ElemSize == <NUM_LIT> || ElemSize == <NUM_LIT> ) ) || ( NumElements == <NUM_LIT> && ( ElemSize == <NUM_LIT> || ElemSize == <NUM_LIT> ) ) || ( NumElements == <NUM_LIT> && ElemSize == <NUM_LIT> ) ) && ElemSize / <NUM_LIT> <= Alignment ) return true ;


<s> void MVEGatherScatterLowering :: lookThroughBitcast ( Value * & Ptr ) { if ( auto * BitCast = dyn_cast < BitCastInst > ( Ptr ) ) { Type * BCTy = BitCast -> getType ( ) ; Type * BCSrcTy = BitCast -> getOperand ( <NUM_LIT> ) -> getType ( ) ; if ( BCTy -> getVectorNumElements ( ) == BCSrcTy -> getVectorNumElements ( ) ) {


<s> Type * BCTy = BitCast -> getType ( ) ; Type * BCSrcTy = BitCast -> getOperand ( <NUM_LIT> ) -> getType ( ) ;


<s> Instruction * Root = I ; Value * Load = tryCreateMaskedGatherOffset ( I , Ptr , Root , Builder ) ; if ( ! Load ) Load = tryCreateMaskedGatherBase ( I , Ptr , Builder ) ; if ( ! Load ) return nullptr ; if ( ! isa < UndefValue > ( PassThru ) && ! match ( PassThru , m_Zero ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ;


<s> IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ; Value * Store = tryCreateMaskedScatterOffset ( I , Ptr , Builder ) ; if ( ! Store ) Store = tryCreateMaskedScatterBase ( I , Ptr , Builder ) ; if ( ! Store ) return nullptr ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Value * Input = I -> getArgOperand ( <NUM_LIT> ) ; Value * Ptr = I -> getArgOperand ( <NUM_LIT> ) ; unsigned Alignment = cast < ConstantInt > ( I -> getArgOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; Type * Ty = Input -> getType ( ) ;

if ( ! isLegalTypeAndAlignment ( Ty -> getVectorNumElements ( ) , Ty -> getScalarSizeInBits ( ) , Alignment ) ) return nullptr ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Instruction * InsertionPoint ; if ( isa < Instruction > ( OffsSecondOperand ) ) InsertionPoint = & cast < Instruction > ( OffsSecondOperand ) -> getParent ( ) -> back ( ) ; else InsertionPoint = & cast < Instruction > ( Phi -> getIncomingBlock ( StartIndex ) -> back ( ) ) ;

Instruction * NewIndex = BinaryOperator :: Create ( Instruction :: Add , Phi -> getIncomingValue ( StartIndex ) , OffsSecondOperand , <STR_LIT> , InsertionPoint ) ;


<s> Instruction * InsertionPoint ; if ( isa < Instruction > ( OffsSecondOperand ) ) InsertionPoint = & cast < Instruction > ( OffsSecondOperand ) -> getParent ( ) -> back ( ) ; else InsertionPoint = & cast < Instruction > ( Phi -> getIncomingBlock ( LoopIncrement == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ) -> back ( ) ) ; Value * StartIndex = BinaryOperator :: Create ( Instruction :: Mul , Phi -> getIncomingValue ( LoopIncrement == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ) , OffsSecondOperand , <STR_LIT> , InsertionPoint ) ; Instruction * Product = BinaryOperator :: Create ( Instruction :: Mul , IncrementPerRound , OffsSecondOperand , <STR_LIT> , InsertionPoint ) ; Instruction * NewIncrement = BinaryOperator :: Create ( Instruction :: Add , Phi , Product , <STR_LIT> , cast < Instruction > ( Phi -> getIncomingBlock ( LoopIncrement ) -> back ( ) ) . getPrevNode ( ) ) ; Phi -> addIncoming ( StartIndex , Phi -> getIncomingBlock ( LoopIncrement == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ) ) ; Phi -> addIncoming ( NewIncrement , Phi -> getIncomingBlock ( LoopIncrement ) ) ; Phi -> removeIncomingValue ( ( unsigned ) <NUM_LIT> ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Instruction * InsertionPoint ; if ( isa < Instruction > ( OffsSecondOperand ) ) InsertionPoint = & cast < Instruction > ( OffsSecondOperand ) -> getParent ( ) -> back ( ) ; else InsertionPoint = & cast < Instruction > ( Phi -> getIncomingBlock ( LoopIncrement == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ) -> back ( ) ) ;

Value * StartIndex = BinaryOperator :: Create ( Instruction :: Mul , Phi -> getIncomingValue ( LoopIncrement == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ) , OffsSecondOperand , <STR_LIT> , InsertionPoint ) ;


<s> else if ( II && II -> getIntrinsicID ( ) == <STR_LIT> ) Scatters . push_back ( II ) ; } } bool Changed = false ; for ( unsigned i = <NUM_LIT> ; i < Gathers . size ( ) ; i ++ ) { IntrinsicInst * I = Gathers [ i ] ; if ( isa < GetElementPtrInst > ( I -> getArgOperand ( <NUM_LIT> ) ) ) optimiseOffsets ( cast < Instruction > ( I -> getArgOperand ( <NUM_LIT> ) ) -> getOperand ( <NUM_LIT> ) , I -> getParent ( ) , & LI ) ; Value * L = lowerGather ( I ) ; if ( L == nullptr ) continue ; SimplifyInstructionsInBlock ( cast < Instruction > ( L ) -> getParent ( ) ) ; Changed = true ; } for ( unsigned i = <NUM_LIT> ; i < Scatters . size ( ) ; i ++ ) { IntrinsicInst * I = Scatters [ i ] ; if ( isa < GetElementPtrInst > ( I -> getArgOperand ( <NUM_LIT> ) ) ) optimiseOffsets ( cast < Instruction > ( I -> getArgOperand ( <NUM_LIT> ) ) -> getOperand ( <NUM_LIT> ) , I -> getParent ( ) , & LI ) ; Value * S = lowerScatter ( I ) ; if ( S == nullptr ) continue ; SimplifyInstructionsInBlock ( cast < Instruction > ( S ) -> getParent ( ) ) ;


<s> } } Value * Offsets ; Value * BasePtr = checkGEP ( Offsets , ResultTy , Ptr , Builder ) ; if ( ! BasePtr ) return nullptr ; int Scale = computeScale ( BasePtr -> getType ( ) -> getPointerElementType ( ) -> getPrimitiveSizeInBits ( ) , OriginalTy -> getScalarSizeInBits ( ) ) ; if ( Scale == - <NUM_LIT> ) return nullptr ; Root = Extend ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; if ( ! match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { ResultTy , BasePtr -> getType ( ) , Offsets -> getType ( ) , Mask -> getType ( ) } , { BasePtr , Offsets , Builder . getInt32 ( OriginalTy -> getScalarSizeInBits ( ) ) , Builder . getInt32 ( Scale ) , Builder . getInt32 ( Unsigned ) , Mask } ) ; else return Builder . CreateIntrinsic ( <STR_LIT> , { ResultTy , BasePtr -> getType ( ) , Offsets -> getType ( ) } , { BasePtr , Offsets , Builder . getInt32 ( OriginalTy -> getScalarSizeInBits ( ) ) , Builder . getInt32 ( Scale ) , Builder . getInt32 ( Unsigned ) } ) ;


<s> Value * MVEGatherScatterLowering :: tryCreateMaskedScatterBase ( IntrinsicInst * I , Value * Ptr , IRBuilder < > & Builder ) { using namespace PatternMatch ; Value * Input = I -> getArgOperand ( <NUM_LIT> ) ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; Type * Ty = Input -> getType ( ) ; if ( ! ( Ty -> getVectorNumElements ( ) == <NUM_LIT> && Ty -> getScalarSizeInBits ( ) == <NUM_LIT> ) ) { return nullptr ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ;


<s> } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ptr -> getType ( ) , Input -> getType ( ) } , { Ptr , Builder . getInt32 ( <NUM_LIT> ) , Input } ) ;


<s> int Scale = computeScale ( BasePtr -> getType ( ) -> getPointerElementType ( ) -> getPrimitiveSizeInBits ( ) , MemoryTy -> getScalarSizeInBits ( ) ) ; if ( Scale == - <NUM_LIT> ) return nullptr ; if ( ! match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { BasePtr -> getType ( ) , Offsets -> getType ( ) , Input -> getType ( ) , Mask -> getType ( ) } , { BasePtr , Offsets , Input , Builder . getInt32 ( MemoryTy -> getScalarSizeInBits ( ) ) , Builder . getInt32 ( Scale ) , Mask } ) ;

else return Builder . CreateIntrinsic ( <STR_LIT> , { BasePtr -> getType ( ) , Offsets -> getType ( ) , Input -> getType ( ) } , { BasePtr , Offsets , Input , Builder . getInt32 ( MemoryTy -> getScalarSizeInBits ( ) ) , Builder . getInt32 ( Scale ) } ) ;


<s> } if ( GEP -> getNumOperands ( ) != <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return nullptr ; } Offsets = GEP -> getOperand ( <NUM_LIT> ) ; assert ( cast < FixedVectorType > ( Ty ) -> getNumElements ( ) == cast < FixedVectorType > ( Offsets -> getType ( ) ) -> getNumElements ( ) ) ; if ( Offsets -> getType ( ) -> getScalarSizeInBits ( ) != <NUM_LIT> ) return nullptr ; if ( ZExtInst * ZextOffs = dyn_cast < ZExtInst > ( Offsets ) ) Offsets = ZextOffs -> getOperand ( <NUM_LIT> ) ; else if ( ! ( cast < FixedVectorType > ( Offsets -> getType ( ) ) -> getNumElements ( ) == <NUM_LIT> && Offsets -> getType ( ) -> getScalarSizeInBits ( ) == <NUM_LIT> ) ) return nullptr ;


<s> if ( ZExtInst * ZextOffs = dyn_cast < ZExtInst > ( Offsets ) ) Offsets = ZextOffs -> getOperand ( <NUM_LIT> ) ; else if ( ! ( cast < FixedVectorType > ( Offsets -> getType ( ) ) -> getNumElements ( ) == <NUM_LIT> && Offsets -> getType ( ) -> getScalarSizeInBits ( ) == <NUM_LIT> ) ) return nullptr ; if ( Ty != Offsets -> getType ( ) ) { if ( ( Ty -> getScalarSizeInBits ( ) < Offsets -> getType ( ) -> getScalarSizeInBits ( ) ) ) {


<s> if ( ! BasePtr ) return nullptr ; Value * Load = tryCreateIncrementingGatScat ( I , BasePtr , Offsets , GEP , Builder ) ; if ( Load ) return Load ; int Scale = computeScale ( BasePtr -> getType ( ) -> getPointerElementType ( ) -> getPrimitiveSizeInBits ( ) , OriginalTy -> getScalarSizeInBits ( ) ) ; if ( Scale == - <NUM_LIT> ) return nullptr ; Root = Extend ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; if ( ! match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { ResultTy , BasePtr -> getType ( ) , Offsets -> getType ( ) , Mask -> getType ( ) } , { BasePtr , Offsets , Builder . getInt32 ( OriginalTy -> getScalarSizeInBits ( ) ) , Builder . getInt32 ( Scale ) , Builder . getInt32 ( Unsigned ) , Mask } ) ; else return Builder . CreateIntrinsic ( <STR_LIT> , { ResultTy , BasePtr -> getType ( ) , Offsets -> getType ( ) } , { BasePtr , Offsets , Builder . getInt32 ( OriginalTy -> getScalarSizeInBits ( ) ) , Builder . getInt32 ( Scale ) , Builder . getInt32 ( Unsigned ) } ) ;


<s> if ( TruncInst * Trunc = dyn_cast < TruncInst > ( Input ) ) { Value * PreTrunc = Trunc -> getOperand ( <NUM_LIT> ) ; Type * PreTruncTy = PreTrunc -> getType ( ) ; if ( PreTruncTy -> getPrimitiveSizeInBits ( ) == <NUM_LIT> ) { Input = PreTrunc ; InputTy = PreTruncTy ; } } if ( InputTy -> getPrimitiveSizeInBits ( ) != <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return nullptr ; } GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( Ptr ) ; Value * Offsets ; Value * BasePtr = checkGEP ( Offsets , InputTy , GEP , Builder ) ; if ( ! BasePtr ) return nullptr ; Value * Store = tryCreateIncrementingGatScat ( I , BasePtr , Offsets , GEP , Builder ) ; if ( Store ) return Store ; int Scale = computeScale ( BasePtr -> getType ( ) -> getPointerElementType ( ) -> getPrimitiveSizeInBits ( ) , MemoryTy -> getScalarSizeInBits ( ) ) ; if ( Scale == - <NUM_LIT> ) return nullptr ; if ( ! match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { BasePtr -> getType ( ) , Offsets -> getType ( ) , Input -> getType ( ) , Mask -> getType ( ) } , { BasePtr , Offsets , Input , Builder . getInt32 ( MemoryTy -> getScalarSizeInBits ( ) ) , Builder . getInt32 ( Scale ) , Mask } ) ; else return Builder . CreateIntrinsic ( <STR_LIT> , { BasePtr -> getType ( ) , Offsets -> getType ( ) , Input -> getType ( ) } , { BasePtr , Offsets , Input , Builder . getInt32 ( MemoryTy -> getScalarSizeInBits ( ) ) , Builder . getInt32 ( Scale ) } ) ;


<s> Value * PreTrunc = Trunc -> getOperand ( <NUM_LIT> ) ; Type * PreTruncTy = PreTrunc -> getType ( ) ; if ( PreTruncTy -> getPrimitiveSizeInBits ( ) == <NUM_LIT> ) { Input = PreTrunc ; InputTy = PreTruncTy ; } } if ( InputTy -> getPrimitiveSizeInBits ( ) != <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return nullptr ; } GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( Ptr ) ; Value * Offsets ; Value * BasePtr = checkGEP ( Offsets , InputTy , GEP , Builder ) ; if ( ! BasePtr ) return nullptr ; Value * Store = tryCreateIncrementingGatScat ( I , BasePtr , Offsets , GEP , Builder ) ; if ( Store ) return Store ; int Scale = computeScale ( BasePtr -> getType ( ) -> getPointerElementType ( ) -> getPrimitiveSizeInBits ( ) , MemoryTy -> getScalarSizeInBits ( ) ) ;

if ( Scale == - <NUM_LIT> ) return nullptr ;


<s> Value * PassThru = I -> getArgOperand ( <NUM_LIT> ) ; if ( ! isLegalTypeAndAlignment ( Ty -> getNumElements ( ) , Ty -> getScalarSizeInBits ( ) , Alignment ) ) return nullptr ; lookThroughBitcast ( Ptr ) ; assert ( Ptr -> getType ( ) -> isVectorTy ( ) && <STR_LIT> ) ; IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ; Instruction * Root = I ; Value * Load = tryCreateMaskedGatherOffset ( I , Ptr , Root , Builder ) ; if ( ! Load ) Load = tryCreateMaskedGatherBase ( I , Ptr , Builder ) ; if ( ! Load ) return nullptr ; if ( ! isa < UndefValue > ( PassThru ) && ! match ( PassThru , m_Zero ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; Load = Builder . CreateSelect ( Mask , Load , PassThru ) ; } Root -> replaceAllUsesWith ( Load ) ; Root -> eraseFromParent ( ) ; if ( Root != I ) I -> eraseFromParent ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Load << <STR_LIT> ) ; return Load ;


<s> IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ; Instruction * Root = I ; Value * Load = tryCreateMaskedGatherOffset ( I , Ptr , Root , Builder ) ; if ( ! Load ) Load = tryCreateMaskedGatherBase ( I , Ptr , Builder ) ; if ( ! Load ) return nullptr ; if ( ! isa < UndefValue > ( PassThru ) && ! match ( PassThru , m_Zero ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ;


<s> return false ; } else { if ( isa < PHINode > ( Offs -> getOperand ( <NUM_LIT> ) ) ) { Phi = cast < PHINode > ( Offs -> getOperand ( <NUM_LIT> ) ) ; OffsSecondOp = <NUM_LIT> ; } else if ( isa < PHINode > ( Offs -> getOperand ( <NUM_LIT> ) ) ) { Phi = cast < PHINode > ( Offs -> getOperand ( <NUM_LIT> ) ) ; OffsSecondOp = <NUM_LIT> ; } else { return false ; } } } if ( Phi -> getParent ( ) != L -> getHeader ( ) || Phi -> getNumIncomingValues ( ) != <NUM_LIT> ) return false ; int IncrementingBlock = - <NUM_LIT> ; for ( int i = <NUM_LIT> ; i < <NUM_LIT> ; i ++ ) if ( auto * Op = dyn_cast < Instruction > ( Phi -> getIncomingValue ( i ) ) ) if ( Op -> getOpcode ( ) == Instruction :: Add && ( Op -> getOperand ( <NUM_LIT> ) == Phi || Op -> getOperand ( <NUM_LIT> ) == Phi ) ) IncrementingBlock = i ; if ( IncrementingBlock == - <NUM_LIT> ) return false ; Instruction * IncInstruction = cast < Instruction > ( Phi -> getIncomingValue ( IncrementingBlock ) ) ; PHINode * NewPhi ; Value * IncrementPerRound = IncInstruction -> getOperand ( ( IncInstruction -> getOperand ( <NUM_LIT> ) == Phi ) ? <NUM_LIT> : <NUM_LIT> ) ; Value * OffsSecondOperand = Offs -> getOperand ( OffsSecondOp ) ; if ( IncrementPerRound -> getType ( ) != OffsSecondOperand -> getType ( ) ) return false ; if ( ! isa < Constant > ( IncrementPerRound ) && ! ( isa < Instruction > ( IncrementPerRound ) && ! L -> contains ( cast < Instruction > ( IncrementPerRound ) ) ) ) return false ; if ( Phi -> getNumUses ( ) == <NUM_LIT> ) { if ( IncInstruction -> getNumUses ( ) != <NUM_LIT> ) { IncInstruction = BinaryOperator :: Create ( Instruction :: BinaryOps ( IncInstruction -> getOpcode ( ) ) , Phi , IncrementPerRound , <STR_LIT> , IncInstruction ) ; Phi -> setIncomingValue ( IncrementingBlock , IncInstruction ) ; } NewPhi = Phi ; } else { NewPhi = PHINode :: Create ( Phi -> getType ( ) , <NUM_LIT> , <STR_LIT> , Phi ) ; std :: vector < Value * > Increases ; NewPhi -> addIncoming ( Phi -> getIncomingValue ( IncrementingBlock == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ) , Phi -> getIncomingBlock ( IncrementingBlock == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ) ) ; IncInstruction = BinaryOperator :: Create ( Instruction :: BinaryOps ( IncInstruction -> getOpcode ( ) ) , NewPhi , IncrementPerRound , <STR_LIT> , IncInstruction ) ; NewPhi -> addIncoming ( IncInstruction , Phi -> getIncomingBlock ( IncrementingBlock ) ) ; IncrementingBlock = <NUM_LIT> ; } IRBuilder < > Builder ( BB -> getContext ( ) ) ; Builder . SetInsertPoint ( Phi ) ; Builder . SetCurrentDebugLocation ( Offs -> getDebugLoc ( ) ) ;


<s> using namespace PatternMatch ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; auto * Ty = cast < FixedVectorType > ( I -> getType ( ) ) ; Value * Ptr = I -> getArgOperand ( <NUM_LIT> ) ; unsigned Alignment = cast < ConstantInt > ( I -> getArgOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; Value * PassThru = I -> getArgOperand ( <NUM_LIT> ) ; if ( ! isLegalTypeAndAlignment ( Ty -> getNumElements ( ) , Ty -> getScalarSizeInBits ( ) , Alignment ) ) return nullptr ; lookThroughBitcast ( Ptr ) ; assert ( Ptr -> getType ( ) -> isVectorTy ( ) && <STR_LIT> ) ; IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ; Instruction * Root = I ; Value * Load = tryCreateMaskedGatherOffset ( I , Ptr , Root , Builder ) ; if ( ! Load ) Load = tryCreateMaskedGatherBase ( I , Ptr , Builder ) ; if ( ! Load ) return nullptr ; if ( ! isa < UndefValue > ( PassThru ) && ! match ( PassThru , m_Zero ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; Load = Builder . CreateSelect ( Mask , Load , PassThru ) ; }


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Value * Input = I -> getArgOperand ( <NUM_LIT> ) ; Value * Ptr = I -> getArgOperand ( <NUM_LIT> ) ; unsigned Alignment = cast < ConstantInt > ( I -> getArgOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; auto * Ty = cast < FixedVectorType > ( Input -> getType ( ) ) ; if ( ! isLegalTypeAndAlignment ( Ty -> getNumElements ( ) , Ty -> getScalarSizeInBits ( ) , Alignment ) ) return nullptr ; lookThroughBitcast ( Ptr ) ; assert ( Ptr -> getType ( ) -> isVectorTy ( ) && <STR_LIT> ) ; IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ; Value * Store = tryCreateMaskedScatterOffset ( I , Ptr , Builder ) ; if ( ! Store ) Store = tryCreateMaskedScatterBase ( I , Ptr , Builder ) ; if ( ! Store ) return nullptr ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ;


<s> Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ; Value * Store = tryCreateMaskedScatterOffset ( I , Ptr , Builder ) ; if ( ! Store ) Store = tryCreateMaskedScatterBase ( I , Ptr , Builder ) ; if ( ! Store ) return nullptr ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; I -> eraseFromParent ( ) ; return Store ;


<s> auto & TM = TPC . getTM < TargetMachine > ( ) ; auto * ST = & TM . getSubtarget < <ISA_LIT>Subtarget > ( F ) ; if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; SmallVector < IntrinsicInst * , <NUM_LIT> > Gathers ; SmallVector < IntrinsicInst * , <NUM_LIT> > Scatters ; for ( BasicBlock & BB : F ) { for ( Instruction & I : BB ) { IntrinsicInst * II = dyn_cast < IntrinsicInst > ( & I ) ; if ( II && II -> getIntrinsicID ( ) == <STR_LIT> ) { Gathers . push_back ( II ) ; if ( isa < GetElementPtrInst > ( II -> getArgOperand ( <NUM_LIT> ) ) ) optimiseOffsets ( cast < Instruction > ( II -> getArgOperand ( <NUM_LIT> ) ) -> getOperand ( <NUM_LIT> ) , II -> getParent ( ) , LI ) ; } else if ( II && II -> getIntrinsicID ( ) == <STR_LIT> ) { Scatters . push_back ( II ) ; if ( isa < GetElementPtrInst > ( II -> getArgOperand ( <NUM_LIT> ) ) ) optimiseOffsets ( cast < Instruction > ( II -> getArgOperand ( <NUM_LIT> ) ) -> getOperand ( <NUM_LIT> ) , II -> getParent ( ) , LI ) ; } } } bool Changed = false ; for ( unsigned i = <NUM_LIT> ; i < Gathers . size ( ) ; i ++ ) { IntrinsicInst * I = Gathers [ i ] ; Value * L = lowerGather ( I ) ; if ( L == nullptr ) continue ; SimplifyInstructionsInBlock ( cast < Instruction > ( L ) -> getParent ( ) ) ; Changed = true ; } for ( unsigned i = <NUM_LIT> ; i < Scatters . size ( ) ; i ++ ) { IntrinsicInst * I = Scatters [ i ] ; Value * S = lowerScatter ( I ) ; if ( S == nullptr ) continue ; SimplifyInstructionsInBlock ( cast < Instruction > ( S ) -> getParent ( ) ) ; Changed = true ; }


<s> if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; SmallVector < IntrinsicInst * , <NUM_LIT> > Gathers ; SmallVector < IntrinsicInst * , <NUM_LIT> > Scatters ; for ( BasicBlock & BB : F ) { for ( Instruction & I : BB ) { IntrinsicInst * II = dyn_cast < IntrinsicInst > ( & I ) ; if ( II && II -> getIntrinsicID ( ) == <STR_LIT> ) { Gathers . push_back ( II ) ; if ( isa < GetElementPtrInst > ( II -> getArgOperand ( <NUM_LIT> ) ) ) optimiseOffsets ( cast < Instruction > ( II -> getArgOperand ( <NUM_LIT> ) ) -> getOperand ( <NUM_LIT> ) , II -> getParent ( ) , LI ) ; } else if ( II && II -> getIntrinsicID ( ) == <STR_LIT> ) { Scatters . push_back ( II ) ; if ( isa < GetElementPtrInst > ( II -> getArgOperand ( <NUM_LIT> ) ) ) optimiseOffsets ( cast < Instruction > ( II -> getArgOperand ( <NUM_LIT> ) ) -> getOperand ( <NUM_LIT> ) , II -> getParent ( ) , LI ) ;


<s> Value * MVEGatherScatterLowering :: tryCreateMaskedScatterBase ( IntrinsicInst * I , Value * Ptr , IRBuilder < > & Builder , int64_t Increment ) { using namespace PatternMatch ; Value * Input = I -> getArgOperand ( <NUM_LIT> ) ; auto * Ty = cast < VectorType > ( Input -> getType ( ) ) ; if ( ! ( Ty -> getNumElements ( ) == <NUM_LIT> && Ty -> getScalarSizeInBits ( ) == <NUM_LIT> ) ) {


<s> Value * MVEGatherScatterLowering :: tryCreateMaskedScatterBase ( IntrinsicInst * I , Value * Ptr , IRBuilder < > & Builder , int64_t Increment ) { using namespace PatternMatch ; Value * Input = I -> getArgOperand ( <NUM_LIT> ) ; auto * Ty = cast < VectorType > ( Input -> getType ( ) ) ; if ( ! ( Ty -> getNumElements ( ) == <NUM_LIT> && Ty -> getScalarSizeInBits ( ) == <NUM_LIT> ) ) { return nullptr ; }

Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ;


<s> static Value * CheckAndCreateOffsetAdd ( Value * X , Value * Y , Value * GEP , IRBuilder < > & Builder ) { auto FixSummands = [ & Builder ] ( FixedVectorType * & VT , Value * & NonVectorVal ) { ConstantInt * Const ; if ( ( Const = dyn_cast < ConstantInt > ( NonVectorVal ) ) && VT -> getElementType ( ) != NonVectorVal -> getType ( ) ) { unsigned TargetElemSize = VT -> getElementType ( ) -> getPrimitiveSizeInBits ( ) ; uint64_t N = Const -> getZExtValue ( ) ; if ( N < ( unsigned ) ( <NUM_LIT> << ( TargetElemSize - <NUM_LIT> ) ) ) { NonVectorVal = Builder . CreateVectorSplat ( VT -> getNumElements ( ) , Builder . getIntN ( TargetElemSize , N ) ) ; return ; } } NonVectorVal = Builder . CreateVectorSplat ( VT -> getNumElements ( ) , NonVectorVal ) ; } ; FixedVectorType * XElType = dyn_cast < FixedVectorType > ( X -> getType ( ) ) ; FixedVectorType * YElType = dyn_cast < FixedVectorType > ( Y -> getType ( ) ) ; if ( XElType && ! YElType ) { FixSummands ( XElType , Y ) ; YElType = cast < FixedVectorType > ( Y -> getType ( ) ) ; } else if ( YElType && ! XElType ) { FixSummands ( YElType , X ) ; XElType = cast < FixedVectorType > ( X -> getType ( ) ) ; } if ( XElType != YElType ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return nullptr ; } if ( XElType -> getElementType ( ) -> getScalarSizeInBits ( ) != <NUM_LIT> ) { Constant * ConstX = dyn_cast < Constant > ( X ) ; Constant * ConstY = dyn_cast < Constant > ( Y ) ; if ( ! ConstX || ! ConstY ) return nullptr ; unsigned TargetElemSize = <NUM_LIT> / XElType -> getNumElements ( ) ;

for ( unsigned i = <NUM_LIT> ; i < XElType -> getNumElements ( ) ; i ++ ) {


<s> Type * Ty = I -> getType ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( Ty -> getVectorNumElements ( ) != <NUM_LIT> || Ty -> getScalarSizeInBits ( ) != <NUM_LIT> ) return nullptr ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ty , Ptr -> getType ( ) } , { Ptr , Builder . getInt32 ( <NUM_LIT> ) } ) ;

else return Builder . CreateIntrinsic ( <STR_LIT> , { Ty , Ptr -> getType ( ) , Mask -> getType ( ) } , { Ptr , Builder . getInt32 ( <NUM_LIT> ) , Mask } ) ;


<s> if ( Ty -> getVectorNumElements ( ) != <NUM_LIT> || Ty -> getScalarSizeInBits ( ) != <NUM_LIT> ) return nullptr ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ty , Ptr -> getType ( ) } , { Ptr , Builder . getInt32 ( <NUM_LIT> ) } ) ; else return Builder . CreateIntrinsic ( <STR_LIT> , { Ty , Ptr -> getType ( ) , Mask -> getType ( ) } , { Ptr , Builder . getInt32 ( <NUM_LIT> ) , Mask } ) ;


<s> Type * OffsType = VectorType :: getInteger ( cast < VectorType > ( Ty ) ) ; if ( OffsType != Offsets -> getType ( ) ) { if ( OffsType -> getScalarSizeInBits ( ) > Offsets -> getType ( ) -> getScalarSizeInBits ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Offsets = Builder . CreateZExt ( Offsets , OffsType , <STR_LIT> ) ; } else {


<s> GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( Ptr ) ; if ( ! GEP ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return nullptr ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; Value * GEPPtr = GEP -> getPointerOperand ( ) ; if ( GEPPtr -> getType ( ) -> isVectorTy ( ) ) { return nullptr ; } if ( GEP -> getNumOperands ( ) != <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return nullptr ; } Offsets = GEP -> getOperand ( <NUM_LIT> ) ; assert ( cast < VectorType > ( Ty ) -> getNumElements ( ) == cast < VectorType > ( Offsets -> getType ( ) ) -> getNumElements ( ) ) ; if ( Offsets -> getType ( ) -> getScalarSizeInBits ( ) != <NUM_LIT> ) return nullptr ; if ( ZExtInst * ZextOffs = dyn_cast < ZExtInst > ( Offsets ) ) Offsets = ZextOffs -> getOperand ( <NUM_LIT> ) ; else if ( ! ( cast < VectorType > ( Offsets -> getType ( ) ) -> getNumElements ( ) == <NUM_LIT> && Offsets -> getType ( ) -> getScalarSizeInBits ( ) == <NUM_LIT> ) ) return nullptr ; if ( Ty != Offsets -> getType ( ) ) { if ( ( Ty -> getScalarSizeInBits ( ) < Offsets -> getType ( ) -> getScalarSizeInBits ( ) ) ) {


<s> return ( IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> ) ;


<s> auto * BCTy = cast < VectorType > ( BitCast -> getType ( ) ) ; auto * BCSrcTy = cast < VectorType > ( BitCast -> getOperand ( <NUM_LIT> ) -> getType ( ) ) ; if ( BCTy -> getNumElements ( ) == BCSrcTy -> getNumElements ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ;

Ptr = BitCast -> getOperand ( <NUM_LIT> ) ;


<s> if ( auto * BitCast = dyn_cast < BitCastInst > ( Ptr ) ) { auto * BCTy = cast < VectorType > ( BitCast -> getType ( ) ) ; auto * BCSrcTy = cast < VectorType > ( BitCast -> getOperand ( <NUM_LIT> ) -> getType ( ) ) ; if ( BCTy -> getNumElements ( ) == BCSrcTy -> getNumElements ( ) ) {


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; auto * Ty = cast < VectorType > ( I -> getType ( ) ) ; Value * Ptr = I -> getArgOperand ( <NUM_LIT> ) ; unsigned Alignment = cast < ConstantInt > ( I -> getArgOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; Value * PassThru = I -> getArgOperand ( <NUM_LIT> ) ; if ( ! isLegalTypeAndAlignment ( Ty -> getNumElements ( ) , Ty -> getScalarSizeInBits ( ) , Alignment ) ) return nullptr ; lookThroughBitcast ( Ptr ) ; assert ( Ptr -> getType ( ) -> isVectorTy ( ) && <STR_LIT> ) ; IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ; Instruction * Root = I ; Value * Load = tryCreateMaskedGatherOffset ( I , Ptr , Root , Builder ) ; if ( ! Load ) Load = tryCreateMaskedGatherBase ( I , Ptr , Builder ) ; if ( ! Load ) return nullptr ; if ( ! isa < UndefValue > ( PassThru ) && ! match ( PassThru , m_Zero ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; Load = Builder . CreateSelect ( Mask , Load , PassThru ) ;


<s> Value * Ptr = I -> getArgOperand ( <NUM_LIT> ) ; unsigned Alignment = cast < ConstantInt > ( I -> getArgOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; auto * Ty = cast < VectorType > ( Input -> getType ( ) ) ; if ( ! isLegalTypeAndAlignment ( Ty -> getNumElements ( ) , Ty -> getScalarSizeInBits ( ) , Alignment ) ) return nullptr ; lookThroughBitcast ( Ptr ) ; assert ( Ptr -> getType ( ) -> isVectorTy ( ) && <STR_LIT> ) ; IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ;

Value * Store = tryCreateMaskedScatterOffset ( I , Ptr , Builder ) ;


<s> Value * MVEGatherScatterLowering :: tryCreateMaskedGatherBase ( IntrinsicInst * I , Value * Ptr , IRBuilder < > & Builder ) { using namespace PatternMatch ; auto * Ty = cast < VectorType > ( I -> getType ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ;

if ( Ty -> getNumElements ( ) != <NUM_LIT> || Ty -> getScalarSizeInBits ( ) != <NUM_LIT> ) return nullptr ;


<s> if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ty , Ptr -> getType ( ) } , { Ptr , Builder . getInt32 ( <NUM_LIT> ) } ) ;


<s> if ( ! ( Ty -> getNumElements ( ) == <NUM_LIT> && Ty -> getScalarSizeInBits ( ) == <NUM_LIT> ) ) { return nullptr ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ptr -> getType ( ) , Input -> getType ( ) } , { Ptr , Builder . getInt32 ( <NUM_LIT> ) , Input } ) ;


<s> unsigned OffsetsElemCount = cast < FixedVectorType > ( Offsets -> getType ( ) ) -> getNumElements ( ) ; assert ( Ty -> getNumElements ( ) == OffsetsElemCount ) ; ZExtInst * ZextOffs = dyn_cast < ZExtInst > ( Offsets ) ; if ( ZextOffs ) Offsets = ZextOffs -> getOperand ( <NUM_LIT> ) ; FixedVectorType * OffsetType = cast < FixedVectorType > ( Offsets -> getType ( ) ) ;

if ( ! ZextOffs || cast < FixedVectorType > ( ZextOffs -> getDestTy ( ) ) -> getElementType ( ) -> getScalarSizeInBits ( ) != <NUM_LIT> ) if ( ! checkOffsetSize ( Offsets , OffsetsElemCount ) ) return nullptr ;


<s> Offsets = GEP -> getOperand ( <NUM_LIT> ) ; unsigned OffsetsElemCount = cast < FixedVectorType > ( Offsets -> getType ( ) ) -> getNumElements ( ) ; assert ( Ty -> getNumElements ( ) == OffsetsElemCount ) ; ZExtInst * ZextOffs = dyn_cast < ZExtInst > ( Offsets ) ; if ( ZextOffs ) Offsets = ZextOffs -> getOperand ( <NUM_LIT> ) ; FixedVectorType * OffsetType = cast < FixedVectorType > ( Offsets -> getType ( ) ) ; if ( ! ZextOffs || cast < FixedVectorType > ( ZextOffs -> getDestTy ( ) ) -> getElementType ( ) -> getScalarSizeInBits ( ) != <NUM_LIT> ) if ( ! checkOffsetSize ( Offsets , OffsetsElemCount ) ) return nullptr ; if ( Ty != Offsets -> getType ( ) ) { if ( ( Ty -> getElementType ( ) -> getScalarSizeInBits ( ) < OffsetType -> getElementType ( ) -> getScalarSizeInBits ( ) ) ) {

Offsets = Builder . CreateTrunc ( Offsets , Ty ) ;


<s> Optional < int64_t > Op0 = getIfConst ( I -> getOperand ( <NUM_LIT> ) ) ; Optional < int64_t > Op1 = getIfConst ( I -> getOperand ( <NUM_LIT> ) ) ; if ( ! Op0 || ! Op1 ) return Optional < int64_t > { } ; if ( I -> getOpcode ( ) == Instruction :: Add ) return Optional < int64_t > { Op0 . getValue ( ) + Op1 . getValue ( ) } ; if ( I -> getOpcode ( ) == Instruction :: Mul ) return Optional < int64_t > { Op0 . getValue ( ) * Op1 . getValue ( ) } ; } return Optional < int64_t > { } ;


<s> else if ( ( Const = getIfConst ( Add -> getOperand ( <NUM_LIT> ) ) ) ) Summand = Add -> getOperand ( <NUM_LIT> ) ; else return ReturnFalse ; int64_t Immediate = Const . getValue ( ) << TypeScale ; if ( Immediate > <NUM_LIT> || Immediate < - <NUM_LIT> || Immediate % <NUM_LIT> != <NUM_LIT> ) return ReturnFalse ; return std :: pair < Value * , int64_t > ( Summand , Immediate ) ;


<s> return false ; } bool Gatscat = true ; for ( User * U : I -> users ( ) ) { if ( ! isa < Instruction > ( U ) ) return false ; if ( isa < GetElementPtrInst > ( U ) || isGatherScatter ( dyn_cast < IntrinsicInst > ( U ) ) ) { return Gatscat ; } else {

unsigned OpCode = cast < Instruction > ( U ) -> getOpcode ( ) ;


<s> void MVEGatherScatterLowering :: lookThroughBitcast ( Value * & Ptr ) { if ( auto * BitCast = dyn_cast < BitCastInst > ( Ptr ) ) { auto * BCTy = cast < FixedVectorType > ( BitCast -> getType ( ) ) ; auto * BCSrcTy = cast < FixedVectorType > ( BitCast -> getOperand ( <NUM_LIT> ) -> getType ( ) ) ; if ( BCTy -> getNumElements ( ) == BCSrcTy -> getNumElements ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ;


<s> if ( ! isLegalTypeAndAlignment ( Ty -> getNumElements ( ) , Ty -> getScalarSizeInBits ( ) , Alignment ) ) return nullptr ; lookThroughBitcast ( Ptr ) ; assert ( Ptr -> getType ( ) -> isVectorTy ( ) && <STR_LIT> ) ; IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ; Instruction * Root = I ; Value * Load = tryCreateMaskedGatherOffset ( I , Ptr , Root , Builder ) ; if ( ! Load ) Load = tryCreateMaskedGatherBase ( I , Ptr , Builder ) ; if ( ! Load ) return nullptr ; if ( ! isa < UndefValue > ( PassThru ) && ! match ( PassThru , m_Zero ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; Load = Builder . CreateSelect ( Mask , Load , PassThru ) ; } Root -> replaceAllUsesWith ( Load ) ;


<s> Value * PassThru = I -> getArgOperand ( <NUM_LIT> ) ; if ( ! isLegalTypeAndAlignment ( Ty -> getNumElements ( ) , Ty -> getScalarSizeInBits ( ) , Alignment ) ) return nullptr ; lookThroughBitcast ( Ptr ) ; assert ( Ptr -> getType ( ) -> isVectorTy ( ) && <STR_LIT> ) ; IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ; Instruction * Root = I ; Value * Load = tryCreateMaskedGatherOffset ( I , Ptr , Root , Builder ) ; if ( ! Load ) Load = tryCreateMaskedGatherBase ( I , Ptr , Builder ) ; if ( ! Load ) return nullptr ; if ( ! isa < UndefValue > ( PassThru ) && ! match ( PassThru , m_Zero ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; Load = Builder . CreateSelect ( Mask , Load , PassThru ) ;


<s> Align Alignment = cast < ConstantInt > ( I -> getArgOperand ( <NUM_LIT> ) ) -> getAlignValue ( ) ; auto * Ty = cast < FixedVectorType > ( Input -> getType ( ) ) ; if ( ! isLegalTypeAndAlignment ( Ty -> getNumElements ( ) , Ty -> getScalarSizeInBits ( ) , Alignment ) ) return nullptr ; lookThroughBitcast ( Ptr ) ; assert ( Ptr -> getType ( ) -> isVectorTy ( ) && <STR_LIT> ) ; IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ;


<s> if ( ! isLegalTypeAndAlignment ( Ty -> getNumElements ( ) , Ty -> getScalarSizeInBits ( ) , Alignment ) ) return nullptr ; lookThroughBitcast ( Ptr ) ; assert ( Ptr -> getType ( ) -> isVectorTy ( ) && <STR_LIT> ) ; IRBuilder < > Builder ( I -> getContext ( ) ) ; Builder . SetInsertPoint ( I ) ; Builder . SetCurrentDebugLocation ( I -> getDebugLoc ( ) ) ; Value * Store = tryCreateMaskedScatterOffset ( I , Ptr , Builder ) ;

if ( ! Store ) Store = tryCreateMaskedScatterBase ( I , Ptr , Builder ) ;


<s> void MVEGatherScatterLowering :: pushOutMul ( PHINode * & Phi , Value * IncrementPerRound , Value * OffsSecondOperand , unsigned LoopIncrement , IRBuilder < > & Builder ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Instruction * InsertionPoint = & cast < Instruction > ( Phi -> getIncomingBlock ( LoopIncrement == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ) -> back ( ) ) ; Value * StartIndex = BinaryOperator :: Create ( Instruction :: Mul , Phi -> getIncomingValue ( LoopIncrement == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ) , OffsSecondOperand , <STR_LIT> , InsertionPoint ) ;

Instruction * Product = BinaryOperator :: Create ( Instruction :: Mul , IncrementPerRound , OffsSecondOperand , <STR_LIT> , InsertionPoint ) ;


<s> if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; SmallVector < IntrinsicInst * , <NUM_LIT> > Gathers ; SmallVector < IntrinsicInst * , <NUM_LIT> > Scatters ; bool Changed = false ; for ( BasicBlock & BB : F ) { for ( Instruction & I : BB ) { IntrinsicInst * II = dyn_cast < IntrinsicInst > ( & I ) ; if ( II && II -> getIntrinsicID ( ) == <STR_LIT> && isa < FixedVectorType > ( II -> getType ( ) ) ) { Gathers . push_back ( II ) ;

Changed |= optimiseAddress ( II -> getArgOperand ( <NUM_LIT> ) , II -> getParent ( ) , LI ) ;


<s> if ( L == nullptr ) return nullptr ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; DataLayout DT = I -> getParent ( ) -> getParent ( ) -> getParent ( ) -> getDataLayout ( ) ; int TypeScale = computeScale ( DT . getTypeSizeInBits ( GEP -> getOperand ( <NUM_LIT> ) -> getType ( ) ) , DT . getTypeSizeInBits ( GEP -> getType ( ) ) / cast < FixedVectorType > ( GEP -> getType ( ) ) -> getNumElements ( ) ) ; if ( TypeScale == - <NUM_LIT> ) return nullptr ; if ( GEP -> hasOneUse ( ) ) { Value * Load = tryCreateIncrementingWBGatScat ( I , BasePtr , Offsets , TypeScale , Builder ) ; if ( Load != nullptr ) return Load ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; std :: pair < Value * , int64_t > Add = getVarAndConst ( Offsets , TypeScale ) ; if ( Add . first == nullptr ) return nullptr ; Value * OffsetsIncoming = Add . first ; int64_t Immediate = Add . second ; Instruction * ScaledOffsets = BinaryOperator :: Create ( Instruction :: Shl , OffsetsIncoming , Builder . CreateVectorSplat ( Ty -> getNumElements ( ) , Builder . getInt32 ( TypeScale ) ) , <STR_LIT> , I ) ; OffsetsIncoming = BinaryOperator :: Create ( Instruction :: Add , ScaledOffsets , Builder . CreateVectorSplat ( Ty -> getNumElements ( ) , Builder . CreatePtrToInt ( BasePtr , cast < VectorType > ( ScaledOffsets -> getType ( ) ) -> getElementType ( ) ) ) , <STR_LIT> , I ) ; if ( I -> getIntrinsicID ( ) == <STR_LIT> ) return cast < IntrinsicInst > ( tryCreateMaskedGatherBase ( I , OffsetsIncoming , Builder , Immediate ) ) ; else return cast < IntrinsicInst > ( tryCreateMaskedScatterBase ( I , OffsetsIncoming , Builder , Immediate ) ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; DataLayout DT = I -> getParent ( ) -> getParent ( ) -> getParent ( ) -> getDataLayout ( ) ; int TypeScale = computeScale ( DT . getTypeSizeInBits ( GEP -> getOperand ( <NUM_LIT> ) -> getType ( ) ) , DT . getTypeSizeInBits ( GEP -> getType ( ) ) / cast < FixedVectorType > ( GEP -> getType ( ) ) -> getNumElements ( ) ) ; if ( TypeScale == - <NUM_LIT> ) return nullptr ; if ( GEP -> hasOneUse ( ) ) { Value * Load = tryCreateIncrementingWBGatScat ( I , BasePtr , Offsets , TypeScale , Builder ) ; if ( Load != nullptr ) return Load ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; std :: pair < Value * , int64_t > Add = getVarAndConst ( Offsets , TypeScale ) ; if ( Add . first == nullptr ) return nullptr ; Value * OffsetsIncoming = Add . first ; int64_t Immediate = Add . second ; Instruction * ScaledOffsets = BinaryOperator :: Create ( Instruction :: Shl , OffsetsIncoming , Builder . CreateVectorSplat ( Ty -> getNumElements ( ) , Builder . getInt32 ( TypeScale ) ) , <STR_LIT> , I ) ; OffsetsIncoming = BinaryOperator :: Create ( Instruction :: Add , ScaledOffsets , Builder . CreateVectorSplat ( Ty -> getNumElements ( ) , Builder . CreatePtrToInt ( BasePtr , cast < VectorType > ( ScaledOffsets -> getType ( ) ) -> getElementType ( ) ) ) , <STR_LIT> , I ) ; if ( I -> getIntrinsicID ( ) == <STR_LIT> ) return cast < IntrinsicInst > ( tryCreateMaskedGatherBase ( I , OffsetsIncoming , Builder , Immediate ) ) ;

else return cast < IntrinsicInst > ( tryCreateMaskedScatterBase ( I , OffsetsIncoming , Builder , Immediate ) ) ;


<s> if ( Add . first == nullptr ) return nullptr ; Value * OffsetsIncoming = Add . first ; int64_t Immediate = Add . second ; if ( OffsetsIncoming != Phi ) return nullptr ; Builder . SetInsertPoint ( & Phi -> getIncomingBlock ( <NUM_LIT> - IncrementIndex ) -> back ( ) ) ; unsigned NumElems = cast < FixedVectorType > ( OffsetsIncoming -> getType ( ) ) -> getNumElements ( ) ; Instruction * ScaledOffsets = BinaryOperator :: Create ( Instruction :: Shl , Phi -> getIncomingValue ( <NUM_LIT> - IncrementIndex ) , Builder . CreateVectorSplat ( NumElems , Builder . getInt32 ( TypeScale ) ) , <STR_LIT> , & Phi -> getIncomingBlock ( <NUM_LIT> - IncrementIndex ) -> back ( ) ) ; OffsetsIncoming = BinaryOperator :: Create ( Instruction :: Add , ScaledOffsets , Builder . CreateVectorSplat ( NumElems , Builder . CreatePtrToInt ( BasePtr , cast < VectorType > ( ScaledOffsets -> getType ( ) ) -> getElementType ( ) ) ) , <STR_LIT> , & Phi -> getIncomingBlock ( <NUM_LIT> - IncrementIndex ) -> back ( ) ) ; OffsetsIncoming = BinaryOperator :: Create ( Instruction :: Sub , OffsetsIncoming , Builder . CreateVectorSplat ( NumElems , Builder . getInt32 ( Immediate ) ) , <STR_LIT> , & Phi -> getIncomingBlock ( <NUM_LIT> - IncrementIndex ) -> back ( ) ) ; Phi -> setIncomingValue ( <NUM_LIT> - IncrementIndex , OffsetsIncoming ) ; Builder . SetInsertPoint ( I ) ; Value * EndResult ; Value * NewInduction ; if ( I -> getIntrinsicID ( ) == <STR_LIT> ) { Value * Load = tryCreateMaskedGatherBaseWB ( I , Phi , Builder , Immediate ) ; EndResult = Builder . CreateExtractValue ( Load , <NUM_LIT> , <STR_LIT> ) ; NewInduction = Builder . CreateExtractValue ( Load , <NUM_LIT> , <STR_LIT> ) ; } else { NewInduction = tryCreateMaskedScatterBaseWB ( I , Phi , Builder , Immediate ) ; EndResult = NewInduction ; } Instruction * AddInst = cast < Instruction > ( Offsets ) ; AddInst -> replaceAllUsesWith ( NewInduction ) ;


<s> if ( Phi == nullptr || Phi -> getNumIncomingValues ( ) != <NUM_LIT> || Phi -> getParent ( ) != L -> getHeader ( ) || Phi -> getNumUses ( ) != <NUM_LIT> ) return nullptr ; unsigned IncrementIndex = Phi -> getIncomingBlock ( <NUM_LIT> ) == L -> getLoopLatch ( ) ? <NUM_LIT> : <NUM_LIT> ; Offsets = Phi -> getIncomingValue ( IncrementIndex ) ; std :: pair < Value * , int64_t > Add = getVarAndConst ( Offsets , TypeScale ) ; if ( Add . first == nullptr ) return nullptr ; Value * OffsetsIncoming = Add . first ; int64_t Immediate = Add . second ; if ( OffsetsIncoming != Phi ) return nullptr ; Builder . SetInsertPoint ( & Phi -> getIncomingBlock ( <NUM_LIT> - IncrementIndex ) -> back ( ) ) ; unsigned NumElems = cast < FixedVectorType > ( OffsetsIncoming -> getType ( ) ) -> getNumElements ( ) ; Instruction * ScaledOffsets = BinaryOperator :: Create ( Instruction :: Shl , Phi -> getIncomingValue ( <NUM_LIT> - IncrementIndex ) , Builder . CreateVectorSplat ( NumElems , Builder . getInt32 ( TypeScale ) ) , <STR_LIT> , & Phi -> getIncomingBlock ( <NUM_LIT> - IncrementIndex ) -> back ( ) ) ; OffsetsIncoming = BinaryOperator :: Create ( Instruction :: Add , ScaledOffsets , Builder . CreateVectorSplat ( NumElems , Builder . CreatePtrToInt ( BasePtr , cast < VectorType > ( ScaledOffsets -> getType ( ) ) -> getElementType ( ) ) ) , <STR_LIT> , & Phi -> getIncomingBlock ( <NUM_LIT> - IncrementIndex ) -> back ( ) ) ; OffsetsIncoming = BinaryOperator :: Create ( Instruction :: Sub , OffsetsIncoming , Builder . CreateVectorSplat ( NumElems , Builder . getInt32 ( Immediate ) ) , <STR_LIT> , & Phi -> getIncomingBlock ( <NUM_LIT> - IncrementIndex ) -> back ( ) ) ; Phi -> setIncomingValue ( <NUM_LIT> - IncrementIndex , OffsetsIncoming ) ; Builder . SetInsertPoint ( I ) ; Value * EndResult ; Value * NewInduction ; if ( I -> getIntrinsicID ( ) == <STR_LIT> ) { Value * Load = tryCreateMaskedGatherBaseWB ( I , Phi , Builder , Immediate ) ; EndResult = Builder . CreateExtractValue ( Load , <NUM_LIT> , <STR_LIT> ) ; NewInduction = Builder . CreateExtractValue ( Load , <NUM_LIT> , <STR_LIT> ) ; } else { NewInduction = tryCreateMaskedScatterBaseWB ( I , Phi , Builder , Immediate ) ;


<s> if ( Ty -> getNumElements ( ) != <NUM_LIT> || Ty -> getScalarSizeInBits ( ) != <NUM_LIT> ) return nullptr ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ;

if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ty , Ptr -> getType ( ) } , { Ptr , Builder . getInt32 ( Increment ) } ) ;


<s> Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ty , Ptr -> getType ( ) } , { Ptr , Builder . getInt32 ( Increment ) } ) ; else return Builder . CreateIntrinsic ( <STR_LIT> , { Ty , Ptr -> getType ( ) , Mask -> getType ( ) } , { Ptr , Builder . getInt32 ( Increment ) , Mask } ) ;


<s> Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ty , Ptr -> getType ( ) } , { Ptr , Builder . getInt32 ( Increment ) } ) ;

else return Builder . CreateIntrinsic ( <STR_LIT> , { Ty , Ptr -> getType ( ) , Mask -> getType ( ) } , { Ptr , Builder . getInt32 ( Increment ) , Mask } ) ;


<s> Value * MVEGatherScatterLowering :: tryCreateMaskedGatherBaseWB ( IntrinsicInst * I , Value * Ptr , IRBuilder < > & Builder , int64_t Increment ) { using namespace PatternMatch ; auto * Ty = cast < FixedVectorType > ( I -> getType ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ;


<s> unsigned Unsigned = <NUM_LIT> ; auto * Extend = Root ; if ( OriginalTy -> getPrimitiveSizeInBits ( ) < <NUM_LIT> ) { if ( ! I -> hasOneUse ( ) ) return nullptr ; Extend = cast < Instruction > ( * I -> users ( ) . begin ( ) ) ; if ( isa < SExtInst > ( Extend ) ) { Unsigned = <NUM_LIT> ; } else if ( ! isa < ZExtInst > ( Extend ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return nullptr ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; ResultTy = Extend -> getType ( ) ; if ( ResultTy -> getPrimitiveSizeInBits ( ) != <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return nullptr ; } } GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( Ptr ) ; Value * Offsets ; Value * BasePtr = checkGEP ( Offsets , cast < FixedVectorType > ( ResultTy ) , GEP , Builder ) ; if ( ! BasePtr ) return nullptr ; Value * Load = tryCreateIncrementingGatScat ( I , BasePtr , Offsets , GEP , Builder ) ; if ( Load ) return Load ; int Scale = computeScale ( BasePtr -> getType ( ) -> getPointerElementType ( ) -> getPrimitiveSizeInBits ( ) , OriginalTy -> getScalarSizeInBits ( ) ) ; if ( Scale == - <NUM_LIT> ) return nullptr ;


<s> } Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ptr -> getType ( ) , Input -> getType ( ) } , { Ptr , Builder . getInt32 ( Increment ) , Input } ) ;

else return Builder . CreateIntrinsic ( <STR_LIT> , { Ptr -> getType ( ) , Input -> getType ( ) , Mask -> getType ( ) } , { Ptr , Builder . getInt32 ( Increment ) , Input , Mask } ) ;


<s> if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ptr -> getType ( ) , Input -> getType ( ) } , { Ptr , Builder . getInt32 ( Increment ) , Input } ) ;


<s> auto * Ty = cast < FixedVectorType > ( Input -> getType ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( Ty -> getNumElements ( ) != <NUM_LIT> || Ty -> getScalarSizeInBits ( ) != <NUM_LIT> ) return nullptr ; Value * Mask = I -> getArgOperand ( <NUM_LIT> ) ; if ( match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { Ptr -> getType ( ) , Input -> getType ( ) } , { Ptr , Builder . getInt32 ( Increment ) , Input } ) ;


<s> if ( ! BasePtr ) return nullptr ; Value * Store = tryCreateIncrementingGatScat ( I , BasePtr , Offsets , GEP , Builder ) ; if ( Store ) return Store ; int Scale = computeScale ( BasePtr -> getType ( ) -> getPointerElementType ( ) -> getPrimitiveSizeInBits ( ) , MemoryTy -> getScalarSizeInBits ( ) ) ; if ( Scale == - <NUM_LIT> ) return nullptr ; if ( ! match ( Mask , m_One ( ) ) ) return Builder . CreateIntrinsic ( <STR_LIT> , { BasePtr -> getType ( ) , Offsets -> getType ( ) , Input -> getType ( ) , Mask -> getType ( ) } , { BasePtr , Offsets , Input , Builder . getInt32 ( MemoryTy -> getScalarSizeInBits ( ) ) , Builder . getInt32 ( Scale ) , Mask } ) ; else return Builder . CreateIntrinsic ( <STR_LIT> , { BasePtr -> getType ( ) , Offsets -> getType ( ) , Input -> getType ( ) } , { BasePtr , Offsets , Input , Builder . getInt32 ( MemoryTy -> getScalarSizeInBits ( ) ) , Builder . getInt32 ( Scale ) } ) ;


<s> Pass * llvm :: createMVELaneInterleavingPass ( ) { return new MVELaneInterleaving ( ) ;


<s> Pass * llvm :: createMVELaneInterleavingPass ( ) {


<s> StringRef getPassName ( ) const override {


<s> } } for ( auto * T : Truncs ) { if ( T -> hasOneUse ( ) && ! isa < StoreInst > ( * T -> user_begin ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * T << <STR_LIT> ) ; return true ; } } for ( auto * E : Exts ) { if ( ! E -> hasOneUse ( ) || cast < Instruction > ( * E -> user_begin ( ) ) -> getOpcode ( ) != Instruction :: Mul ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * E << <STR_LIT> ) ;


<s> } } for ( auto * E : Exts ) { if ( ! E -> hasOneUse ( ) || cast < Instruction > ( * E -> user_begin ( ) ) -> getOpcode ( ) != Instruction :: Mul ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * E << <STR_LIT> ) ; return false ;


<s> bool Changed = false ; SmallPtrSet < Instruction * , <NUM_LIT> > Visited ; for ( Instruction & I : reverse ( instructions ( F ) ) ) { if ( I . getType ( ) -> isVectorTy ( ) && ( isa < TruncInst > ( I ) || isa < FPTruncInst > ( I ) ) && ! Visited . count ( & I ) ) Changed |= tryInterleave ( & I , Visited ) ; } return Changed ;


<s> case <STR_LIT> : if ( ST -> hasMVEFloatOps ( ) ) continue ; break ; default : break ; } if ( IsMasked ( & I ) ) { auto * VecTy = getVectorType ( Int ) ; unsigned Lanes = VecTy -> getNumElements ( ) ; unsigned ElementWidth = VecTy -> getScalarSizeInBits ( ) ; unsigned MaxWidth = TTI -> getRegisterBitWidth ( true ) ; if ( Lanes * ElementWidth > MaxWidth || Lanes == MaxWidth ) return false ; MaskedInsts . push_back ( cast < IntrinsicInst > ( & I ) ) ; continue ; } for ( const Use & U : Int -> args ( ) ) { if ( isa < VectorType > ( U -> getType ( ) ) ) return false ; } } } if ( ! ActiveLaneMask ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; }


<s> case <STR_LIT> : if ( ST -> hasMVEFloatOps ( ) ) continue ; break ; default : break ; } if ( IsMasked ( & I ) ) { auto * VecTy = getVectorType ( Int ) ; unsigned Lanes = VecTy -> getNumElements ( ) ; unsigned ElementWidth = VecTy -> getScalarSizeInBits ( ) ; unsigned MaxWidth = TTI -> getRegisterBitWidth ( true ) ; if ( Lanes * ElementWidth > MaxWidth || Lanes == MaxWidth ) return false ; MaskedInsts . push_back ( cast < IntrinsicInst > ( & I ) ) ; continue ; } for ( const Use & U : Int -> args ( ) ) { if ( isa < VectorType > ( U -> getType ( ) ) ) return false ; } } } if ( ! ActiveLaneMask ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } return ! MaskedInsts . empty ( ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } } unsigned SizeInBits = TripCount -> getType ( ) -> getScalarSizeInBits ( ) ; auto MaxMinusVW = APInt ( SizeInBits , ~ <NUM_LIT> ) - APInt ( SizeInBits , VectorWidth ) ; APInt UpperboundTC = SE -> getUnsignedRangeMax ( TC ) ; if ( UpperboundTC . ugt ( MaxMinusVW ) && ! ForceTailPredication ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; dbgs ( ) << UpperboundTC << <STR_LIT> << MaxMinusVW << <STR_LIT> ; ) ; return false ; } auto * ECPlusVWMinus1 = SE -> getAddExpr ( EC , SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth - <NUM_LIT> ) ) ) ; auto * Ceil = SE -> getUDivExpr ( ECPlusVWMinus1 , SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth ) ) ) ; ConstantRange RangeCeil = SE -> getUnsignedRange ( Ceil ) ; ConstantRange RangeTC = SE -> getUnsignedRange ( TC ) ; if ( ! RangeTC . isSingleElement ( ) ) { auto ZeroRange = ConstantRange ( APInt ( TripCount -> getType ( ) -> getScalarSizeInBits ( ) , <NUM_LIT> ) ) ; RangeTC = RangeTC . unionWith ( ZeroRange ) ; } if ( ! RangeTC . contains ( RangeCeil ) && ! ForceTailPredication ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto * IV = ActiveLaneMask -> getOperand ( <NUM_LIT> ) ; auto * IVExpr = SE -> getSCEV ( IV ) ; auto * AddExpr = dyn_cast < SCEVAddRecExpr > ( IVExpr ) ; if ( ! AddExpr ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; IVExpr -> dump ( ) ) ; return false ; } if ( AddExpr -> getLoop ( ) != L ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto * Step = dyn_cast < SCEVConstant > ( AddExpr -> getOperand ( <NUM_LIT> ) ) ; if ( ! Step ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; AddExpr -> getOperand ( <NUM_LIT> ) -> dump ( ) ) ; return false ; }

auto StepValue = Step -> getValue ( ) -> getSExtValue ( ) ;


<s> } SmallPtrSet < Instruction * , <NUM_LIT> > Dead ; while ( ! MaybeDead . empty ( ) ) { auto * I = MaybeDead . front ( ) ; MaybeDead . remove ( I ) ; if ( I -> hasNUsesOrMore ( <NUM_LIT> ) ) continue ; for ( auto & U : I -> operands ( ) ) if ( auto * OpI = dyn_cast < Instruction > ( U ) ) MaybeDead . insert ( OpI ) ; I -> dropAllReferences ( ) ; Dead . insert ( I ) ; } for ( auto * I : Dead ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; I -> dump ( ) ) ; I -> eraseFromParent ( ) ; } for ( auto I : L -> blocks ( ) ) DeleteDeadPHIs ( I ) ;


<s> if ( Const -> getAPInt ( ) != - VF -> getValue ( ) ) return nullptr ; } else return nullptr ; return dyn_cast < SCEVMulExpr > ( S -> getOperand ( <NUM_LIT> ) ) ; } ; auto VisitMul = [ & ] ( const SCEVMulExpr * S ) -> const SCEVUDivExpr * { if ( auto * Const = dyn_cast < SCEVConstant > ( S -> getOperand ( <NUM_LIT> ) ) ) { if ( Const -> getValue ( ) != VF ) return nullptr ; } else return nullptr ; return dyn_cast < SCEVUDivExpr > ( S -> getOperand ( <NUM_LIT> ) ) ; } ; auto VisitDiv = [ & ] ( const SCEVUDivExpr * S ) -> const SCEV * { if ( auto * Const = dyn_cast < SCEVConstant > ( S -> getRHS ( ) ) ) { if ( Const -> getValue ( ) != VF ) return nullptr ; } else return nullptr ; if ( auto * RoundUp = dyn_cast < SCEVAddExpr > ( S -> getLHS ( ) ) ) { if ( auto * Const = dyn_cast < SCEVConstant > ( RoundUp -> getOperand ( <NUM_LIT> ) ) ) { if ( Const -> getAPInt ( ) != ( VF -> getValue ( ) - <NUM_LIT> ) ) return nullptr ; } else return nullptr ; return RoundUp -> getOperand ( <NUM_LIT> ) ; } return nullptr ; } ; const SCEV * Elems = nullptr ; if ( auto * TC = dyn_cast < SCEVAddExpr > ( TripCountSE ) ) if ( auto * Div = dyn_cast < SCEVUDivExpr > ( TC -> getOperand ( <NUM_LIT> ) ) ) if ( auto * Add = dyn_cast < SCEVAddExpr > ( Div -> getLHS ( ) ) ) if ( auto * Mul = VisitAdd ( Add ) ) if ( auto * Div = VisitMul ( Mul ) ) if ( auto * Res = VisitDiv ( Div ) ) Elems = Res ; if ( ! Elems ) return false ; Instruction * InsertPt = L -> getLoopPreheader ( ) -> getTerminator ( ) ; if ( ! isSafeToExpandAt ( Elems , InsertPt , * SE ) ) return false ; auto DL = L -> getHeader ( ) -> getModule ( ) -> getDataLayout ( ) ; SCEVExpander Expander ( * SE , DL , <STR_LIT> ) ; TCP . NumElements = Expander . expandCodeFor ( Elems , Elems -> getType ( ) , InsertPt ) ;

if ( ! MatchElemCountLoopSetup ( L , TCP . Shuffle , TCP . NumElements ) ) return false ;


<s> static VectorType * getVectorType ( IntrinsicInst * I ) { unsigned TypeOp = I -> getIntrinsicID ( ) == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ;

auto * PtrTy = cast < PointerType > ( I -> getOperand ( TypeOp ) -> getType ( ) ) ;


<s> auto * PtrTy = cast < PointerType > ( I -> getOperand ( TypeOp ) -> getType ( ) ) ; return cast < VectorType > ( PtrTy -> getElementType ( ) ) ;


<s> void MVETailPredication :: InsertVCTPIntrinsic ( TripCountPattern & TCP , DenseMap < Instruction * , Instruction * > & NewPredicates ) { IRBuilder < > Builder ( L -> getHeader ( ) -> getFirstNonPHI ( ) ) ; Module * M = L -> getHeader ( ) -> getModule ( ) ; Type * Ty = IntegerType :: get ( M -> getContext ( ) , <NUM_LIT> ) ; PHINode * Processed = Builder . CreatePHI ( Ty , <NUM_LIT> ) ; Processed -> addIncoming ( TCP . NumElements , L -> getLoopPreheader ( ) ) ; Builder . SetInsertPoint ( cast < Instruction > ( TCP . Predicate ) ) ; ConstantInt * Factor = ConstantInt :: get ( cast < IntegerType > ( Ty ) , TCP . VecTy -> getNumElements ( ) ) ; <STR_LIT> VCTPID ; switch ( TCP . VecTy -> getNumElements ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; } Function * VCTP = <STR_LIT> ( M , VCTPID ) ; Value * TailPredicate = Builder . CreateCall ( VCTP , Processed ) ;

TCP . Predicate -> replaceAllUsesWith ( TailPredicate ) ;


<s> VectorType * VecTy = getVectorType ( cast < IntrinsicInst > ( & I ) ) ; unsigned Lanes = VecTy -> getNumElements ( ) ; unsigned ElementWidth = VecTy -> getScalarSizeInBits ( ) ; unsigned MaxWidth = TTI -> getRegisterBitWidth ( true ) ; if ( Lanes * ElementWidth > MaxWidth || Lanes == MaxWidth ) return false ; MaskedInsts . push_back ( cast < IntrinsicInst > ( & I ) ) ;

} else if ( auto * Int = dyn_cast < IntrinsicInst > ( & I ) ) {


<s> Constant * Const = nullptr ; if ( ! match ( TCP . Induction , m_Add ( m_Instruction ( BroadcastSplat ) , m_Constant ( Const ) ) ) && ! match ( TCP . Induction , m_Or ( m_Instruction ( BroadcastSplat ) , m_Constant ( Const ) ) ) ) return false ; if ( auto * CDS = dyn_cast < ConstantDataSequential > ( Const ) ) { for ( unsigned i = <NUM_LIT> ; i < CDS -> getNumElements ( ) ; ++ i ) { if ( CDS -> getElementAsInteger ( i ) != i ) return false ; } } else return false ; Instruction * Insert = nullptr ; if ( ! match ( BroadcastSplat , m_ShuffleVector ( m_Instruction ( Insert ) , m_Undef ( ) , m_ZeroMask ( ) ) ) ) return false ; Instruction * IV = nullptr ; if ( ! match ( Insert , m_InsertElement ( m_Undef ( ) , m_Instruction ( IV ) , m_Zero ( ) ) ) ) return false ; auto * Phi = dyn_cast < PHINode > ( IV ) ;


<s> Instruction * Insert = nullptr ; if ( ! match ( Shuffle , m_ShuffleVector ( m_Instruction ( Insert ) , m_Undef ( ) , m_ZeroMask ( ) ) ) ) return false ; Instruction * BECount = nullptr ; if ( ! match ( Insert , m_InsertElement ( m_Undef ( ) , m_Instruction ( BECount ) , m_Zero ( ) ) ) ) return false ;


<s> LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; TTI = & getAnalysis < TargetTransformInfoWrapperPass > ( ) . getTTI ( F ) ; SE = & getAnalysis < ScalarEvolutionWrapperPass > ( ) . getSE ( ) ; auto * TLIP = getAnalysisIfAvailable < TargetLibraryInfoWrapperPass > ( ) ; TLI = TLIP ? & TLIP -> getTLI ( * L -> getHeader ( ) -> getParent ( ) ) : nullptr ; DL = & L -> getHeader ( ) -> getModule ( ) -> getDataLayout ( ) ; this -> L = L ; if ( ! ST -> hasMVEIntegerOps ( ) || ! ST -> hasV8_1MMainlineOps ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } BasicBlock * Preheader = L -> getLoopPreheader ( ) ; if ( ! Preheader ) return false ;

auto FindLoopIterations = [ ] ( BasicBlock * BB ) -> IntrinsicInst * {


<s> SetVector < Instruction * > Predicates ; DenseMap < Instruction * , Instruction * > NewPredicates ; bool FoundScalarTC = false ; for ( auto * I : MaskedInsts ) { <STR_LIT> ID = I -> getIntrinsicID ( ) ; unsigned PredOp = ID == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ; auto * Predicate = dyn_cast < Instruction > ( I -> getArgOperand ( PredOp ) ) ; if ( ! Predicate || Predicates . count ( Predicate ) ) continue ; TripCountPattern TCP ( Predicate , TripCount , getVectorType ( I ) ) ; if ( ! ( ComputeConstElements ( TCP ) || ComputeRuntimeElements ( TCP ) ) ) continue ; LLVM_DEBUG ( FoundScalarTC = true ) ; if ( ! isTailPredicate ( TCP ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; Predicates . insert ( Predicate ) ; InsertVCTPIntrinsic ( TCP , NewPredicates ) ;


<s> break ; } } } } SmallPtrSet < Instruction * , <NUM_LIT> > Dead ; while ( ! MaybeDead . empty ( ) ) { auto * I = MaybeDead . front ( ) ; MaybeDead . remove ( I ) ; if ( I -> hasNUsesOrMore ( <NUM_LIT> ) ) continue ; for ( auto & U : I -> operands ( ) ) { if ( auto * OpI = dyn_cast < Instruction > ( U ) ) MaybeDead . insert ( OpI ) ; } I -> dropAllReferences ( ) ;


<s> } return nullptr ; } ; const SCEV * Elems = nullptr ; if ( auto * TC = dyn_cast < SCEVAddExpr > ( TripCountSE ) ) if ( auto * Div = dyn_cast < SCEVUDivExpr > ( TC -> getOperand ( <NUM_LIT> ) ) ) if ( auto * Add = dyn_cast < SCEVAddExpr > ( Div -> getLHS ( ) ) ) if ( auto * Mul = VisitAdd ( Add ) ) if ( auto * Div = VisitMul ( Mul ) ) if ( auto * Res = VisitDiv ( Div ) ) Elems = Res ; if ( ! Elems ) return nullptr ; Instruction * InsertPt = L -> getLoopPreheader ( ) -> getTerminator ( ) ; if ( ! isSafeToExpandAt ( Elems , InsertPt , * SE ) ) return nullptr ; auto DL = L -> getHeader ( ) -> getModule ( ) -> getDataLayout ( ) ; SCEVExpander Expander ( * SE , DL , <STR_LIT> ) ;

return Expander . expandCodeFor ( Elems , Elems -> getType ( ) , InsertPt ) ;


<s> auto * PtrTy = cast < PointerType > ( I -> getOperand ( TypeOp ) -> getType ( ) ) ;

return cast < VectorType > ( PtrTy -> getElementType ( ) ) ;


<s> static VectorType * getVectorType ( IntrinsicInst * I ) { unsigned TypeOp = I -> getIntrinsicID ( ) == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ; auto * PtrTy = cast < PointerType > ( I -> getOperand ( TypeOp ) -> getType ( ) ) ; return cast < VectorType > ( PtrTy -> getElementType ( ) ) ;


<s> unsigned MaxWidth = TTI -> getRegisterBitWidth ( true ) ; if ( Lanes * ElementWidth > MaxWidth || Lanes == MaxWidth ) return false ; MaskedInsts . push_back ( cast < IntrinsicInst > ( & I ) ) ; } else if ( auto * Int = dyn_cast < IntrinsicInst > ( & I ) ) { for ( auto & U : Int -> args ( ) ) {

if ( isa < VectorType > ( U -> getType ( ) ) ) return false ;


<s> Instruction * Induction = nullptr ; if ( ! match ( I , m_ICmp ( Pred , m_Instruction ( Induction ) , m_Instruction ( Shuffle ) ) ) || Pred != ICmpInst :: ICMP_ULE || ! L -> isLoopInvariant ( Shuffle ) ) return false ; Instruction * Insert = nullptr ; if ( ! match ( Shuffle , m_ShuffleVector ( m_Instruction ( Insert ) , m_Undef ( ) , m_Zero ( ) ) ) ) return false ; Instruction * BECount = nullptr ; if ( ! match ( Insert , m_InsertElement ( m_Undef ( ) , m_Instruction ( BECount ) , m_Zero ( ) ) ) ) return false ; Value * TripCount = nullptr ; if ( ! match ( BECount , m_Add ( m_Value ( TripCount ) , m_AllOnes ( ) ) ) ) return false ; if ( TripCount != NumElements ) return false ; Instruction * BroadcastSplat = nullptr ; Constant * Const = nullptr ; if ( ! match ( Induction , m_Add ( m_Instruction ( BroadcastSplat ) , m_Constant ( Const ) ) ) ) return false ; if ( auto * CDS = dyn_cast < ConstantDataSequential > ( Const ) ) { for ( unsigned i = <NUM_LIT> ; i < CDS -> getNumElements ( ) ; ++ i ) { if ( CDS -> getElementAsInteger ( i ) != i ) return false ; } } else return false ; if ( ! match ( BroadcastSplat , m_ShuffleVector ( m_Instruction ( Insert ) , m_Undef ( ) , m_Zero ( ) ) ) ) return false ;


<s> Instruction * Induction = nullptr ; if ( ! match ( I , m_ICmp ( Pred , m_Instruction ( Induction ) , m_Instruction ( Shuffle ) ) ) || Pred != ICmpInst :: ICMP_ULE || ! L -> isLoopInvariant ( Shuffle ) ) return false ; Instruction * Insert = nullptr ; if ( ! match ( Shuffle , m_ShuffleVector ( m_Instruction ( Insert ) , m_Undef ( ) , m_Zero ( ) ) ) ) return false ; Instruction * BECount = nullptr ; if ( ! match ( Insert , m_InsertElement ( m_Undef ( ) , m_Instruction ( BECount ) , m_Zero ( ) ) ) ) return false ; Value * TripCount = nullptr ; if ( ! match ( BECount , m_Add ( m_Value ( TripCount ) , m_AllOnes ( ) ) ) ) return false ; if ( TripCount != NumElements ) return false ; Instruction * BroadcastSplat = nullptr ; Constant * Const = nullptr ; if ( ! match ( Induction , m_Add ( m_Instruction ( BroadcastSplat ) , m_Constant ( Const ) ) ) ) return false ; if ( auto * CDS = dyn_cast < ConstantDataSequential > ( Const ) ) { for ( unsigned i = <NUM_LIT> ; i < CDS -> getNumElements ( ) ; ++ i ) { if ( CDS -> getElementAsInteger ( i ) != i ) return false ; } } else return false ; if ( ! match ( BroadcastSplat , m_ShuffleVector ( m_Instruction ( Insert ) , m_Undef ( ) , m_Zero ( ) ) ) ) return false ; Instruction * IV = nullptr ;

if ( ! match ( Insert , m_InsertElement ( m_Undef ( ) , m_Instruction ( IV ) , m_Zero ( ) ) ) ) return false ;


<s> this -> L = L ; if ( ! ST -> hasMVEIntegerOps ( ) || ! ST -> hasV8_1MMainlineOps ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } BasicBlock * Preheader = L -> getLoopPreheader ( ) ; if ( ! Preheader ) return false ; auto FindLoopIterations = [ ] ( BasicBlock * BB ) -> IntrinsicInst * { for ( auto & I : * BB ) { auto * Call = dyn_cast < IntrinsicInst > ( & I ) ; if ( ! Call ) continue ; <STR_LIT> ID = Call -> getIntrinsicID ( ) ; if ( ID == <STR_LIT> || ID == <STR_LIT> ) return cast < IntrinsicInst > ( & I ) ; } return nullptr ; } ; IntrinsicInst * Setup = FindLoopIterations ( Preheader ) ; if ( ! Setup ) {

if ( ! Preheader -> getSinglePredecessor ( ) ) return false ;


<s> unsigned VectorWidth = cast < FixedVectorType > ( ActiveLaneMask -> getType ( ) ) -> getNumElements ( ) ; Builder . SetInsertPoint ( L -> getHeader ( ) -> getFirstNonPHI ( ) ) ; PHINode * Processed = Builder . CreatePHI ( Ty , <NUM_LIT> ) ; Processed -> addIncoming ( ActiveLaneMask -> getOperand ( <NUM_LIT> ) , L -> getLoopPreheader ( ) ) ; Builder . SetInsertPoint ( ActiveLaneMask ) ; ConstantInt * Factor = ConstantInt :: get ( cast < IntegerType > ( Ty ) , VectorWidth ) ;


<s> dbgs ( ) << <STR_LIT> ; EC -> dump ( ) ; dbgs ( ) << <STR_LIT> << VectorWidth << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; Ceil -> dump ( ) ; ) ; bool Zero = SE -> getMinusSCEV ( SE -> getBackedgeTakenCount ( L ) , SE -> getUDivExpr ( SE -> getAddExpr ( SE -> getMulExpr ( Ceil , VW ) , SE -> getNegativeSCEV ( VW ) ) , VW ) ) -> isZero ( ) ; if ( ! Zero ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } } auto * IV = ActiveLaneMask -> getOperand ( <NUM_LIT> ) ; auto * IVExpr = SE -> getSCEV ( IV ) ; auto * AddExpr = dyn_cast < SCEVAddRecExpr > ( IVExpr ) ; if ( ! AddExpr ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; IVExpr -> dump ( ) ) ; return false ; } if ( AddExpr -> getLoop ( ) != L ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto * Base = dyn_cast < SCEVConstant > ( AddExpr -> getOperand ( <NUM_LIT> ) ) ; if ( ! Base || ! Base -> isZero ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto * Step = dyn_cast < SCEVConstant > ( AddExpr -> getOperand ( <NUM_LIT> ) ) ; if ( ! Step ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; AddExpr -> getOperand ( <NUM_LIT> ) -> dump ( ) ) ; return false ; } auto StepValue = Step -> getValue ( ) -> getSExtValue ( ) ; if ( VectorWidth == StepValue ) return true ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << StepValue << <STR_LIT> << VectorWidth << <STR_LIT> ) ;


<s> BasicBlock * Preheader = L -> getLoopPreheader ( ) ; if ( ! Preheader ) return false ; auto FindLoopIterations = [ ] ( BasicBlock * BB ) -> IntrinsicInst * { for ( auto & I : * BB ) { auto * Call = dyn_cast < IntrinsicInst > ( & I ) ; if ( ! Call ) continue ; <STR_LIT> ID = Call -> getIntrinsicID ( ) ; if ( ID == <STR_LIT> || ID == <STR_LIT> ) return cast < IntrinsicInst > ( & I ) ; } return nullptr ; } ;

IntrinsicInst * Setup = FindLoopIterations ( Preheader ) ;


<s> MaskedInsts . clear ( ) ; Function & F = * L -> getHeader ( ) -> getParent ( ) ; auto & TPC = getAnalysis < TargetPassConfig > ( ) ; auto & TM = TPC . getTM < TargetMachine > ( ) ; ST = & TM . getSubtarget < <ISA_LIT>Subtarget > ( F ) ; TTI = & getAnalysis < TargetTransformInfoWrapperPass > ( ) . getTTI ( F ) ; SE = & getAnalysis < ScalarEvolutionWrapperPass > ( ) . getSE ( ) ; this -> L = L ; if ( ! ST -> hasMVEIntegerOps ( ) || ! ST -> hasV8_1MMainlineOps ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ;


<s> if ( ! Exit ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } bool ClonedVCTPInExitBlock = false ; for ( auto & Pair : NewPredicates ) { Instruction * OldPred = Pair . first ; Instruction * NewPred = Pair . second ; for ( auto & I : * Exit ) { if ( I . isSameOperationAs ( OldPred ) ) { Instruction * PredClone = NewPred -> clone ( ) ; PredClone -> insertBefore ( & I ) ; I . replaceAllUsesWith ( PredClone ) ; MaybeDead . insert ( & I ) ; ClonedVCTPInExitBlock = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; I . dump ( ) ; dbgs ( ) << <STR_LIT> ; PredClone -> dump ( ) ) ;


<s> if ( ! match ( Insert , m_InsertElement ( m_Undef ( ) , m_Instruction ( BECount ) , m_Zero ( ) ) ) ) return false ; Value * TripCount = nullptr ; if ( ! match ( BECount , m_Add ( m_Value ( TripCount ) , m_AllOnes ( ) ) ) ) return false ; if ( TripCount != NumElements || ! L -> isLoopInvariant ( BECount ) ) return false ;


<s> void MVETailPredication :: RematerializeIterCount ( ) { SmallVector < WeakTrackingVH , <NUM_LIT> > DeadInsts ; SCEVExpander Rewriter ( * SE , * DL , <STR_LIT> ) ;

ReplaceExitVal ReplaceExitValue = AlwaysRepl ;


<s> rewriteLoopExitValues ( L , LI , TLI , SE , Rewriter , DT , ReplaceExitValue , DeadInsts ) ;


<s> auto * ST = & TM . getSubtarget < <ISA_LIT>Subtarget > ( F ) ; DT = & getAnalysis < DominatorTreeWrapperPass > ( ) . getDomTree ( ) ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; TTI = & getAnalysis < TargetTransformInfoWrapperPass > ( ) . getTTI ( F ) ; SE = & getAnalysis < ScalarEvolutionWrapperPass > ( ) . getSE ( ) ; auto * TLIP = getAnalysisIfAvailable < TargetLibraryInfoWrapperPass > ( ) ; TLI = TLIP ? & TLIP -> getTLI ( * L -> getHeader ( ) -> getParent ( ) ) : nullptr ; DL = & L -> getHeader ( ) -> getModule ( ) -> getDataLayout ( ) ; this -> L = L ; if ( ! ST -> hasMVEIntegerOps ( ) || ! ST -> hasV8_1MMainlineOps ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } BasicBlock * Preheader = L -> getLoopPreheader ( ) ; if ( ! Preheader ) return false ; auto FindLoopIterations = [ ] ( BasicBlock * BB ) -> IntrinsicInst * { for ( auto & I : * BB ) { auto * Call = dyn_cast < IntrinsicInst > ( & I ) ; if ( ! Call ) continue ; <STR_LIT> ID = Call -> getIntrinsicID ( ) ; if ( ID == <STR_LIT> || ID == <STR_LIT> ) return cast < IntrinsicInst > ( & I ) ; } return nullptr ; } ; IntrinsicInst * Setup = FindLoopIterations ( Preheader ) ; if ( ! Setup ) { if ( ! Preheader -> getSinglePredecessor ( ) ) return false ; Setup = FindLoopIterations ( Preheader -> getSinglePredecessor ( ) ) ; if ( ! Setup ) return false ; } IntrinsicInst * Decrement = nullptr ; for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( IsDecrement ( I ) ) { Decrement = cast < IntrinsicInst > ( & I ) ; break ; } } } if ( ! Decrement ) return false ; ClonedVCTPInExitBlock = false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * L << * Setup << <STR_LIT> << * Decrement << <STR_LIT> ) ; if ( TryConvert ( Setup -> getArgOperand ( <NUM_LIT> ) ) ) { if ( ClonedVCTPInExitBlock ) RematerializeIterCount ( ) ; return true ; }


<s> auto & TPC = getAnalysis < TargetPassConfig > ( ) ; auto & TM = TPC . getTM < TargetMachine > ( ) ; auto * ST = & TM . getSubtarget < <ISA_LIT>Subtarget > ( F ) ; DT = & getAnalysis < DominatorTreeWrapperPass > ( ) . getDomTree ( ) ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; TTI = & getAnalysis < TargetTransformInfoWrapperPass > ( ) . getTTI ( F ) ; SE = & getAnalysis < ScalarEvolutionWrapperPass > ( ) . getSE ( ) ; auto * TLIP = getAnalysisIfAvailable < TargetLibraryInfoWrapperPass > ( ) ; TLI = TLIP ? & TLIP -> getTLI ( * L -> getHeader ( ) -> getParent ( ) ) : nullptr ; DL = & L -> getHeader ( ) -> getModule ( ) -> getDataLayout ( ) ; this -> L = L ; if ( ! ST -> hasMVEIntegerOps ( ) || ! ST -> hasV8_1MMainlineOps ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } BasicBlock * Preheader = L -> getLoopPreheader ( ) ; if ( ! Preheader ) return false ; auto FindLoopIterations = [ ] ( BasicBlock * BB ) -> IntrinsicInst * { for ( auto & I : * BB ) { auto * Call = dyn_cast < IntrinsicInst > ( & I ) ; if ( ! Call ) continue ; <STR_LIT> ID = Call -> getIntrinsicID ( ) ; if ( ID == <STR_LIT> || ID == <STR_LIT> ) return cast < IntrinsicInst > ( & I ) ;