<s> auto * Predicate = dyn_cast < Instruction > ( I -> getArgOperand ( PredOp ) ) ; if ( ! Predicate || Predicates . count ( Predicate ) ) continue ; TripCountPattern TCP ( Predicate , TripCount , getVectorType ( I ) ) ; if ( ! ( ComputeConstElements ( TCP ) || ComputeRuntimeElements ( TCP ) ) ) continue ; if ( ! isTailPredicate ( TCP ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; Predicates . insert ( Predicate ) ; InsertVCTPIntrinsic ( TCP , NewPredicates ) ; }

if ( ! NewPredicates . size ( ) ) return false ;


<s> IRBuilder < > Builder ( L -> getLoopPreheader ( ) -> getTerminator ( ) ) ; Module * M = L -> getHeader ( ) -> getModule ( ) ; Type * Ty = IntegerType :: get ( M -> getContext ( ) , <NUM_LIT> ) ; unsigned VectorWidth = cast < FixedVectorType > ( ActiveLaneMask -> getType ( ) ) -> getNumElements ( ) ; Builder . SetInsertPoint ( L -> getHeader ( ) -> getFirstNonPHI ( ) ) ; PHINode * Processed = Builder . CreatePHI ( Ty , <NUM_LIT> ) ; Processed -> addIncoming ( ActiveLaneMask -> getOperand ( <NUM_LIT> ) , L -> getLoopPreheader ( ) ) ; Builder . SetInsertPoint ( ActiveLaneMask ) ; ConstantInt * Factor = ConstantInt :: get ( cast < IntegerType > ( Ty ) , VectorWidth ) ; <STR_LIT> VCTPID ; switch ( VectorWidth ) { default : llvm_unreachable ( <STR_LIT> ) ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; } Function * VCTP = <STR_LIT> ( M , VCTPID ) ;

Value * VCTPCall = Builder . CreateCall ( VCTP , Processed ) ;


<s> case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; } Function * VCTP = <STR_LIT> ( M , VCTPID ) ; Value * VCTPCall = Builder . CreateCall ( VCTP , Processed ) ; ActiveLaneMask -> replaceAllUsesWith ( VCTPCall ) ; Value * Remaining = Builder . CreateSub ( Processed , Factor ) ; Processed -> addIncoming ( Remaining , L -> getLoopLatch ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Processed << <STR_LIT> << <STR_LIT> << * VCTPCall << <STR_LIT> ) ;


<s> } if ( IsMasked ( & I ) ) { auto * VecTy = getVectorType ( Int ) ; unsigned Lanes = VecTy -> getNumElements ( ) ; unsigned ElementWidth = VecTy -> getScalarSizeInBits ( ) ; unsigned MaxWidth = TTI -> getRegisterBitWidth ( true ) ; if ( Lanes * ElementWidth > MaxWidth || Lanes == MaxWidth ) return false ; MaskedInsts . push_back ( cast < IntrinsicInst > ( & I ) ) ; continue ; } for ( const Use & U : Int -> args ( ) ) { if ( isa < VectorType > ( U -> getType ( ) ) ) return false ; } }


<s> case <STR_LIT> : case <STR_LIT> : continue ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( ST -> hasMVEFloatOps ( ) ) continue ; LLVM_FALLTHROUGH ; default : break ; } if ( IsMasked ( & I ) ) { auto * VecTy = getVectorType ( Int ) ; unsigned Lanes = VecTy -> getNumElements ( ) ; unsigned ElementWidth = VecTy -> getScalarSizeInBits ( ) ; unsigned MaxWidth = TTI -> getRegisterBitWidth ( true ) ; if ( Lanes * ElementWidth > MaxWidth || Lanes == MaxWidth ) return false ; MaskedInsts . push_back ( cast < IntrinsicInst > ( & I ) ) ; continue ; } for ( const Use & U : Int -> args ( ) ) { if ( isa < VectorType > ( U -> getType ( ) ) ) return false ; } } } if ( ! ActiveLaneMask ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ;


<s> uint64_t UpperboundTC = SE -> getSignedRange ( TC ) . getUpper ( ) . getZExtValue ( ) ; if ( UpperboundTC > MaxMinusVW && ! ForceTailPredication ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; dbgs ( ) << UpperboundTC << <STR_LIT> << MaxMinusVW << <STR_LIT> ; ) ; return false ; } auto * One = SE -> getOne ( TripCount -> getType ( ) ) ; auto * ElementCount = SE -> getAddExpr ( BTC , One ) ; auto * ECPlusVWMinus1 = SE -> getAddExpr ( ElementCount , SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth - <NUM_LIT> ) ) ) ; auto * Ceil = SE -> getUDivExpr ( ECPlusVWMinus1 , SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth ) ) ) ; ConstantRange RangeCeil = SE -> getSignedRange ( Ceil ) ; ConstantRange RangeTC = SE -> getSignedRange ( TC ) ; if ( ! RangeTC . isSingleElement ( ) ) { auto ZeroRange = ConstantRange ( APInt ( TripCount -> getType ( ) -> getScalarSizeInBits ( ) , <NUM_LIT> ) ) ; RangeTC = RangeTC . unionWith ( ZeroRange ) ; } if ( ! RangeTC . contains ( RangeCeil ) && ! ForceTailPredication ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto * IV = ActiveLaneMask -> getOperand ( <NUM_LIT> ) ; auto * IVExpr = SE -> getSCEV ( IV ) ; auto * AddExpr = dyn_cast < SCEVAddRecExpr > ( IVExpr ) ; if ( ! AddExpr ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; IVExpr -> dump ( ) ) ; return false ; } if ( AddExpr -> getLoop ( ) != L ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto * Step = dyn_cast < SCEVConstant > ( AddExpr -> getOperand ( <NUM_LIT> ) ) ; if ( ! Step ) {


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ActiveLaneMask << <STR_LIT> ) ; auto * VecTy = getVectorType ( I ) ; if ( ! IsSafeActiveMask ( ActiveLaneMask , TripCount , VecTy ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; InsertVCTPIntrinsic ( ActiveLaneMask , TripCount , VecTy ) ; } Cleanup ( Predicates , L ) ;


<s> BasicBlock * Preheader = L -> getLoopPreheader ( ) ; if ( ! Preheader ) return false ; auto FindLoopIterations = [ ] ( BasicBlock * BB ) -> IntrinsicInst * { for ( auto & I : * BB ) { auto * Call = dyn_cast < IntrinsicInst > ( & I ) ; if ( ! Call ) continue ; <STR_LIT> ID = Call -> getIntrinsicID ( ) ; if ( ID == <STR_LIT> || ID == <STR_LIT> ) return cast < IntrinsicInst > ( & I ) ; } return nullptr ; } ; IntrinsicInst * Setup = FindLoopIterations ( Preheader ) ;


<s> auto & TM = TPC . getTM < TargetMachine > ( ) ; ST = & TM . getSubtarget < <ISA_LIT>Subtarget > ( F ) ; TTI = & getAnalysis < TargetTransformInfoWrapperPass > ( ) . getTTI ( F ) ; SE = & getAnalysis < ScalarEvolutionWrapperPass > ( ) . getSE ( ) ; this -> L = L ; if ( ! ST -> hasMVEIntegerOps ( ) || ! ST -> hasV8_1MMainlineOps ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } BasicBlock * Preheader = L -> getLoopPreheader ( ) ; if ( ! Preheader ) return false ; auto FindLoopIterations = [ ] ( BasicBlock * BB ) -> IntrinsicInst * { for ( auto & I : * BB ) {

auto * Call = dyn_cast < IntrinsicInst > ( & I ) ;


<s> BasicBlock * Exit = L -> getUniqueExitBlock ( ) ; if ( ! Exit ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return ; } SmallPtrSet < Instruction * , <NUM_LIT> > Dead ; while ( ! MaybeDead . empty ( ) ) { auto * I = MaybeDead . front ( ) ; MaybeDead . remove ( I ) ; if ( I -> hasNUsesOrMore ( <NUM_LIT> ) ) continue ; for ( auto & U : I -> operands ( ) ) if ( auto * OpI = dyn_cast < Instruction > ( U ) ) MaybeDead . insert ( OpI ) ; Dead . insert ( I ) ;


<s> AU . addRequired < LoopInfoWrapperPass > ( ) ; AU . addRequired < TargetPassConfig > ( ) ;

AU . addRequired < TargetTransformInfoWrapperPass > ( ) ;


<s> if ( ID == <STR_LIT> || ID == <STR_LIT> ) VecTy = dyn_cast < FixedVectorType > ( I -> getType ( ) -> getContainedType ( <NUM_LIT> ) ) ; else VecTy = dyn_cast < FixedVectorType > ( I -> getType ( ) ) ; } else if ( ID == <STR_LIT> ) { VecTy = dyn_cast < FixedVectorType > ( I -> getOperand ( <NUM_LIT> ) -> getType ( ) ) ; } else { VecTy = dyn_cast < FixedVectorType > ( I -> getOperand ( <NUM_LIT> ) -> getType ( ) ) ; } assert ( VecTy && <STR_LIT> ) ;


<s> static bool IsMasked ( Instruction * I ) {

auto * Call = dyn_cast < IntrinsicInst > ( I ) ;


<s> if ( ST -> hasMVEFloatOps ( ) ) continue ; break ; default : break ; } if ( IsMasked ( & I ) ) { auto * VecTy = getVectorType ( Int ) ; unsigned Lanes = VecTy -> getNumElements ( ) ; unsigned ElementWidth = VecTy -> getScalarSizeInBits ( ) ; unsigned MaxWidth = TTI -> getRegisterBitWidth ( true ) ; if ( Lanes * ElementWidth > MaxWidth || Lanes == MaxWidth ) return false ; MaskedInsts . push_back ( cast < IntrinsicInst > ( & I ) ) ; continue ; } for ( const Use & U : Int -> args ( ) ) { if ( isa < VectorType > ( U -> getType ( ) ) ) return false ; } } }


<s> case <STR_LIT> : case <STR_LIT> : if ( ST -> hasMVEFloatOps ( ) ) continue ; break ; default : break ; } if ( IsMasked ( & I ) ) { auto * VecTy = getVectorType ( Int ) ; unsigned Lanes = VecTy -> getNumElements ( ) ; unsigned ElementWidth = VecTy -> getScalarSizeInBits ( ) ; unsigned MaxWidth = TTI -> getRegisterBitWidth ( true ) ; if ( Lanes * ElementWidth > MaxWidth || Lanes == MaxWidth ) return false ; MaskedInsts . push_back ( cast < IntrinsicInst > ( & I ) ) ; continue ; } for ( const Use & U : Int -> args ( ) ) { if ( isa < VectorType > ( U -> getType ( ) ) ) return false ; }


<s> bool ForceTailPredication = EnableTailPredication == TailPredication :: ForceEnabledNoReductions || EnableTailPredication == TailPredication :: ForceEnabled ; Value * ElemCount = ActiveLaneMask -> getOperand ( <NUM_LIT> ) ; auto * EC = SE -> getSCEV ( ElemCount ) ; auto * TC = SE -> getSCEV ( TripCount ) ; int VectorWidth = VecTy -> getNumElements ( ) ; ConstantInt * ConstElemCount = nullptr ; if ( ! SE -> isLoopInvariant ( EC , L ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( ( ConstElemCount = dyn_cast < ConstantInt > ( ElemCount ) ) ) { ConstantInt * TC = dyn_cast < ConstantInt > ( TripCount ) ; if ( ! TC ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } uint64_t TC1 = TC -> getZExtValue ( ) * VectorWidth ; uint64_t TC2 = ConstElemCount -> getZExtValue ( ) + <NUM_LIT> ; if ( TC1 != TC2 ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << TC1 << <STR_LIT> << TC2 << <STR_LIT> ) ; return false ; } } else if ( ! ForceTailPredication ) { auto * VW = SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth ) ) ; auto * ECPlusVWMinus1 = SE -> getAddExpr ( EC , SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth - <NUM_LIT> ) ) ) ; auto * Ceil = SE -> getUDivExpr ( ECPlusVWMinus1 , VW ) ; ( void ) TC ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; TC -> dump ( ) ;

dbgs ( ) << <STR_LIT> ; EC -> dump ( ) ;


<s> bool MVETailPredication :: runOnLoop ( Loop * L , LPPassManager & ) { if ( skipLoop ( L ) || ! EnableTailPredication ) return false ; MaskedInsts . clear ( ) ; Function & F = * L -> getHeader ( ) -> getParent ( ) ; auto & TPC = getAnalysis < TargetPassConfig > ( ) ; auto & TM = TPC . getTM < TargetMachine > ( ) ; ST = & TM . getSubtarget < <ISA_LIT>Subtarget > ( F ) ; TTI = & getAnalysis < TargetTransformInfoWrapperPass > ( ) . getTTI ( F ) ; SE = & getAnalysis < ScalarEvolutionWrapperPass > ( ) . getSE ( ) ; this -> L = L ; if ( ! ST -> hasMVEIntegerOps ( ) || ! ST -> hasV8_1MMainlineOps ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } BasicBlock * Preheader = L -> getLoopPreheader ( ) ; if ( ! Preheader ) return false ; auto FindLoopIterations = [ ] ( BasicBlock * BB ) -> IntrinsicInst * { for ( auto & I : * BB ) { auto * Call = dyn_cast < IntrinsicInst > ( & I ) ; if ( ! Call ) continue ; <STR_LIT> ID = Call -> getIntrinsicID ( ) ; if ( ID == <STR_LIT> || ID == <STR_LIT> ) return cast < IntrinsicInst > ( & I ) ; } return nullptr ; } ; IntrinsicInst * Setup = FindLoopIterations ( Preheader ) ; if ( ! Setup ) { if ( ! Preheader -> getSinglePredecessor ( ) ) return false ; Setup = FindLoopIterations ( Preheader -> getSinglePredecessor ( ) ) ; if ( ! Setup ) return false ; }


<s> bool MVETailPredication :: TryConvert ( Value * TripCount ) { if ( ! IsPredicatedVectorLoop ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; SetVector < Instruction * > Predicates ; auto getPredicateOp = [ ] ( IntrinsicInst * I ) { unsigned IntrinsicID = I -> getIntrinsicID ( ) ; if ( IntrinsicID == <STR_LIT> || IntrinsicID == <STR_LIT> ) return <NUM_LIT> ; return ( IntrinsicID == <STR_LIT> || isGather ( I ) ) ? <NUM_LIT> : <NUM_LIT> ; } ; for ( auto * I : MaskedInsts ) { Value * PredOp = I -> getArgOperand ( getPredicateOp ( I ) ) ; auto * Predicate = dyn_cast < Instruction > ( PredOp ) ; if ( ! Predicate || Predicates . count ( Predicate ) ) continue ; auto * ActiveLaneMask = dyn_cast < IntrinsicInst > ( Predicate ) ; if ( ! ActiveLaneMask || ActiveLaneMask -> getIntrinsicID ( ) != <STR_LIT> ) continue ; Predicates . insert ( Predicate ) ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ActiveLaneMask << <STR_LIT> ) ;


<s> if ( ! match ( BroadcastSplat , m_Shuffle ( m_Instruction ( Insert ) , m_Undef ( ) , m_ZeroMask ( ) ) ) ) return false ; Instruction * IV = nullptr ; if ( ! match ( Insert , m_InsertElt ( m_Undef ( ) , m_Instruction ( IV ) , m_Zero ( ) ) ) ) return false ; auto * Phi = dyn_cast < PHINode > ( IV ) ; if ( ! Phi ) return false ; Value * OnEntry = Phi -> getIncomingValueForBlock ( L -> getLoopPreheader ( ) ) ; if ( ! match ( OnEntry , m_Zero ( ) ) ) return false ; Value * InLoop = Phi -> getIncomingValueForBlock ( L -> getLoopLatch ( ) ) ;

unsigned Lanes = cast < FixedVectorType > ( Insert -> getType ( ) ) -> getNumElements ( ) ;


<s> } else return false ; Instruction * Insert = nullptr ; if ( ! match ( BroadcastSplat , m_Shuffle ( m_Instruction ( Insert ) , m_Undef ( ) , m_ZeroMask ( ) ) ) ) return false ; Instruction * IV = nullptr ; if ( ! match ( Insert , m_InsertElt ( m_Undef ( ) , m_Instruction ( IV ) , m_Zero ( ) ) ) ) return false ; auto * Phi = dyn_cast < PHINode > ( IV ) ; if ( ! Phi ) return false ; Value * OnEntry = Phi -> getIncomingValueForBlock ( L -> getLoopPreheader ( ) ) ; if ( ! match ( OnEntry , m_Zero ( ) ) ) return false ; Value * InLoop = Phi -> getIncomingValueForBlock ( L -> getLoopLatch ( ) ) ; unsigned Lanes = cast < FixedVectorType > ( Insert -> getType ( ) ) -> getNumElements ( ) ;


<s> Value * TripCount = nullptr ; if ( ! match ( BECount , m_Add ( m_Value ( TripCount ) , m_AllOnes ( ) ) ) ) return false ; if ( TripCount != NumElements || ! L -> isLoopInvariant ( BECount ) ) return false ;


<s> Instruction * Insert = nullptr ; if ( ! match ( Shuffle , m_Shuffle ( m_Instruction ( Insert ) , m_Undef ( ) , m_ZeroMask ( ) ) ) ) return false ; Instruction * BECount = nullptr ;

if ( ! match ( Insert , m_InsertElt ( m_Undef ( ) , m_Instruction ( BECount ) , m_Zero ( ) ) ) ) return false ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; TC -> dump ( ) ; dbgs ( ) << <STR_LIT> ; EC -> dump ( ) ; dbgs ( ) << <STR_LIT> << VectorWidth << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; Ceil -> dump ( ) ; ) ; bool Zero = SE -> getMinusSCEV ( SE -> getBackedgeTakenCount ( L ) , SE -> getUDivExpr ( SE -> getAddExpr ( SE -> getMulExpr ( Ceil , VW ) , SE -> getNegativeSCEV ( VW ) ) , VW ) ) -> isZero ( ) ; if ( ! Zero ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } } auto * IV = ActiveLaneMask -> getOperand ( <NUM_LIT> ) ; auto * IVExpr = SE -> getSCEV ( IV ) ; auto * AddExpr = dyn_cast < SCEVAddRecExpr > ( IVExpr ) ; if ( ! AddExpr ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; IVExpr -> dump ( ) ) ; return false ; } if ( AddExpr -> getLoop ( ) != L ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto * Base = dyn_cast < SCEVConstant > ( AddExpr -> getOperand ( <NUM_LIT> ) ) ; if ( ! Base || ! Base -> isZero ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto * Step = dyn_cast < SCEVConstant > ( AddExpr -> getOperand ( <NUM_LIT> ) ) ; if ( ! Step ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; AddExpr -> getOperand ( <NUM_LIT> ) -> dump ( ) ) ; return false ; } auto StepValue = Step -> getValue ( ) -> getSExtValue ( ) ; if ( VectorWidth == StepValue ) return true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << StepValue << <STR_LIT> << VectorWidth << <STR_LIT> ) ;


<s> DenseMap < Instruction * , Instruction * > NewPredicates ; for ( auto * I : MaskedInsts ) { <STR_LIT> ID = I -> getIntrinsicID ( ) ; unsigned PredOp = ID == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ; auto * Predicate = dyn_cast < Instruction > ( I -> getArgOperand ( PredOp ) ) ; if ( ! Predicate || Predicates . count ( Predicate ) ) continue ; VectorType * VecTy = getVectorType ( I ) ; Value * NumElements = ComputeElements ( TripCount , VecTy ) ; if ( ! NumElements ) continue ; if ( ! isTailPredicate ( Predicate , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; Predicates . insert ( Predicate ) ; IRBuilder < > Builder ( L -> getHeader ( ) -> getFirstNonPHI ( ) ) ; PHINode * Processed = Builder . CreatePHI ( Ty , <NUM_LIT> ) ; Processed -> addIncoming ( NumElements , L -> getLoopPreheader ( ) ) ; Builder . SetInsertPoint ( cast < Instruction > ( Predicate ) ) ; ConstantInt * Factor = ConstantInt :: get ( cast < IntegerType > ( Ty ) , VecTy -> getNumElements ( ) ) ; <STR_LIT> VCTPID ; switch ( VecTy -> getNumElements ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; } Function * VCTP = <STR_LIT> ( M , VCTPID ) ; Value * TailPredicate = Builder . CreateCall ( VCTP , Processed ) ; Predicate -> replaceAllUsesWith ( TailPredicate ) ; NewPredicates [ Predicate ] = cast < Instruction > ( TailPredicate ) ; Value * Remaining = Builder . CreateSub ( Processed , Factor ) ; Processed -> addIncoming ( Remaining , L -> getLoopLatch ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Processed << <STR_LIT> << <STR_LIT> << * TailPredicate << <STR_LIT> ) ; } Cleanup ( NewPredicates , Predicates , L ) ;


<s> continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; Predicates . insert ( Predicate ) ; IRBuilder < > Builder ( L -> getHeader ( ) -> getFirstNonPHI ( ) ) ; PHINode * Processed = Builder . CreatePHI ( Ty , <NUM_LIT> ) ; Processed -> addIncoming ( NumElements , L -> getLoopPreheader ( ) ) ; Builder . SetInsertPoint ( cast < Instruction > ( Predicate ) ) ; ConstantInt * Factor = ConstantInt :: get ( cast < IntegerType > ( Ty ) , VecTy -> getNumElements ( ) ) ; <STR_LIT> VCTPID ; switch ( VecTy -> getNumElements ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; } Function * VCTP = <STR_LIT> ( M , VCTPID ) ; Value * TailPredicate = Builder . CreateCall ( VCTP , Processed ) ; Predicate -> replaceAllUsesWith ( TailPredicate ) ; NewPredicates [ Predicate ] = cast < Instruction > ( TailPredicate ) ; Value * Remaining = Builder . CreateSub ( Processed , Factor ) ;

Processed -> addIncoming ( Remaining , L -> getLoopLatch ( ) ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return ; } for ( auto & Pair : NewPredicates ) { Instruction * OldPred = Pair . first ; Instruction * NewPred = Pair . second ; for ( auto & I : * Exit ) { if ( I . isSameOperationAs ( OldPred ) ) { Instruction * PredClone = NewPred -> clone ( ) ; PredClone -> insertBefore ( & I ) ; I . replaceAllUsesWith ( PredClone ) ; MaybeDead . insert ( & I ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; I . dump ( ) ; dbgs ( ) << <STR_LIT> ; PredClone -> dump ( ) ) ;


<s> Type * Ty = IntegerType :: get ( M -> getContext ( ) , <NUM_LIT> ) ; PHINode * Processed = Builder . CreatePHI ( Ty , <NUM_LIT> ) ; Processed -> addIncoming ( NumElements , L -> getLoopPreheader ( ) ) ; Builder . SetInsertPoint ( cast < Instruction > ( Predicate ) ) ; ConstantInt * Factor = ConstantInt :: get ( cast < IntegerType > ( Ty ) , VecTy -> getNumElements ( ) ) ; <STR_LIT> VCTPID ; switch ( VecTy -> getNumElements ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; } Function * VCTP = <STR_LIT> ( M , VCTPID ) ; Value * TailPredicate = Builder . CreateCall ( VCTP , Processed ) ; Predicate -> replaceAllUsesWith ( TailPredicate ) ;

NewPredicates [ Predicate ] = cast < Instruction > ( TailPredicate ) ;


<s> Instruction * BroadcastSplat = nullptr ; Constant * Const = nullptr ; if ( ! match ( Induction , m_Add ( m_Instruction ( BroadcastSplat ) , m_Constant ( Const ) ) ) ) return false ; if ( auto * CDS = dyn_cast < ConstantDataSequential > ( Const ) ) { for ( unsigned i = <NUM_LIT> ; i < CDS -> getNumElements ( ) ; ++ i ) { if ( CDS -> getElementAsInteger ( i ) != i ) return false ; } } else return false ; if ( ! match ( BroadcastSplat , m_ShuffleVector ( m_Instruction ( Insert ) , m_Undef ( ) , m_Zero ( ) ) ) ) return false ; Instruction * IV = nullptr ;

if ( ! match ( Insert , m_InsertElement ( m_Undef ( ) , m_Instruction ( IV ) , m_Zero ( ) ) ) ) return false ;


<s> return false ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; SetVector < Instruction * > Predicates ; DenseMap < Instruction * , Instruction * > NewPredicates ; for ( auto * I : MaskedInsts ) { <STR_LIT> ID = I -> getIntrinsicID ( ) ; unsigned PredOp = ID == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ; auto * Predicate = dyn_cast < Instruction > ( I -> getArgOperand ( PredOp ) ) ; if ( ! Predicate || Predicates . count ( Predicate ) ) continue ; VectorType * VecTy = getVectorType ( I ) ; Value * NumElements = ComputeElements ( TripCount , VecTy ) ; if ( ! NumElements ) continue ; if ( ! isTailPredicate ( Predicate , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; Predicates . insert ( Predicate ) ;

InsertVCTPIntrinsic ( Predicate , NewPredicates , VecTy , NumElements ) ;


<s> DenseMap < Instruction * , Instruction * > NewPredicates ; for ( auto * I : MaskedInsts ) { <STR_LIT> ID = I -> getIntrinsicID ( ) ; unsigned PredOp = ID == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ; auto * Predicate = dyn_cast < Instruction > ( I -> getArgOperand ( PredOp ) ) ; if ( ! Predicate || Predicates . count ( Predicate ) ) continue ; VectorType * VecTy = getVectorType ( I ) ; Value * NumElements = ComputeElements ( TripCount , VecTy ) ; if ( ! NumElements ) continue ; if ( ! isTailPredicate ( Predicate , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; continue ; }


<s> auto FindLoopIterations = [ ] ( BasicBlock * BB ) -> IntrinsicInst * { for ( auto & I : * BB ) { auto * Call = dyn_cast < IntrinsicInst > ( & I ) ; if ( ! Call ) continue ; <STR_LIT> ID = Call -> getIntrinsicID ( ) ; if ( ID == <STR_LIT> || ID == <STR_LIT> ) return cast < IntrinsicInst > ( & I ) ; } return nullptr ; } ; IntrinsicInst * Setup = FindLoopIterations ( Preheader ) ; if ( ! Setup ) { if ( ! Preheader -> getSinglePredecessor ( ) ) return false ; Setup = FindLoopIterations ( Preheader -> getSinglePredecessor ( ) ) ; if ( ! Setup ) return false ; } IntrinsicInst * Decrement = nullptr ; for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( IsDecrement ( I ) ) { Decrement = cast < IntrinsicInst > ( & I ) ; break ; }


<s> if ( ! isTailPredicate ( Predicate , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; Predicates . insert ( Predicate ) ; InsertVCTPIntrinsic ( Predicate , NewPredicates , VecTy , NumElements ) ; }

ClonedVCTPInExitBlock = Cleanup ( NewPredicates , Predicates , L ) ;


<s> bool MVETailPredication :: IsPredicatedVectorLoop ( ) { for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( IsMasked ( & I ) ) { VectorType * VecTy = getVectorType ( cast < IntrinsicInst > ( & I ) ) ; unsigned Lanes = VecTy -> getNumElements ( ) ;

unsigned ElementWidth = VecTy -> getScalarSizeInBits ( ) ;


<s> Instruction * Induction = nullptr ; if ( ! match ( V , m_ICmp ( Pred , m_Instruction ( Induction ) , m_Instruction ( Shuffle ) ) ) || Pred != ICmpInst :: ICMP_ULE || ! L -> isLoopInvariant ( Shuffle ) ) return false ; Instruction * Insert = nullptr ; if ( ! match ( Shuffle , m_ShuffleVector ( m_Instruction ( Insert ) , m_Undef ( ) , m_Zero ( ) ) ) ) return false ; Instruction * BECount = nullptr ; if ( ! match ( Insert , m_InsertElement ( m_Undef ( ) , m_Instruction ( BECount ) , m_Zero ( ) ) ) ) return false ; Value * TripCount = nullptr ; if ( ! match ( BECount , m_Add ( m_Value ( TripCount ) , m_AllOnes ( ) ) ) ) return false ; if ( TripCount != NumElements ) return false ; Instruction * BroadcastSplat = nullptr ; Constant * Const = nullptr ; if ( ! match ( Induction , m_Add ( m_Instruction ( BroadcastSplat ) , m_Constant ( Const ) ) ) ) return false ; if ( auto * CDS = dyn_cast < ConstantDataSequential > ( Const ) ) { for ( unsigned i = <NUM_LIT> ; i < CDS -> getNumElements ( ) ; ++ i ) { if ( CDS -> getElementAsInteger ( i ) != i ) return false ; } } else return false ; if ( ! match ( BroadcastSplat , m_ShuffleVector ( m_Instruction ( Insert ) , m_Undef ( ) , m_Zero ( ) ) ) ) return false ; Instruction * IV = nullptr ; if ( ! match ( Insert , m_InsertElement ( m_Undef ( ) , m_Instruction ( IV ) , m_Zero ( ) ) ) ) return false ; auto * Phi = dyn_cast < PHINode > ( IV ) ; if ( ! Phi ) return false ; Value * OnEntry = Phi -> getIncomingValueForBlock ( L -> getLoopPreheader ( ) ) ; if ( ! match ( OnEntry , m_Zero ( ) ) ) return false ; Value * InLoop = Phi -> getIncomingValueForBlock ( L -> getLoopLatch ( ) ) ; unsigned Lanes = cast < VectorType > ( Insert -> getType ( ) ) -> getNumElements ( ) ; Instruction * LHS = nullptr ; if ( ! match ( InLoop , m_Add ( m_Instruction ( LHS ) , m_SpecificInt ( Lanes ) ) ) ) return false ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Predicate << <STR_LIT> ) ; Predicates . insert ( Predicate ) ; IRBuilder < > Builder ( L -> getHeader ( ) -> getFirstNonPHI ( ) ) ; PHINode * Processed = Builder . CreatePHI ( Ty , <NUM_LIT> ) ; Processed -> addIncoming ( NumElements , L -> getLoopPreheader ( ) ) ; Builder . SetInsertPoint ( cast < Instruction > ( Predicate ) ) ; ConstantInt * Factor = ConstantInt :: get ( cast < IntegerType > ( Ty ) , VecTy -> getNumElements ( ) ) ; <STR_LIT> VCTPID ; switch ( VecTy -> getNumElements ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ;


<s> Processed -> addIncoming ( NumElements , L -> getLoopPreheader ( ) ) ; Builder . SetInsertPoint ( cast < Instruction > ( Predicate ) ) ; ConstantInt * Factor = ConstantInt :: get ( cast < IntegerType > ( Ty ) , VecTy -> getNumElements ( ) ) ; <STR_LIT> VCTPID ; switch ( VecTy -> getNumElements ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; } Function * VCTP = <STR_LIT> ( M , VCTPID ) ; Value * Remaining = Builder . CreateSub ( Processed , Factor ) ; Value * TailPredicate = Builder . CreateCall ( VCTP , Remaining ) ; Predicate -> replaceAllUsesWith ( TailPredicate ) ; Processed -> addIncoming ( Remaining , L -> getLoopLatch ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Processed << <STR_LIT> << <STR_LIT> << * TailPredicate << <STR_LIT> ) ; } for ( auto I : L -> blocks ( ) ) DeleteDeadPHIs ( I ) ; return true ;


<s> if ( VectorWidth != <NUM_LIT> && VectorWidth != <NUM_LIT> && VectorWidth != <NUM_LIT> ) return false ; ConstantInt * ConstElemCount = nullptr ; if ( ! SE -> isLoopInvariant ( EC , L ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( ( ConstElemCount = dyn_cast < ConstantInt > ( ElemCount ) ) ) { ConstantInt * TC = dyn_cast < ConstantInt > ( TripCount ) ; if ( ! TC ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } uint64_t TC1 = TC -> getZExtValue ( ) ; uint64_t TC2 = ( ConstElemCount -> getZExtValue ( ) + VectorWidth - <NUM_LIT> ) / VectorWidth ; if ( TC1 != TC2 ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << TC1 << <STR_LIT> << TC2 << <STR_LIT> ) ; return false ; } } else if ( ! ForceTailPredication ) { auto * VW = SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth ) ) ; auto * ECPlusVWMinus1 = SE -> getAddExpr ( EC , SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth - <NUM_LIT> ) ) ) ; auto * Ceil = SE -> getUDivExpr ( ECPlusVWMinus1 , VW ) ; ( void ) TC ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; TC -> dump ( ) ; dbgs ( ) << <STR_LIT> ; EC -> dump ( ) ; dbgs ( ) << <STR_LIT> << VectorWidth << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; Ceil -> dump ( ) ; ) ; bool Zero = SE -> getMinusSCEV ( SE -> getBackedgeTakenCount ( L ) , SE -> getUDivExpr ( SE -> getAddExpr ( SE -> getMulExpr ( Ceil , VW ) , SE -> getNegativeSCEV ( VW ) ) , VW ) ) -> isZero ( ) ; if ( ! Zero ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ;


<s> ConstantInt * ConstElemCount = nullptr ; if ( ! SE -> isLoopInvariant ( EC , L ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( ( ConstElemCount = dyn_cast < ConstantInt > ( ElemCount ) ) ) { ConstantInt * TC = dyn_cast < ConstantInt > ( TripCount ) ; if ( ! TC ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } uint64_t TC1 = TC -> getZExtValue ( ) ; uint64_t TC2 = ( ConstElemCount -> getZExtValue ( ) + VectorWidth - <NUM_LIT> ) / VectorWidth ; if ( TC1 != TC2 ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << TC1 << <STR_LIT> << TC2 << <STR_LIT> ) ; return false ; } } else if ( ! ForceTailPredication ) { auto * VW = SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth ) ) ; auto * ECPlusVWMinus1 = SE -> getAddExpr ( EC , SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth - <NUM_LIT> ) ) ) ; auto * Ceil = SE -> getUDivExpr ( ECPlusVWMinus1 , VW ) ; ( void ) TC ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ;

dbgs ( ) << <STR_LIT> ; TC -> dump ( ) ;


<s> } } } SmallPtrSet < Instruction * , <NUM_LIT> > Dead ; while ( ! MaybeDead . empty ( ) ) { auto * I = MaybeDead . front ( ) ; MaybeDead . remove ( I ) ; if ( I -> hasNUsesOrMore ( <NUM_LIT> ) ) continue ; for ( auto & U : I -> operands ( ) ) if ( auto * OpI = dyn_cast < Instruction > ( U ) ) MaybeDead . insert ( OpI ) ; Dead . insert ( I ) ; } for ( auto * I : Dead ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; I -> dump ( ) ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } bool ClonedVCTPInExitBlock = false ; for ( auto & Pair : NewPredicates ) { Instruction * OldPred = Pair . first ; Instruction * NewPred = Pair . second ; for ( auto & I : * Exit ) { if ( I . isSameOperationAs ( OldPred ) ) { Instruction * PredClone = NewPred -> clone ( ) ; PredClone -> insertBefore ( & I ) ; I . replaceAllUsesWith ( PredClone ) ; MaybeDead . insert ( & I ) ; ClonedVCTPInExitBlock = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; I . dump ( ) ; dbgs ( ) << <STR_LIT> ; PredClone -> dump ( ) ) ; break ; } } } SmallPtrSet < Instruction * , <NUM_LIT> > Dead ; while ( ! MaybeDead . empty ( ) ) { auto * I = MaybeDead . front ( ) ; MaybeDead . remove ( I ) ;

if ( I -> hasNUsesOrMore ( <NUM_LIT> ) ) continue ;


<s> Pass * llvm :: createMVETailPredicationPass ( ) {


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < ScalarEvolutionWrapperPass > ( ) ; AU . addRequired < LoopInfoWrapperPass > ( ) ;

AU . addRequired < TargetPassConfig > ( ) ;


<s> IRBuilder < > Builder ( Preheader -> getTerminator ( ) ) ; Value * NumElements = Builder . CreateAdd ( BTC , ConstantInt :: get ( BTC -> getType ( ) , <NUM_LIT> ) , <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * NumElements << <STR_LIT> ) ; return NumElements ;


<s> auto * PtrTy = cast < PointerType > ( I -> getOperand ( TypeOp ) -> getType ( ) ) ; auto * VecTy = cast < FixedVectorType > ( PtrTy -> getElementType ( ) ) ; assert ( VecTy && <STR_LIT> ) ;


<s> auto * PtrTy = cast < PointerType > ( I -> getOperand ( TypeOp ) -> getType ( ) ) ;

auto * VecTy = cast < FixedVectorType > ( PtrTy -> getElementType ( ) ) ;


<s> Type * Ty = IntegerType :: get ( M -> getContext ( ) , <NUM_LIT> ) ; Value * BTC = ActiveLaneMask -> getOperand ( <NUM_LIT> ) ; Builder . SetInsertPoint ( L -> getLoopPreheader ( ) -> getTerminator ( ) ) ; Value * NumElements = getNumElements ( L -> getLoopPreheader ( ) , BTC ) ; Builder . SetInsertPoint ( L -> getHeader ( ) -> getFirstNonPHI ( ) ) ; PHINode * Processed = Builder . CreatePHI ( Ty , <NUM_LIT> ) ; Processed -> addIncoming ( NumElements , L -> getLoopPreheader ( ) ) ; Builder . SetInsertPoint ( ActiveLaneMask ) ; ConstantInt * Factor = ConstantInt :: get ( cast < IntegerType > ( Ty ) , VecTy -> getNumElements ( ) ) ; <STR_LIT> VCTPID ; switch ( VecTy -> getNumElements ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; case <NUM_LIT> : VCTPID = <STR_LIT> ; break ; } Function * VCTP = <STR_LIT> ( M , VCTPID ) ;

Value * VCTPCall = Builder . CreateCall ( VCTP , Processed ) ;


<s> static bool IsDecrement ( Instruction & I ) { auto * Call = dyn_cast < IntrinsicInst > ( & I ) ;


<s> if ( ! Call ) return false ; <STR_LIT> ID = Call -> getIntrinsicID ( ) ; return ID == <STR_LIT> || ID == <STR_LIT> ;


<s> if ( IsMasked ( & I ) ) { FixedVectorType * VecTy = getVectorType ( cast < IntrinsicInst > ( & I ) ) ; unsigned Lanes = VecTy -> getNumElements ( ) ; unsigned ElementWidth = VecTy -> getScalarSizeInBits ( ) ; unsigned MaxWidth = TTI -> getRegisterBitWidth ( true ) ; if ( Lanes * ElementWidth > MaxWidth || Lanes == MaxWidth ) return false ; MaskedInsts . push_back ( cast < IntrinsicInst > ( & I ) ) ; } else if ( auto * Int = dyn_cast < IntrinsicInst > ( & I ) ) { if ( Int -> getIntrinsicID ( ) == <STR_LIT> ) continue ;


<s> return false ; } auto * One = SE -> getOne ( TripCount -> getType ( ) ) ; auto * ElementCount = SE -> getAddExpr ( BTC , One ) ; auto * ECPlusVWMinus1 = SE -> getAddExpr ( ElementCount , SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth - <NUM_LIT> ) ) ) ; auto * Ceil = SE -> getUDivExpr ( ECPlusVWMinus1 , SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth ) ) ) ; ConstantRange RangeCeil = SE -> getSignedRange ( Ceil ) ; ConstantRange RangeTC = SE -> getSignedRange ( TC ) ; if ( ! RangeTC . isSingleElement ( ) ) { auto ZeroRange = ConstantRange ( APInt ( TripCount -> getType ( ) -> getScalarSizeInBits ( ) , <NUM_LIT> ) ) ; RangeTC = RangeTC . unionWith ( ZeroRange ) ; } if ( ! RangeTC . contains ( RangeCeil ) && ! ForceTailPredication ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto * IV = ActiveLaneMask -> getOperand ( <NUM_LIT> ) ; auto * IVExpr = SE -> getSCEV ( IV ) ; auto * AddExpr = dyn_cast < SCEVAddRecExpr > ( IVExpr ) ; if ( ! AddExpr ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; IVExpr -> dump ( ) ) ;


<s> auto * BTC = SE -> getSCEV ( BackedgeTakenCount ) ; if ( ! llvm :: cannotBeMaxInLoop ( BTC , L , * SE , false ) && ! ForceTailPredication ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; BTC -> dump ( ) ) ; return false ; } auto * TC = SE -> getSCEV ( TripCount ) ; unsigned SizeInBits = TripCount -> getType ( ) -> getScalarSizeInBits ( ) ; auto Diff = APInt ( SizeInBits , ~ <NUM_LIT> ) - APInt ( SizeInBits , VectorWidth ) ; uint64_t MaxMinusVW = Diff . getZExtValue ( ) ; uint64_t UpperboundTC = SE -> getSignedRange ( TC ) . getUpper ( ) . getZExtValue ( ) ; if ( UpperboundTC > MaxMinusVW && ! ForceTailPredication ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; dbgs ( ) << UpperboundTC << <STR_LIT> << MaxMinusVW << <STR_LIT> ; ) ; return false ; } auto * One = SE -> getOne ( TripCount -> getType ( ) ) ; auto * ElementCount = SE -> getAddExpr ( BTC , One ) ; auto * ECPlusVWMinus1 = SE -> getAddExpr ( ElementCount , SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth - <NUM_LIT> ) ) ) ;

auto * Ceil = SE -> getUDivExpr ( ECPlusVWMinus1 , SE -> getSCEV ( ConstantInt :: get ( TripCount -> getType ( ) , VectorWidth ) ) ) ;


<s> MVETailPredication ( ) : LoopPass ( ID ) {


<s> rewriteLoopExitValues ( L , LI , TLI , SE , TTI , Rewriter , DT , ReplaceExitValue , DeadInsts ) ;


<s> IRBuilder < > Builder ( ActiveLaneMask -> getParent ( ) -> getFirstNonPHI ( ) ) ; Value * Index = ActiveLaneMask -> getOperand ( <NUM_LIT> ) ; Value * SplatIndex = Builder . CreateVectorSplat ( VectorWidth , Index , <STR_LIT> ) ; SmallVector < Constant * , <NUM_LIT> > Indices ; for ( int i = <NUM_LIT> ; i < VectorWidth ; ++ i ) Indices . push_back ( ConstantInt :: get ( Index -> getType ( ) , i ) ) ; Constant * CV = ConstantVector :: get ( Indices ) ; Value * Induction = Builder . CreateAdd ( SplatIndex , CV , <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * SplatIndex << <STR_LIT> ; dbgs ( ) << <STR_LIT> << * Induction << <STR_LIT> ) ; auto * Preheader = L -> getLoopPreheader ( ) ; auto * BTC = ActiveLaneMask -> getOperand ( <NUM_LIT> ) ; Value * SplatBTC = nullptr ; if ( auto * C = dyn_cast < ConstantInt > ( BTC ) ) { Builder . SetInsertPoint ( Preheader -> getTerminator ( ) ) ; SplatBTC = Builder . CreateVectorSplat ( VectorWidth , C ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * SplatBTC << <STR_LIT> ) ; } else { Instruction * InsertElem ; for ( auto & V : * Preheader ) { InsertElem = dyn_cast < InsertElementInst > ( & V ) ; if ( ! InsertElem ) continue ; ConstantInt * CI = dyn_cast < ConstantInt > ( InsertElem -> getOperand ( <NUM_LIT> ) ) ; if ( ! CI ) continue ; if ( InsertElem -> getOperand ( <NUM_LIT> ) != BTC || CI -> getSExtValue ( ) != <NUM_LIT> ) continue ; if ( ( SplatBTC = dyn_cast < ShuffleVectorInst > ( * InsertElem -> users ( ) . begin ( ) ) ) ) break ; } } if ( ! SplatBTC ) { Builder . SetInsertPoint ( Preheader -> getTerminator ( ) ) ;

Value * Undef = UndefValue :: get ( FixedVectorType :: get ( BTC -> getType ( ) , VectorWidth ) ) ;


<s> for ( auto & I : * BB ) { auto * Call = dyn_cast < IntrinsicInst > ( & I ) ; if ( ! Call ) continue ; <STR_LIT> ID = Call -> getIntrinsicID ( ) ; if ( ID == <STR_LIT> || ID == <STR_LIT> ) return cast < IntrinsicInst > ( & I ) ; } return nullptr ; } ; IntrinsicInst * Setup = FindLoopIterations ( Preheader ) ; if ( ! Setup ) { if ( ! Preheader -> getSinglePredecessor ( ) ) return false ; Setup = FindLoopIterations ( Preheader -> getSinglePredecessor ( ) ) ; if ( ! Setup ) return false ; } IntrinsicInst * Decrement = nullptr ; for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( IsDecrement ( I ) ) { Decrement = cast < IntrinsicInst > ( & I ) ; break ; } } } if ( ! Decrement ) return false ; ClonedVCTPInExitBlock = false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * L << * Setup << <STR_LIT> << * Decrement << <STR_LIT> ) ;

if ( TryConvert ( Setup -> getArgOperand ( <NUM_LIT> ) ) ) {


<s> DT = & getAnalysis < DominatorTreeWrapperPass > ( ) . getDomTree ( ) ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; TTI = & getAnalysis < TargetTransformInfoWrapperPass > ( ) . getTTI ( F ) ; SE = & getAnalysis < ScalarEvolutionWrapperPass > ( ) . getSE ( ) ; auto * TLIP = getAnalysisIfAvailable < TargetLibraryInfoWrapperPass > ( ) ; TLI = TLIP ? & TLIP -> getTLI ( * L -> getHeader ( ) -> getParent ( ) ) : nullptr ; DL = & L -> getHeader ( ) -> getModule ( ) -> getDataLayout ( ) ; this -> L = L ; ActiveLaneMask = nullptr ; if ( ! ST -> hasMVEIntegerOps ( ) || ! ST -> hasV8_1MMainlineOps ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } BasicBlock * Preheader = L -> getLoopPreheader ( ) ; if ( ! Preheader ) return false ; auto FindLoopIterations = [ ] ( BasicBlock * BB ) -> IntrinsicInst * { for ( auto & I : * BB ) { auto * Call = dyn_cast < IntrinsicInst > ( & I ) ; if ( ! Call ) continue ; <STR_LIT> ID = Call -> getIntrinsicID ( ) ; if ( ID == <STR_LIT> || ID == <STR_LIT> ) return cast < IntrinsicInst > ( & I ) ; } return nullptr ; } ; IntrinsicInst * Setup = FindLoopIterations ( Preheader ) ; if ( ! Setup ) { if ( ! Preheader -> getSinglePredecessor ( ) ) return false ;

Setup = FindLoopIterations ( Preheader -> getSinglePredecessor ( ) ) ;


<s> if ( ! Predicate || Predicates . count ( Predicate ) ) continue ; ActiveLaneMask = dyn_cast < IntrinsicInst > ( Predicate ) ; if ( ! ActiveLaneMask || ActiveLaneMask -> getIntrinsicID ( ) != <STR_LIT> ) continue ; Predicates . insert ( Predicate ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ActiveLaneMask << <STR_LIT> ) ; VecTy = getVectorType ( I ) ; if ( ! IsSafeActiveMask ( TripCount , VecTy ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ;

InsertVCTPIntrinsic ( ActiveLaneMask , TripCount , VecTy , NewPredicates ) ;


<s> for ( auto * I : MaskedInsts ) { unsigned PredOp = I -> getIntrinsicID ( ) == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ; auto * Predicate = dyn_cast < Instruction > ( I -> getArgOperand ( PredOp ) ) ; if ( ! Predicate || Predicates . count ( Predicate ) ) continue ; ActiveLaneMask = dyn_cast < IntrinsicInst > ( Predicate ) ;

if ( ! ActiveLaneMask || ActiveLaneMask -> getIntrinsicID ( ) != <STR_LIT> ) continue ;


<s> static inline bool isLoopStart ( MachineInstr & MI ) { return MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ;


<s> MachineBasicBlock * MBB = MI -> getParent ( ) ; assert ( MI -> getOpcode ( ) == <STR_LIT> && <STR_LIT> ) ; if ( UseCmp ) { MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <NUM_LIT> ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT> ) ; } else { MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ;


<s> MachineBasicBlock * MBB = MI -> getParent ( ) ; assert ( MI -> getOpcode ( ) == <STR_LIT> && <STR_LIT> ) ; if ( UseCmp ) { MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <NUM_LIT> ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT> ) ; } else { MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ;


<s> return MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ;


<s> static inline bool isLoopStart ( MachineInstr & MI ) {

return MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ;


<s> MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT> ) ; MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( BrOpc ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT> ) ;


<s> MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( BrOpc ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT> ) ; MI -> eraseFromParent ( ) ;


<s> assert ( isWhileLoopStart ( MI ) && <STR_LIT> ) ;

unsigned Op = MI . getOpcode ( ) == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ;


<s> return MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ;


<s> static inline bool isLoopStart ( const MachineInstr & MI ) {

return isDoLoopStart ( MI ) || isWhileLoopStart ( MI ) ;


<s> static inline bool isWhileLoopStart ( const MachineInstr & MI ) { return MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ;


<s> inline void RevertDoLoopStart ( MachineInstr * MI , const TargetInstrInfo * TII ) { MachineBasicBlock * MBB = MI -> getParent ( ) ;

BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . add ( MI -> getOperand ( <NUM_LIT> ) ) . add ( MI -> getOperand ( <NUM_LIT> ) ) . add ( predOps ( <STR_LIT> ) ) ;


<s> MIB . addReg ( <STR_LIT> ) ; } MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( BrOpc ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <STR_LIT> ) ;


<s> MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <NUM_LIT> ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT> , RegState :: Define ) ; } MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( BrOpc ) ) ; MIB . addMBB ( getWhileLoopStartTargetBB ( * MI ) ) ; MIB . addImm ( <STR_LIT> ) ;


<s> if ( ! HasVCTP || MI . getOpcode ( ) != <STR_LIT> ) continue ; MachineInstrBuilder MIBuilder = BuildMI ( MBB , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( <STR_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; ( void ) MIBuilder ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; dbgs ( ) << <STR_LIT> ; MIBuilder . getInstr ( ) -> dump ( ) ) ; DeadInstructions . push_back ( & MI ) ;


<s> } } Register PhiReg = LoopPhi -> getOperand ( <NUM_LIT> ) . getReg ( ) ; Register DecReg = LoopDec -> getOperand ( <NUM_LIT> ) . getReg ( ) ; Register StartReg = LoopStart -> getOperand ( <NUM_LIT> ) . getReg ( ) ; SmallVector < MachineInstr * , <NUM_LIT> > Copies ; auto CheckUsers = [ & Copies ] ( Register BaseReg , ArrayRef < MachineInstr * > ExpectedUsers , MachineRegisterInfo * MRI ) { SmallVector < Register , <NUM_LIT> > Worklist ; Worklist . push_back ( BaseReg ) ; while ( ! Worklist . empty ( ) ) { Register Reg = Worklist . pop_back_val ( ) ; for ( MachineInstr & MI : MRI -> use_nodbg_instructions ( Reg ) ) { if ( llvm :: is_contained ( ExpectedUsers , & MI ) ) continue ; if ( MI . getOpcode ( ) != TargetOpcode :: COPY || ! MI . getOperand ( <NUM_LIT> ) . getReg ( ) . isVirtual ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; return false ; } Worklist . push_back ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Copies . push_back ( & MI ) ; } } return true ; } ; if ( ! CheckUsers ( PhiReg , { LoopDec } , MRI ) || ! CheckUsers ( DecReg , { LoopPhi , LoopEnd } , MRI ) || ! CheckUsers ( StartReg , { LoopPhi } , MRI ) ) { if ( LoopStart -> getOpcode ( ) == <STR_LIT> ) { RevertWhileLoopStartLR ( LoopStart , TII ) ; RevertLoopDec ( LoopDec , TII ) ; RevertLoopEnd ( LoopEnd , TII ) ; return true ; } return false ;


<s> if ( Visited . count ( MBB ) ) continue ; for ( MachineInstr & MI : * MBB ) { if ( ! IsInvalidTPInstruction ( MI ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ;

MachineInstrBuilder MIB = BuildMI ( * PreHeader , PreHeader -> getFirstTerminator ( ) , LoopStart -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ;


<s> MachineInstr * MVETPAndVPTOptimisations :: CheckForLRUseInPredecessors ( MachineBasicBlock * PreHeader , MachineInstr * LoopStart ) { SmallVector < MachineBasicBlock * > Worklist ; SmallPtrSet < MachineBasicBlock * , <NUM_LIT> > Visited ; Worklist . push_back ( PreHeader ) ; Visited . insert ( LoopStart -> getParent ( ) ) ; while ( ! Worklist . empty ( ) ) { MachineBasicBlock * MBB = Worklist . pop_back_val ( ) ; if ( Visited . count ( MBB ) ) continue ; for ( MachineInstr & MI : * MBB ) { if ( ! IsInvalidTPInstruction ( MI ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ;


<s> MachineInstr * LoopEnd , * LoopPhi , * LoopStart , * LoopDec ; if ( ! findLoopComponents ( ML , MRI , LoopStart , LoopPhi , LoopDec , LoopEnd ) ) return false ; if ( LoopDec != LoopEnd || LoopStart -> getOpcode ( ) != <STR_LIT> ) return false ; SmallVector < MachineInstr * , <NUM_LIT> > VCTPs ; for ( MachineBasicBlock * BB : ML -> blocks ( ) ) for ( MachineInstr & MI : * BB ) if ( isVCTP ( & MI ) ) VCTPs . push_back ( & MI ) ; if ( VCTPs . empty ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MachineInstr * FirstVCTP = * VCTPs . begin ( ) ; for ( MachineInstr * VCTP : VCTPs ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * VCTP ) ; if ( VCTP -> getOpcode ( ) != FirstVCTP -> getOpcode ( ) || VCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) != FirstVCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } } Register CountReg = FirstVCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! CountReg . isVirtual ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MachineInstr * Phi = LookThroughCOPY ( MRI -> getVRegDef ( CountReg ) , MRI ) ; if ( ! Phi || Phi -> getOpcode ( ) != TargetOpcode :: PHI || Phi -> getNumOperands ( ) != <NUM_LIT> || ( Phi -> getOperand ( <NUM_LIT> ) . getMBB ( ) != ML -> getLoopLatch ( ) && Phi -> getOperand ( <NUM_LIT> ) . getMBB ( ) != ML -> getLoopLatch ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } CountReg = Phi -> getOperand ( <NUM_LIT> ) . getMBB ( ) == ML -> getLoopLatch ( ) ? Phi -> getOperand ( <NUM_LIT> ) . getReg ( ) : Phi -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineBasicBlock * MBB = LoopStart -> getParent ( ) ; MachineBasicBlock :: iterator InsertPt = MBB -> getFirstTerminator ( ) ; for ( MachineInstr & Use : MRI -> use_instructions ( LoopStart -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) if ( ( InsertPt != MBB -> end ( ) && ! DT -> dominates ( & * InsertPt , & Use ) ) || ! DT -> dominates ( ML -> getHeader ( ) , Use . getParent ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; }

MachineInstrBuilder MI = BuildMI ( * MBB , InsertPt , LoopStart -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . add ( LoopStart -> getOperand ( <NUM_LIT> ) ) . add ( LoopStart -> getOperand ( <NUM_LIT> ) ) . addReg ( CountReg ) ;


<s> for ( MachineBasicBlock * BB : ML -> blocks ( ) ) for ( MachineInstr & MI : * BB ) if ( isVCTP ( & MI ) ) VCTPs . push_back ( & MI ) ; if ( VCTPs . empty ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MachineInstr * FirstVCTP = * VCTPs . begin ( ) ; for ( MachineInstr * VCTP : VCTPs ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * VCTP ) ; if ( VCTP -> getOpcode ( ) != FirstVCTP -> getOpcode ( ) || VCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) != FirstVCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } } Register CountReg = FirstVCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! CountReg . isVirtual ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MachineInstr * Phi = LookThroughCOPY ( MRI -> getVRegDef ( CountReg ) , MRI ) ; if ( ! Phi || Phi -> getOpcode ( ) != TargetOpcode :: PHI || Phi -> getNumOperands ( ) != <NUM_LIT> || ( Phi -> getOperand ( <NUM_LIT> ) . getMBB ( ) != ML -> getLoopLatch ( ) && Phi -> getOperand ( <NUM_LIT> ) . getMBB ( ) != ML -> getLoopLatch ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } CountReg = Phi -> getOperand ( <NUM_LIT> ) . getMBB ( ) == ML -> getLoopLatch ( ) ? Phi -> getOperand ( <NUM_LIT> ) . getReg ( ) : Phi -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineBasicBlock * MBB = LoopStart -> getParent ( ) ; MachineBasicBlock :: iterator InsertPt = MBB -> getFirstTerminator ( ) ; for ( MachineInstr & Use : MRI -> use_instructions ( LoopStart -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) if ( ( InsertPt != MBB -> end ( ) && ! DT -> dominates ( & * InsertPt , & Use ) ) || ! DT -> dominates ( ML -> getHeader ( ) , Use . getParent ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MachineInstrBuilder MI = BuildMI ( * MBB , InsertPt , LoopStart -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . add ( LoopStart -> getOperand ( <NUM_LIT> ) ) . add ( LoopStart -> getOperand ( <NUM_LIT> ) ) . addReg ( CountReg ) ; ( void ) MI ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * LoopStart << <STR_LIT> << * MI . getInstr ( ) ) ; MRI -> constrainRegClass ( CountReg , & <STR_LIT> ) ; LoopStart -> eraseFromParent ( ) ; return true ;


<s> if ( llvm :: is_contained ( ExpectedUsers , & MI ) ) continue ; if ( MI . getOpcode ( ) != TargetOpcode :: COPY || ! MI . getOperand ( <NUM_LIT> ) . getReg ( ) . isVirtual ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; return false ; } Worklist . push_back ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Copies . push_back ( & MI ) ; } } return true ; } ; if ( ! CheckUsers ( PhiReg , { LoopDec } , MRI ) || ! CheckUsers ( DecReg , { LoopPhi , LoopEnd } , MRI ) || ! CheckUsers ( StartReg , { LoopPhi } , MRI ) ) { if ( LoopStart -> getOpcode ( ) == <STR_LIT> ) { RevertWhileLoopStartLR ( LoopStart , TII ) ; RevertLoopDec ( LoopDec , TII ) ; RevertLoopEnd ( LoopEnd , TII ) ; return true ; } return false ; } MRI -> constrainRegClass ( StartReg , & <STR_LIT> ) ; MRI -> constrainRegClass ( PhiReg , & <STR_LIT> ) ; MRI -> constrainRegClass ( DecReg , & <STR_LIT> ) ; if ( LoopPhi -> getOperand ( <NUM_LIT> ) . getMBB ( ) == ML -> getLoopLatch ( ) ) { LoopPhi -> getOperand ( <NUM_LIT> ) . setReg ( StartReg ) ; LoopPhi -> getOperand ( <NUM_LIT> ) . setReg ( DecReg ) ; } else { LoopPhi -> getOperand ( <NUM_LIT> ) . setReg ( StartReg ) ; LoopPhi -> getOperand ( <NUM_LIT> ) . setReg ( DecReg ) ; } SmallVector < MachineOperand , <NUM_LIT> > Cond ; MachineBasicBlock * TBB = nullptr , * FBB = nullptr ; if ( ! TII -> analyzeBranch ( * LoopEnd -> getParent ( ) , TBB , FBB , Cond ) && ! FBB ) {

MachineFunction :: iterator MBBI = ++ LoopEnd -> getParent ( ) -> getIterator ( ) ;


<s> const <ISA_LIT>Subtarget & STI = Fn . getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( ! STI . isThumb2 ( ) || ! STI . hasLOB ( ) ) return false ; TII = static_cast < const Thumb2InstrInfo * > ( STI . getInstrInfo ( ) ) ; MRI = & Fn . getRegInfo ( ) ; MachineLoopInfo * MLI = & getAnalysis < MachineLoopInfo > ( ) ; MachineDominatorTree * DT = & getAnalysis < MachineDominatorTree > ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << Fn . getName ( ) << '\n' ) ; bool Modified = false ; for ( MachineLoop * ML : MLI -> getBase ( ) . getLoopsInPreorder ( ) ) { Modified |= LowerWhileLoopStart ( ML ) ; Modified |= MergeLoopEnd ( ML ) ; Modified |= ConvertTailPredLoop ( ML , DT ) ; } for ( MachineBasicBlock & MBB : Fn ) { Modified |= HintDoLoopStartReg ( MBB ) ; Modified |= ReplaceConstByVPNOTs ( MBB , DT ) ;

Modified |= ReplaceVCMPsByVPNOTs ( MBB ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << Fn . getName ( ) << '\n' ) ; bool Modified = false ; for ( MachineLoop * ML : MLI -> getBase ( ) . getLoopsInPreorder ( ) ) { Modified |= LowerWhileLoopStart ( ML ) ; Modified |= MergeLoopEnd ( ML ) ; Modified |= ConvertTailPredLoop ( ML , DT ) ; } for ( MachineBasicBlock & MBB : Fn ) { Modified |= HintDoLoopStartReg ( MBB ) ; Modified |= ReplaceConstByVPNOTs ( MBB , DT ) ; Modified |= ReplaceVCMPsByVPNOTs ( MBB ) ; Modified |= ReduceOldVCCRValueUses ( MBB ) ; Modified |= ConvertVPSEL ( MBB ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> static bool CanHaveSwappedOperands ( unsigned Opcode ) { switch ( Opcode ) { default : return true ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> while ( ! Worklist . empty ( ) ) { MachineBasicBlock * MBB = Worklist . pop_back_val ( ) ; if ( Visited . count ( MBB ) ) continue ; for ( MachineInstr & MI : * MBB ) { if ( ! IsInvalidTPInstruction ( MI ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MachineInstrBuilder MIB = BuildMI ( * PreHeader , PreHeader -> getFirstTerminator ( ) , LoopStart -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . add ( LoopStart -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( LoopStart -> getOperand ( <NUM_LIT> ) ) ; LoopStart -> getOperand ( <NUM_LIT> ) . setIsKill ( false ) ; RevertWhileLoopStartLR ( LoopStart , TII , <STR_LIT> , true ) ; return MIB ; }


<s> Worklist . push_back ( PreHeader ) ; Visited . insert ( LoopStart -> getParent ( ) ) ; while ( ! Worklist . empty ( ) ) { MachineBasicBlock * MBB = Worklist . pop_back_val ( ) ; if ( Visited . count ( MBB ) ) continue ; for ( MachineInstr & MI : * MBB ) { if ( ! IsInvalidTPInstruction ( MI ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MachineInstrBuilder MIB = BuildMI ( * PreHeader , PreHeader -> getFirstTerminator ( ) , LoopStart -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . add ( LoopStart -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( LoopStart -> getOperand ( <NUM_LIT> ) ) ; LoopStart -> getOperand ( <NUM_LIT> ) . setIsKill ( false ) ; RevertWhileLoopStartLR ( LoopStart , TII , <STR_LIT> , true ) ; return MIB ; } Visited . insert ( MBB ) ; for ( auto * Pred : MBB -> predecessors ( ) ) Worklist . push_back ( Pred ) ; }


<s> SmallVector < MachineInstr * , <NUM_LIT> > DeadInstructions ; for ( MachineInstr & MI : MBB . instrs ( ) ) { if ( isVCTP ( & MI ) ) { HasVCTP = true ; continue ; } if ( ! HasVCTP || MI . getOpcode ( ) != <STR_LIT> ) continue ; MachineInstrBuilder MIBuilder = BuildMI ( MBB , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( <STR_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; ( void ) MIBuilder ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; dbgs ( ) << <STR_LIT> ; MIBuilder . getInstr ( ) -> dump ( ) ) ;


<s> bool HasVCTP = false ; SmallVector < MachineInstr * , <NUM_LIT> > DeadInstructions ; for ( MachineInstr & MI : MBB . instrs ( ) ) { if ( isVCTP ( & MI ) ) { HasVCTP = true ; continue ; } if ( ! HasVCTP || MI . getOpcode ( ) != <STR_LIT> ) continue ; MachineInstrBuilder MIBuilder = BuildMI ( MBB , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( <STR_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; ( void ) MIBuilder ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; dbgs ( ) << <STR_LIT> ; MIBuilder . getInstr ( ) -> dump ( ) ) ; DeadInstructions . push_back ( & MI ) ; } for ( MachineInstr * DeadInstruction : DeadInstructions ) DeadInstruction -> eraseFromParent ( ) ;


<s> FunctionPass * llvm :: createMVETPAndVPTOptimisationsPass ( ) { return new MVETPAndVPTOptimisations ( ) ;


<s> FunctionPass * llvm :: createMVETPAndVPTOptimisationsPass ( ) {

return new MVETPAndVPTOptimisations ( ) ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < MachineLoopInfo > ( ) ; AU . addPreserved < MachineLoopInfo > ( ) ;

AU . addRequired < MachineDominatorTree > ( ) ;


<s> AU . addRequired < MachineDominatorTree > ( ) ;

AU . addPreserved < MachineDominatorTree > ( ) ;


<s> assert ( IsVCMP ( Instr . getOpcode ( ) ) && <STR_LIT> ) ; return <STR_LIT> ( Instr . getOperand ( <NUM_LIT> ) . getImm ( ) ) ;


<s> static <STR_LIT> GetCondCode ( MachineInstr & Instr ) { assert ( IsVCMP ( Instr . getOpcode ( ) ) && <STR_LIT> ) ;

return <STR_LIT> ( Instr . getOperand ( <NUM_LIT> ) . getImm ( ) ) ;


<s> StringRef getPassName ( ) const override { return <STR_LIT> ;


<s> Register R = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; MF -> getRegInfo ( ) . setRegAllocationHint ( R , <STR_LIT> , <NUM_LIT> ) ; Changed = true ;


<s> static bool IsVCMP ( unsigned Opcode ) { return VCMPOpcodeToVPT ( Opcode ) != <NUM_LIT> ;


<s> static bool IsVPNOTEquivalent ( MachineInstr & Cond , MachineInstr & Prev ) { assert ( IsVCMP ( Cond . getOpcode ( ) ) && IsVCMP ( Prev . getOpcode ( ) ) ) ; if ( Cond . getOpcode ( ) != Prev . getOpcode ( ) ) return false ;

MachineOperand & CondOP1 = Cond . getOperand ( <NUM_LIT> ) , & CondOP2 = Cond . getOperand ( <NUM_LIT> ) ;


<s> MachineOperand & CondOP1 = Cond . getOperand ( <NUM_LIT> ) , & CondOP2 = Cond . getOperand ( <NUM_LIT> ) ; MachineOperand & PrevOP1 = Prev . getOperand ( <NUM_LIT> ) , & PrevOP2 = Prev . getOperand ( <NUM_LIT> ) ; <STR_LIT> ExpectedCode = GetCondCode ( Cond ) ; ExpectedCode = <STR_LIT> ( ExpectedCode ) ; if ( ExpectedCode == GetCondCode ( Prev ) ) if ( CondOP1 . isIdenticalTo ( PrevOP1 ) && CondOP2 . isIdenticalTo ( PrevOP2 ) ) return true ; if ( ! CanHaveSwappedOperands ( Cond . getOpcode ( ) ) ) return false ; ExpectedCode = <STR_LIT> ( ExpectedCode ) ; return ExpectedCode == GetCondCode ( Prev ) && CondOP1 . isIdenticalTo ( PrevOP2 ) && CondOP2 . isIdenticalTo ( PrevOP1 ) ;


<s> static bool IsWritingToVCCR ( MachineInstr & Instr ) { if ( Instr . getNumOperands ( ) == <NUM_LIT> ) return false ; MachineOperand & Dst = Instr . getOperand ( <NUM_LIT> ) ; if ( ! Dst . isReg ( ) ) return false ; Register DstReg = Dst . getReg ( ) ;

if ( ! DstReg . isVirtual ( ) ) return false ;


<s> static MachineInstr * LookThroughCOPY ( MachineInstr * MI , MachineRegisterInfo * MRI ) { while ( MI && MI -> getOpcode ( ) == TargetOpcode :: COPY && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) . isVirtual ( ) ) MI = MRI -> getVRegDef ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << ML -> getHeader ( ) -> getName ( ) << <STR_LIT> ) ; MachineInstr * LoopEnd , * LoopPhi , * LoopStart , * LoopDec ; if ( ! findLoopComponents ( ML , MRI , LoopStart , LoopPhi , LoopDec , LoopEnd ) ) return false ; if ( LoopStart -> getOpcode ( ) != <STR_LIT> ) return false ; Register LR = LoopStart -> getOperand ( <NUM_LIT> ) . getReg ( ) ; auto WLSIt = find_if ( MRI -> use_nodbg_instructions ( LR ) , [ ] ( auto & MI ) { return MI . getOpcode ( ) == <STR_LIT> ; } ) ; if ( ! MergeEndDec || WLSIt == MRI -> use_instr_nodbg_end ( ) ) { RevertWhileLoopSetup ( LoopStart , TII ) ; RevertLoopDec ( LoopStart , TII ) ; RevertLoopEnd ( LoopStart , TII ) ; return true ; } MachineInstrBuilder MI = BuildMI ( * WLSIt -> getParent ( ) , * WLSIt , WLSIt -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , LR ) . add ( LoopStart -> getOperand ( <NUM_LIT> ) ) . add ( WLSIt -> getOperand ( <NUM_LIT> ) ) ; ( void ) MI ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI . getInstr ( ) ) ;


<s> for ( ; Iter != MBB . end ( ) ; ++ Iter ) { if ( MachineOperand * MO = Iter -> findRegisterUseOperand ( VPNOTOperand , true ) ) { VPNOTOperandKiller = MO ; } if ( Iter -> findRegisterUseOperandIdx ( Reg ) != - <NUM_LIT> ) { MustMove = true ; continue ; } if ( Iter -> findRegisterUseOperandIdx ( VPNOTResult ) == - <NUM_LIT> ) continue ; HasUser = true ; if ( ! MustMove ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; VPNOT . dump ( ) ; dbgs ( ) << <STR_LIT> ; Iter -> dump ( ) ) ; MBB . splice ( Iter , & MBB , VPNOT . getIterator ( ) ) ; if ( VPNOTOperandKiller ) VPNOTOperandKiller -> setIsKill ( false ) ;


<s> if ( MachineOperand * MO = Iter -> findRegisterUseOperand ( VPNOTOperand , true ) ) { VPNOTOperandKiller = MO ; } if ( Iter -> findRegisterUseOperandIdx ( Reg ) != - <NUM_LIT> ) { MustMove = true ; continue ; } if ( Iter -> findRegisterUseOperandIdx ( VPNOTResult ) == - <NUM_LIT> ) continue ; HasUser = true ; if ( ! MustMove ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; VPNOT . dump ( ) ; dbgs ( ) << <STR_LIT> ; Iter -> dump ( ) ) ;

MBB . splice ( Iter , & MBB , VPNOT . getIterator ( ) ) ;


<s> MVETPAndVPTOptimisations ( ) : MachineFunctionPass ( ID ) {


<s> if ( IsWritingToVCCR ( Instr ) ) PrevVCMP = nullptr ; continue ; } if ( ! PrevVCMP || ! IsVPNOTEquivalent ( Instr , * PrevVCMP ) ) { PrevVCMP = & Instr ; continue ; } Register PrevVCMPResultReg = PrevVCMP -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstrBuilder MIBuilder = BuildMI ( MBB , & Instr , Instr . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . add ( Instr . getOperand ( <NUM_LIT> ) ) . addReg ( PrevVCMPResultReg ) ; addUnpredicatedMveVpredNOp ( MIBuilder ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MIBuilder . getInstr ( ) -> dump ( ) ; dbgs ( ) << <STR_LIT> ; Instr . dump ( ) ) ; if ( PrevVCMPResultKiller ) PrevVCMPResultKiller -> setIsKill ( false ) ; DeadInstructions . push_back ( & Instr ) ; PrevVCMP = nullptr ; PrevVCMPResultKiller = nullptr ;


<s> PrevVCMP = & Instr ; continue ; } Register PrevVCMPResultReg = PrevVCMP -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstrBuilder MIBuilder = BuildMI ( MBB , & Instr , Instr . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . add ( Instr . getOperand ( <NUM_LIT> ) ) . addReg ( PrevVCMPResultReg ) ; addUnpredicatedMveVpredNOp ( MIBuilder ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MIBuilder . getInstr ( ) -> dump ( ) ; dbgs ( ) << <STR_LIT> ; Instr . dump ( ) ) ; if ( PrevVCMPResultKiller ) PrevVCMPResultKiller -> setIsKill ( false ) ; DeadInstructions . push_back ( & Instr ) ; PrevVCMP = nullptr ; PrevVCMPResultKiller = nullptr ; } for ( MachineInstr * DeadInstruction : DeadInstructions ) DeadInstruction -> eraseFromParent ( ) ; return ! DeadInstructions . empty ( ) ;


<s> MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT> , RegState :: Define ) ; for ( MachineInstr & I : MBB -> terminators ( ) ) { if ( I . getOpcode ( ) == <STR_LIT> ) { MachineInstrBuilder MIB = BuildMI ( * MBB , & I , I . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT> ) ; I . eraseFromParent ( ) ; break ;


<s> for ( MachineLoop * ML : MLI -> getBase ( ) . getLoopsInPreorder ( ) ) { Modified |= LowerWhileLoopStart ( ML ) ; Modified |= MergeLoopEnd ( ML ) ; Modified |= ConvertTailPredLoop ( ML , DT ) ; } for ( MachineBasicBlock & MBB : Fn ) { Modified |= HintDoLoopStartReg ( MBB ) ; Modified |= ReplaceConstByVPNOTs ( MBB , DT ) ; Modified |= ReplaceVCMPsByVPNOTs ( MBB ) ; Modified |= ReduceOldVCCRValueUses ( MBB ) ; Modified |= ConvertVPSEL ( MBB ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return Modified ;


<s> FunctionPass * llvm :: createMVEVPTBlockPass ( ) { return new MVEVPTBlock ( ) ;


<s> if ( CmpMI -> readsRegister ( <STR_LIT> , TRI ) ) break ; } if ( CmpMI == MI ) return nullptr ; NewOpcode = VCMPOpcodeToVPT ( CmpMI -> getOpcode ( ) ) ; if ( NewOpcode == <NUM_LIT> ) return nullptr ; if ( registerDefinedBetween ( CmpMI -> getOperand ( <NUM_LIT> ) . getReg ( ) , std :: next ( CmpMI ) , MI , TRI ) ) return nullptr ; if ( registerDefinedBetween ( CmpMI -> getOperand ( <NUM_LIT> ) . getReg ( ) , std :: next ( CmpMI ) , MI , TRI ) ) return nullptr ; return & * CmpMI ;


<s> -- CmpMI ; if ( CmpMI -> modifiesRegister ( <STR_LIT> , TRI ) ) break ; if ( CmpMI -> readsRegister ( <STR_LIT> , TRI ) ) break ; } if ( CmpMI == MI ) return nullptr ; NewOpcode = VCMPOpcodeToVPT ( CmpMI -> getOpcode ( ) ) ;

if ( NewOpcode == <NUM_LIT> ) return nullptr ;


<s> AddedInstIter != Iter ; ++ AddedInstIter ) { if ( AddedInstIter -> isDebugInstr ( ) ) continue ; dbgs ( ) << <STR_LIT> ; AddedInstIter -> dump ( ) ;


<s> static <STR_LIT> GetInitialBlockMask ( unsigned BlockSize ) { switch ( BlockSize ) { case <NUM_LIT> : return <STR_LIT> :: T ; case <NUM_LIT> : return <STR_LIT> :: TT ; case <NUM_LIT> : return <STR_LIT> :: TTT ;


<s> static <STR_LIT> GetInitialBlockMask ( unsigned BlockSize ) { switch ( BlockSize ) { case <NUM_LIT> :


<s> <STR_LIT> BlockMask = CreateVPTBlock ( MBIter , EndIter , DeadInstructions ) ; MachineInstrBuilder MIBuilder ; unsigned NewOpcode ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << ( unsigned ) BlockMask << <STR_LIT> ) ; if ( MachineInstr * VCMP = findVCMPToFoldIntoVPST ( MI , TRI , NewOpcode ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; VCMP -> dump ( ) ) ; MIBuilder = BuildMI ( Block , MI , DL , TII -> get ( NewOpcode ) ) ; MIBuilder . addImm ( ( uint64_t ) BlockMask ) ; MIBuilder . add ( VCMP -> getOperand ( <NUM_LIT> ) ) ; MIBuilder . add ( VCMP -> getOperand ( <NUM_LIT> ) ) ; MIBuilder . add ( VCMP -> getOperand ( <NUM_LIT> ) ) ; for ( MachineInstr & MII : make_range ( VCMP -> getIterator ( ) , MI -> getIterator ( ) ) ) { MII . clearRegisterKills ( VCMP -> getOperand ( <NUM_LIT> ) . getReg ( ) , TRI ) ; MII . clearRegisterKills ( VCMP -> getOperand ( <NUM_LIT> ) . getReg ( ) , TRI ) ; } VCMP -> eraseFromParent ( ) ; } else { MIBuilder = BuildMI ( Block , MI , DL , TII -> get ( <STR_LIT> ) ) ; MIBuilder . addImm ( ( uint64_t ) BlockMask ) ; }

for ( MachineInstr * DeadMI : DeadInstructions ) DeadMI -> eraseFromParent ( ) ;


<s> static bool IsVPRDefinedOrKilledByBlock ( MachineBasicBlock :: iterator Iter , MachineBasicBlock :: iterator End ) {

for ( ; Iter != End ; ++ Iter ) if ( Iter -> definesRegister ( <STR_LIT> ) || Iter -> killsRegister ( <STR_LIT> ) ) return true ;


<s> MVEVPTBlock ( ) : MachineFunctionPass ( ID ) {


<s> bool MVEVPTBlock :: runOnMachineFunction ( MachineFunction & Fn ) { const <ISA_LIT>Subtarget & STI = Fn . getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( ! STI . isThumb2 ( ) || ! STI . hasMVEIntegerOps ( ) ) return false ; TII = static_cast < const Thumb2InstrInfo * > ( STI . getInstrInfo ( ) ) ; TRI = STI . getRegisterInfo ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << Fn . getName ( ) << '\n' ) ; bool Modified = false ; for ( MachineBasicBlock & MBB : Fn ) Modified |= InsertVPTBlocks ( MBB ) ;


<s> if ( ! STI . isThumb2 ( ) || ! STI . hasMVEIntegerOps ( ) ) return false ; TII = static_cast < const Thumb2InstrInfo * > ( STI . getInstrInfo ( ) ) ; TRI = STI . getRegisterInfo ( ) ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << Fn . getName ( ) << '\n' ) ;


<s> Register PredReg ; NumInstrsSteppedOver = <NUM_LIT> ; while ( Iter != EndIter ) { if ( Iter -> isDebugInstr ( ) ) { ++ Iter ; continue ; } NextPred = getVPTInstrPredicate ( * Iter , PredReg ) ; assert ( NextPred != <STR_LIT> && <STR_LIT> ) ; if ( NextPred == <STR_LIT> || MaxSteps == <NUM_LIT> ) break ;


<s> int VPTInstCnt = <NUM_LIT> ; <STR_LIT> NextPred ; ++ MBIter ; while ( MBIter != EndIter && VPTInstCnt < <NUM_LIT> ) { NextPred = getVPTInstrPredicate ( * MBIter , PredReg ) ; assert ( NextPred != <STR_LIT> && <STR_LIT> ) ; if ( NextPred != Pred ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MBIter -> dump ( ) ) ; ++ VPTInstCnt ; ++ MBIter ; } ; MachineInstrBuilder MIBuilder = BuildMI ( Block , MI , dl , TII -> get ( <STR_LIT> ) ) ; switch ( VPTInstCnt ) { case <NUM_LIT> : MIBuilder . addImm ( VPTMaskValue :: T ) ; break ; case <NUM_LIT> :

MIBuilder . addImm ( VPTMaskValue :: TT ) ;


<s> if ( ! RDA -> hasSameReachingDef ( Def , MI , Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ) || ! RDA -> hasSameReachingDef ( Def , MI , Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) return nullptr ;


<s> auto * Def = RDA -> getReachingMIDef ( MI , <STR_LIT> ) ; if ( ! Def ) return nullptr ; if ( ! ( NewOpcode = VCMPOpcodeToVPT ( Def -> getOpcode ( ) ) ) ) return nullptr ;

if ( ! RDA -> hasSameReachingDef ( Def , MI , Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ) || ! RDA -> hasSameReachingDef ( Def , MI , Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) return nullptr ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . setPreservesCFG ( ) ;

AU . addRequired < ReachingDefAnalysis > ( ) ;


<s> BlockMask = VPTMaskValue :: TTTT ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } ; MachineInstrBuilder MIBuilder ; unsigned NewOpcode ; MachineInstr * VCMP = findVCMPToFoldIntoVPST ( MI , RDA , NewOpcode ) ; if ( VCMP ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; VCMP -> dump ( ) ) ; MIBuilder = BuildMI ( Block , MI , dl , TII -> get ( NewOpcode ) ) ; MIBuilder . addImm ( BlockMask ) ; MIBuilder . add ( VCMP -> getOperand ( <NUM_LIT> ) ) ; MIBuilder . add ( VCMP -> getOperand ( <NUM_LIT> ) ) ; MIBuilder . add ( VCMP -> getOperand ( <NUM_LIT> ) ) ; VCMP -> eraseFromParent ( ) ; } else { MIBuilder = BuildMI ( Block , MI , dl , TII -> get ( <STR_LIT> ) ) ; MIBuilder . addImm ( BlockMask ) ; }

finalizeBundle ( Block , MachineBasicBlock :: instr_iterator ( MIBuilder . getInstr ( ) ) , MBIter ) ;


<s> if ( ! STI . isThumb2 ( ) || ! STI . hasMVEIntegerOps ( ) ) return false ; TII = static_cast < const Thumb2InstrInfo * > ( STI . getInstrInfo ( ) ) ; RDA = & getAnalysis < ReachingDefAnalysis > ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << Fn . getName ( ) << '\n' ) ;


<s> RDA = & getAnalysis < ReachingDefAnalysis > ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << Fn . getName ( ) << '\n' ) ; bool Modified = false ; for ( MachineBasicBlock & MBB : Fn ) Modified |= InsertVPTBlocks ( MBB ) ;


<s> bool MVEVPTBlock :: InsertVPTBlocks ( MachineBasicBlock & Block ) { bool Modified = false ; MachineBasicBlock :: instr_iterator MBIter = Block . instr_begin ( ) ; MachineBasicBlock :: instr_iterator EndIter = Block . instr_end ( ) ; while ( MBIter != EndIter ) { MachineInstr * MI = & * MBIter ; unsigned PredReg = <NUM_LIT> ; DebugLoc dl = MI -> getDebugLoc ( ) ; <STR_LIT> Pred = getVPTInstrPredicate ( * MI , PredReg ) ; assert ( Pred != <STR_LIT> && <STR_LIT> ) ; if ( Pred == <STR_LIT> ) { ++ MBIter ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MI -> dump ( ) ) ; int VPTInstCnt = <NUM_LIT> ; <STR_LIT> NextPred ; ++ MBIter ; while ( MBIter != EndIter && VPTInstCnt < <NUM_LIT> ) { NextPred = getVPTInstrPredicate ( * MBIter , PredReg ) ; assert ( NextPred != <STR_LIT> && <STR_LIT> ) ; if ( NextPred != Pred ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MBIter -> dump ( ) ) ; ++ VPTInstCnt ; ++ MBIter ; } ; unsigned BlockMask = <NUM_LIT> ; switch ( VPTInstCnt ) { case <NUM_LIT> : BlockMask = VPTMaskValue :: T ; break ; case <NUM_LIT> : BlockMask = VPTMaskValue :: TT ; break ; case <NUM_LIT> : BlockMask = VPTMaskValue :: TTT ; break ; case <NUM_LIT> : BlockMask = VPTMaskValue :: TTTT ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } ; MachineInstrBuilder MIBuilder ; unsigned NewOpcode ; MachineInstr * VCMP = findVCMPToFoldIntoVPST ( MI , TRI , NewOpcode ) ; if ( VCMP ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; VCMP -> dump ( ) ) ; MIBuilder = BuildMI ( Block , MI , dl , TII -> get ( NewOpcode ) ) ; MIBuilder . addImm ( BlockMask ) ;

MIBuilder . add ( VCMP -> getOperand ( <NUM_LIT> ) ) ;


<s> if ( Pred == <STR_LIT> ) { ++ MBIter ; continue ; } MachineInstrBuilder MIBuilder = BuildMI ( Block , MBIter , dl , TII -> get ( <STR_LIT> ) ) ; MachineBasicBlock :: iterator VPSTInsertPos = MIBuilder . getInstr ( ) ; int VPTInstCnt = <NUM_LIT> ; <STR_LIT> NextPred ; do { ++ MBIter ; NextPred = getVPTInstrPredicate ( * MBIter , PredReg ) ; } while ( NextPred != <STR_LIT> && NextPred == Pred && ++ VPTInstCnt < <NUM_LIT> ) ; switch ( VPTInstCnt ) { case <NUM_LIT> : MIBuilder . addImm ( VPTMaskValue :: T ) ; break ; case <NUM_LIT> : MIBuilder . addImm ( VPTMaskValue :: TT ) ; break ; case <NUM_LIT> : MIBuilder . addImm ( VPTMaskValue :: TTT ) ; break ; case <NUM_LIT> : MIBuilder . addImm ( VPTMaskValue :: TTTT ) ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } ; MachineInstr * LastMI = & * MBIter ;

finalizeBundle ( Block , VPSTInsertPos . getInstrIterator ( ) , ++ LastMI -> getIterator ( ) ) ;


<s> bool MVEVPTBlock :: InsertVPTBlocks ( MachineBasicBlock & Block ) { bool Modified = false ; MachineBasicBlock :: instr_iterator MBIter = Block . instr_begin ( ) ; MachineBasicBlock :: instr_iterator EndIter = Block . instr_end ( ) ; SmallVector < MachineInstr * , <NUM_LIT> > RemovedVCMPs ; while ( MBIter != EndIter ) { MachineInstr * MI = & * MBIter ; unsigned PredReg = <NUM_LIT> ; DebugLoc dl = MI -> getDebugLoc ( ) ; <STR_LIT> Pred = getVPTInstrPredicate ( * MI , PredReg ) ; assert ( Pred != <STR_LIT> && <STR_LIT> ) ; if ( Pred == <STR_LIT> ) { ++ MBIter ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MI -> dump ( ) ) ; int VPTInstCnt = <NUM_LIT> ; <STR_LIT> NextPred ; ++ MBIter ; while ( MBIter != EndIter && VPTInstCnt < <NUM_LIT> ) { NextPred = getVPTInstrPredicate ( * MBIter , PredReg ) ; assert ( NextPred != <STR_LIT> && <STR_LIT> ) ; if ( NextPred != Pred ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MBIter -> dump ( ) ) ; ++ VPTInstCnt ; ++ MBIter ;


<s> TII = static_cast < const Thumb2InstrInfo * > ( STI . getInstrInfo ( ) ) ; RDA = & getAnalysis < ReachingDefAnalysis > ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << Fn . getName ( ) << '\n' ) ; bool Modified = false ; for ( MachineBasicBlock & MBB : Fn ) Modified |= InsertVPTBlocks ( MBB ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return Modified ;


<s> const <ISA_LIT>Subtarget & STI = static_cast < const <ISA_LIT>Subtarget & > ( Fn . getSubtarget ( ) ) ; if ( ! STI . isThumb2 ( ) || ! STI . hasMVEIntegerOps ( ) ) return false ;

TII = static_cast < const Thumb2InstrInfo * > ( STI . getInstrInfo ( ) ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; Iter -> dump ( ) ) ; unsigned BlockSize ; StepOverPredicatedInstrs ( Iter , EndIter , <NUM_LIT> , BlockSize ) ; LLVM_DEBUG ( for ( MachineBasicBlock :: instr_iterator AddedInstIter = std :: next ( BlockBeg ) ; AddedInstIter != Iter ; ++ AddedInstIter ) { dbgs ( ) << <STR_LIT> ; AddedInstIter -> dump ( ) ; } ) ; <STR_LIT> BlockMask = get<ISA_LIT>VPTBlockMask ( BlockSize ) ; <STR_LIT> CurrentPredicate = <STR_LIT> ; while ( BlockSize < <NUM_LIT> && Iter != EndIter && Iter -> getOpcode ( ) == <STR_LIT> ) { unsigned ElseInstCnt = <NUM_LIT> ; MachineBasicBlock :: instr_iterator VPNOTBlockEndIter = std :: next ( Iter ) ; if ( ! StepOverPredicatedInstrs ( VPNOTBlockEndIter , EndIter , ( <NUM_LIT> - BlockSize ) , ElseInstCnt ) ) break ; if ( ! IsVPRDefinedOrKilledByBlock ( Iter , VPNOTBlockEndIter ) ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; Iter -> dump ( ) ; ) ; BlockSize += ElseInstCnt ; assert ( BlockSize <= <NUM_LIT> && <STR_LIT> ) ; DeadInstructions . push_back ( & * Iter ) ; ++ Iter ; bool ChangeToElse = ( CurrentPredicate == <STR_LIT> ) ; for ( ; Iter != VPNOTBlockEndIter ; ++ Iter ) { int OpIdx = findFirstVPTPredOperandIdx ( * Iter ) ; assert ( OpIdx != - <NUM_LIT> ) ; if ( ChangeToElse ) { Iter -> getOperand ( OpIdx ) . setImm ( <STR_LIT> ) ; BlockMask = ExpandBlockMask ( BlockMask , <STR_LIT> ) ; if ( Iter -> definesRegister ( <STR_LIT> ) ) ChangeToElse = false ;

} else BlockMask = ExpandBlockMask ( BlockMask , <STR_LIT> ) ;


<s> return ChooseMask ( PredBlockMask :: TTT , PredBlockMask :: TTE ) ; case PredBlockMask :: TE : return ChooseMask ( PredBlockMask :: TET , PredBlockMask :: TEE ) ; case PredBlockMask :: TTT : return ChooseMask ( PredBlockMask :: TTTT , PredBlockMask :: TTTE ) ; case PredBlockMask :: TTE : return ChooseMask ( PredBlockMask :: TTET , PredBlockMask :: TTEE ) ; case PredBlockMask :: TET : return ChooseMask ( PredBlockMask :: TETT , PredBlockMask :: TETE ) ; case PredBlockMask :: TEE : return ChooseMask ( PredBlockMask :: TEET , PredBlockMask :: TEEE ) ; default :


<s> return ChooseMask ( PredBlockMask :: TTT , PredBlockMask :: TTE ) ; case PredBlockMask :: TE : return ChooseMask ( PredBlockMask :: TET , PredBlockMask :: TEE ) ; case PredBlockMask :: TTT : return ChooseMask ( PredBlockMask :: TTTT , PredBlockMask :: TTTE ) ; case PredBlockMask :: TTE : return ChooseMask ( PredBlockMask :: TTET , PredBlockMask :: TTEE ) ; case PredBlockMask :: TET : return ChooseMask ( PredBlockMask :: TETT , PredBlockMask :: TETE ) ; case PredBlockMask :: TEE : return ChooseMask ( PredBlockMask :: TEET , PredBlockMask :: TEEE ) ;


<s> while ( MBIter != EndIter ) { MachineInstr * MI = & * MBIter ; Register PredReg ; DebugLoc DL = MI -> getDebugLoc ( ) ; <STR_LIT> Pred = getVPTInstrPredicate ( * MI , PredReg ) ; assert ( Pred != <STR_LIT> && <STR_LIT> ) ; if ( Pred == <STR_LIT> ) { ++ MBIter ; continue ; } <STR_LIT> BlockMask = CreateVPTBlock ( MBIter , EndIter , DeadInstructions ) ; MachineInstrBuilder MIBuilder ; unsigned NewOpcode ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << ( unsigned ) BlockMask << <STR_LIT> ) ;

if ( MachineInstr * VCMP = findVCMPToFoldIntoVPST ( MI , TRI , NewOpcode ) ) {


<s> MachineBasicBlock :: instr_iterator EndIter = Block . instr_end ( ) ; SmallSet < MachineInstr * , <NUM_LIT> > RemovedVCMPs ; while ( MBIter != EndIter ) { MachineInstr * MI = & * MBIter ; unsigned PredReg = <NUM_LIT> ; DebugLoc dl = MI -> getDebugLoc ( ) ; <STR_LIT> Pred = getVPTInstrPredicate ( * MI , PredReg ) ; assert ( Pred != <STR_LIT> && <STR_LIT> ) ; if ( Pred == <STR_LIT> ) { ++ MBIter ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MI -> dump ( ) ) ; int VPTInstCnt = <NUM_LIT> ; <STR_LIT> NextPred ; ++ MBIter ; while ( MBIter != EndIter && VPTInstCnt < <NUM_LIT> ) { NextPred = getVPTInstrPredicate ( * MBIter , PredReg ) ; assert ( NextPred != <STR_LIT> && <STR_LIT> ) ; if ( NextPred != Pred ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MBIter -> dump ( ) ) ; ++ VPTInstCnt ; ++ MBIter ; } ; unsigned BlockMask = get<ISA_LIT>VPTBlockMask ( VPTInstCnt ) ; MachineInstrBuilder MIBuilder ; unsigned NewOpcode ; MachineInstr * VCMP = findVCMPToFoldIntoVPST ( MI , RDA , NewOpcode ) ; if ( VCMP ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; VCMP -> dump ( ) ) ; MIBuilder = BuildMI ( Block , MI , dl , TII -> get ( NewOpcode ) ) ; MIBuilder . addImm ( BlockMask ) ; MIBuilder . add ( VCMP -> getOperand ( <NUM_LIT> ) ) ; MIBuilder . add ( VCMP -> getOperand ( <NUM_LIT> ) ) ; MIBuilder . add ( VCMP -> getOperand ( <NUM_LIT> ) ) ; RemovedVCMPs . insert ( VCMP ) ; } else { MIBuilder = BuildMI ( Block , MI , dl , TII -> get ( <STR_LIT> ) ) ; MIBuilder . addImm ( BlockMask ) ;


<s> ++ MBIter ; while ( MBIter != EndIter && VPTInstCnt < <NUM_LIT> ) { NextPred = getVPTInstrPredicate ( * MBIter , PredReg ) ; assert ( NextPred != <STR_LIT> && <STR_LIT> ) ; if ( NextPred != Pred ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MBIter -> dump ( ) ) ; ++ VPTInstCnt ; ++ MBIter ; } ; unsigned BlockMask = get<ISA_LIT>VPTBlockMask ( VPTInstCnt ) ; MachineInstrBuilder MIBuilder ; unsigned NewOpcode ; MachineInstr * VCMP = findVCMPToFoldIntoVPST ( MI , TRI , NewOpcode ) ; if ( VCMP ) {

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; VCMP -> dump ( ) ) ;


<s> NextPred = getVPTInstrPredicate ( * MBIter , PredReg ) ; assert ( NextPred != <STR_LIT> && <STR_LIT> ) ; if ( NextPred != Pred ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MBIter -> dump ( ) ) ; ++ VPTInstCnt ; ++ MBIter ; } ; unsigned BlockMask = get<ISA_LIT>VPTBlockMask ( VPTInstCnt ) ; MachineInstrBuilder MIBuilder ; unsigned NewOpcode ; MachineInstr * VCMP = findVCMPToFoldIntoVPST ( MI , TRI , NewOpcode ) ; if ( VCMP ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; VCMP -> dump ( ) ) ; MIBuilder = BuildMI ( Block , MI , dl , TII -> get ( NewOpcode ) ) ; MIBuilder . addImm ( BlockMask ) ;


<s> assert ( getVPTInstrPredicate ( * Iter ) == <STR_LIT> && <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; Iter -> dump ( ) ) ; unsigned BlockSize ; StepOverPredicatedInstrs ( Iter , EndIter , <NUM_LIT> , BlockSize ) ; LLVM_DEBUG ( for ( MachineBasicBlock :: instr_iterator AddedInstIter = std :: next ( BlockBeg ) ; AddedInstIter != Iter ; ++ AddedInstIter ) { dbgs ( ) << <STR_LIT> ; AddedInstIter -> dump ( ) ; } ) ; <STR_LIT> BlockMask = GetInitialBlockMask ( BlockSize ) ; <STR_LIT> CurrentPredicate = <STR_LIT> ; while ( BlockSize < <NUM_LIT> && Iter != EndIter && Iter -> getOpcode ( ) == <STR_LIT> ) { unsigned ElseInstCnt = <NUM_LIT> ; MachineBasicBlock :: instr_iterator VPNOTBlockEndIter = std :: next ( Iter ) ; if ( ! StepOverPredicatedInstrs ( VPNOTBlockEndIter , EndIter , ( <NUM_LIT> - BlockSize ) , ElseInstCnt ) ) break ; if ( ! IsVPRDefinedOrKilledByBlock ( Iter , VPNOTBlockEndIter ) ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; Iter -> dump ( ) ; ) ; BlockSize += ElseInstCnt ; assert ( BlockSize <= <NUM_LIT> && <STR_LIT> ) ; DeadInstructions . push_back ( & * Iter ) ;


<s> AddedInstIter != Iter ; ++ AddedInstIter ) { dbgs ( ) << <STR_LIT> ;


<s> assert ( Pred != <STR_LIT> && <STR_LIT> ) ; if ( Pred == <STR_LIT> ) { ++ MBIter ; continue ; } <STR_LIT> BlockMask = CreateVPTBlock ( MBIter , EndIter , DeadInstructions ) ; MachineInstrBuilder MIBuilder ; unsigned NewOpcode ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << ( unsigned ) BlockMask << <STR_LIT> ) ; if ( MachineInstr * VCMP = findVCMPToFoldIntoVPST ( MI , TRI , NewOpcode ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; VCMP -> dump ( ) ) ; MIBuilder = BuildMI ( Block , MI , DL , TII -> get ( NewOpcode ) ) ; MIBuilder . addImm ( ( uint64_t ) BlockMask ) ; MIBuilder . add ( VCMP -> getOperand ( <NUM_LIT> ) ) ; MIBuilder . add ( VCMP -> getOperand ( <NUM_LIT> ) ) ; MIBuilder . add ( VCMP -> getOperand ( <NUM_LIT> ) ) ; for ( MachineInstr & MI : make_range ( VCMP -> getIterator ( ) , MI -> getIterator ( ) ) ) { MI . clearRegisterKills ( VCMP -> getOperand ( <NUM_LIT> ) . getReg ( ) , TRI ) ; MI . clearRegisterKills ( VCMP -> getOperand ( <NUM_LIT> ) . getReg ( ) , TRI ) ; } VCMP -> eraseFromParent ( ) ; } else { MIBuilder = BuildMI ( Block , MI , DL , TII -> get ( <STR_LIT> ) ) ; MIBuilder . addImm ( ( uint64_t ) BlockMask ) ; } finalizeBundle ( Block , MachineBasicBlock :: instr_iterator ( MIBuilder . getInstr ( ) ) , MBIter ) ; Modified = true ; } for ( MachineInstr * DeadMI : DeadInstructions ) { if ( DeadMI -> isInsideBundle ( ) ) DeadMI -> eraseFromBundle ( ) ; else DeadMI -> eraseFromParent ( ) ; }


<s> const <ISA_LIT>Subtarget & STI = static_cast < const <ISA_LIT>Subtarget & > ( Fn . getSubtarget ( ) ) ; if ( ! STI . isThumb2 ( ) || ! STI . hasMVEIntegerOps ( ) ) return false ; TII = static_cast < const Thumb2InstrInfo * > ( STI . getInstrInfo ( ) ) ; TRI = STI . getRegisterInfo ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << Fn . getName ( ) << '\n' ) ; bool Modified = false ; for ( MachineBasicBlock & MBB : Fn ) Modified |= InsertVPTBlocks ( MBB ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ;


<s> if ( ! STI . isThumb2 ( ) || ! STI . hasMVEIntegerOps ( ) ) return false ; TII = static_cast < const Thumb2InstrInfo * > ( STI . getInstrInfo ( ) ) ; TRI = STI . getRegisterInfo ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << Fn . getName ( ) << '\n' ) ; bool Modified = false ; for ( MachineBasicBlock & MBB : Fn ) Modified |= InsertVPTBlocks ( MBB ) ;


<s> NextPred = getVPTInstrPredicate ( * Iter , PredReg ) ; assert ( NextPred != <STR_LIT> && <STR_LIT> ) ; if ( NextPred == <STR_LIT> || MaxSteps == <NUM_LIT> ) break ; -- MaxSteps ; ++ Iter ; ++ NumInstrsSteppedOver ; } ; return NumInstrsSteppedOver != <NUM_LIT> && ( NextPred == <STR_LIT> || Iter == EndIter ) ;


<s> while ( Iter != EndIter ) { NextPred = getVPTInstrPredicate ( * Iter , PredReg ) ; assert ( NextPred != <STR_LIT> && <STR_LIT> ) ; if ( NextPred == <STR_LIT> || MaxSteps == <NUM_LIT> ) break ;


<s> for ( ; Iter != End ; ++ Iter ) { bool IsInteresting = false ; if ( MachineOperand * MO = Iter -> findRegisterUseOperand ( VCCRValue ) ) { IsInteresting = true ; if ( Iter -> getOpcode ( ) == <STR_LIT> ) { Register Result = Iter -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MRI -> replaceRegWith ( Result , LastVPNOTResult ) ; DeadInstructions . push_back ( & * Iter ) ; Modified = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Result ) << <STR_LIT> << printReg ( LastVPNOTResult ) << <STR_LIT> ) ; } else { MachineInstr & VPNOT = ReplaceRegisterUseWithVPNOT ( MBB , * Iter , * MO , LastVPNOTResult ) ; Modified = true ; LastVPNOTResult = VPNOT . getOperand ( <NUM_LIT> ) . getReg ( ) ; std :: swap ( VCCRValue , OppositeVCCRValue ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( VCCRValue ) << <STR_LIT> << printReg ( LastVPNOTResult ) << <STR_LIT> << * Iter ) ; } } else { if ( MachineOperand * MO = Iter -> findRegisterUseOperand ( OppositeVCCRValue ) ) { IsInteresting = true ; if ( LastVPNOTResult != OppositeVCCRValue ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( OppositeVCCRValue ) << <STR_LIT> << printReg ( LastVPNOTResult ) << <STR_LIT> ; Iter -> dump ( ) ) ; MO -> setReg ( LastVPNOTResult ) ; Modified = true ;


<s> continue ; } if ( ! PrevVCMP || ! IsVPNOTEquivalent ( Instr , * PrevVCMP ) ) { PrevVCMP = & Instr ; continue ; } Register PrevVCMPResultReg = PrevVCMP -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstrBuilder MIBuilder = BuildMI ( MBB , & Instr , Instr . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . add ( Instr . getOperand ( <NUM_LIT> ) ) . addReg ( PrevVCMPResultReg ) ; addUnpredicatedMveVpredNOp ( MIBuilder ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MIBuilder . getInstr ( ) -> dump ( ) ; dbgs ( ) << <STR_LIT> ; Instr . dump ( ) ) ; if ( PrevVCMPResultKiller ) PrevVCMPResultKiller -> setIsKill ( false ) ;

DeadInstructions . push_back ( & Instr ) ;


<s> bool MVEVPTOptimisations :: runOnMachineFunction ( MachineFunction & Fn ) { const <ISA_LIT>Subtarget & STI = static_cast < const <ISA_LIT>Subtarget & > ( Fn . getSubtarget ( ) ) ;

if ( ! STI . isThumb2 ( ) || ! STI . hasMVEIntegerOps ( ) ) return false ;


<s> return false ; } MachineInstr * FirstVCTP = * VCTPs . begin ( ) ; for ( MachineInstr * VCTP : VCTPs ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * VCTP ) ; if ( VCTP -> getOpcode ( ) != FirstVCTP -> getOpcode ( ) || VCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) != FirstVCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } } Register CountReg = FirstVCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! CountReg . isVirtual ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MachineInstr * Phi = LookThroughCOPY ( MRI -> getVRegDef ( CountReg ) , MRI ) ; if ( ! Phi || Phi -> getOpcode ( ) != TargetOpcode :: PHI || Phi -> getNumOperands ( ) != <NUM_LIT> || ( Phi -> getOperand ( <NUM_LIT> ) . getMBB ( ) != ML -> getLoopLatch ( ) && Phi -> getOperand ( <NUM_LIT> ) . getMBB ( ) != ML -> getLoopLatch ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } CountReg = Phi -> getOperand ( <NUM_LIT> ) . getMBB ( ) == ML -> getLoopLatch ( ) ? Phi -> getOperand ( <NUM_LIT> ) . getReg ( ) : Phi -> getOperand ( <NUM_LIT> ) . getReg ( ) ;

MachineBasicBlock * MBB = LoopStart -> getParent ( ) ;


<s> return false ; } MachineInstr * Phi = LookThroughCOPY ( MRI -> getVRegDef ( CountReg ) , MRI ) ; if ( ! Phi || Phi -> getOpcode ( ) != TargetOpcode :: PHI || Phi -> getNumOperands ( ) != <NUM_LIT> || ( Phi -> getOperand ( <NUM_LIT> ) . getMBB ( ) != ML -> getLoopLatch ( ) && Phi -> getOperand ( <NUM_LIT> ) . getMBB ( ) != ML -> getLoopLatch ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } CountReg = Phi -> getOperand ( <NUM_LIT> ) . getMBB ( ) == ML -> getLoopLatch ( ) ? Phi -> getOperand ( <NUM_LIT> ) . getReg ( ) : Phi -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineBasicBlock * MBB = LoopStart -> getParent ( ) ; MachineBasicBlock :: iterator InsertPt = MBB -> getFirstTerminator ( ) ; for ( MachineInstr & Use : MRI -> use_instructions ( LoopStart -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) if ( ( InsertPt != MBB -> end ( ) && ! DT -> dominates ( & * InsertPt , & Use ) ) || ! DT -> dominates ( ML -> getHeader ( ) , Use . getParent ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ;


<s> SmallVector < MachineInstr * , <NUM_LIT> > Copies ; auto CheckUsers = [ & Copies ] ( Register BaseReg , ArrayRef < MachineInstr * > ExpectedUsers , MachineRegisterInfo * MRI ) { SmallVector < Register , <NUM_LIT> > Worklist ; Worklist . push_back ( BaseReg ) ; while ( ! Worklist . empty ( ) ) { Register Reg = Worklist . pop_back_val ( ) ; for ( MachineInstr & MI : MRI -> use_nodbg_instructions ( Reg ) ) { if ( count ( ExpectedUsers , & MI ) ) continue ; if ( MI . getOpcode ( ) != TargetOpcode :: COPY || ! MI . getOperand ( <NUM_LIT> ) . getReg ( ) . isVirtual ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; return false ; } Worklist . push_back ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Copies . push_back ( & MI ) ; } } return true ; } ; if ( ! CheckUsers ( PhiReg , { LoopDec } , MRI ) || ! CheckUsers ( DecReg , { LoopPhi , LoopEnd } , MRI ) || ! CheckUsers ( StartReg , { LoopPhi } , MRI ) ) return false ; MRI -> constrainRegClass ( StartReg , & <STR_LIT> ) ; MRI -> constrainRegClass ( PhiReg , & <STR_LIT> ) ; MRI -> constrainRegClass ( DecReg , & <STR_LIT> ) ; if ( LoopPhi -> getOperand ( <NUM_LIT> ) . getMBB ( ) == ML -> getLoopLatch ( ) ) { LoopPhi -> getOperand ( <NUM_LIT> ) . setReg ( StartReg ) ; LoopPhi -> getOperand ( <NUM_LIT> ) . setReg ( DecReg ) ; } else { LoopPhi -> getOperand ( <NUM_LIT> ) . setReg ( StartReg ) ; LoopPhi -> getOperand ( <NUM_LIT> ) . setReg ( DecReg ) ; } MachineInstrBuilder MI = BuildMI ( * LoopEnd -> getParent ( ) , * LoopEnd , LoopEnd -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , DecReg ) . addReg ( PhiReg ) . add ( LoopEnd -> getOperand ( <NUM_LIT> ) ) ; ( void ) MI ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI . getInstr ( ) ) ; LoopDec -> eraseFromParent ( ) ; LoopEnd -> eraseFromParent ( ) ; for ( auto * MI : Copies ) MI -> eraseFromParent ( ) ;


<s> return true ; } } } Register PhiReg = LoopPhi -> getOperand ( <NUM_LIT> ) . getReg ( ) ; Register DecReg = LoopDec -> getOperand ( <NUM_LIT> ) . getReg ( ) ; Register StartReg = LoopStart -> getOperand ( <NUM_LIT> ) . getReg ( ) ; SmallVector < MachineInstr * , <NUM_LIT> > Copies ; auto CheckUsers = [ & Copies ] ( Register BaseReg , ArrayRef < MachineInstr * > ExpectedUsers , MachineRegisterInfo * MRI ) { SmallVector < Register , <NUM_LIT> > Worklist ; Worklist . push_back ( BaseReg ) ; while ( ! Worklist . empty ( ) ) { Register Reg = Worklist . pop_back_val ( ) ; for ( MachineInstr & MI : MRI -> use_nodbg_instructions ( Reg ) ) { if ( count ( ExpectedUsers , & MI ) ) continue ; if ( MI . getOpcode ( ) != TargetOpcode :: COPY || ! MI . getOperand ( <NUM_LIT> ) . getReg ( ) . isVirtual ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; return false ;


<s> return - <NUM_LIT> ; } ; unsigned Imm = getImm ( GPR ) ; if ( Imm == - <NUM_LIT> ) { LastVPTReg = <NUM_LIT> ; continue ; } unsigned NotImm = ~ Imm & <NUM_LIT> ; if ( LastVPTReg != <NUM_LIT> && LastVPTReg != VPR && LastVPTImm == Imm ) { Instr . getOperand ( PIdx + <NUM_LIT> ) . setReg ( LastVPTReg ) ; if ( MRI -> use_empty ( VPR ) ) { DeadInstructions . insert ( Copy ) ; if ( MRI -> hasOneUse ( GPR ) ) DeadInstructions . insert ( MRI -> getVRegDef ( GPR ) ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Instr ) ; } else if ( LastVPTReg != <NUM_LIT> && LastVPTImm == NotImm ) { Register NewVPR = MRI -> createVirtualRegister ( & <STR_LIT> ) ; auto VPNot = BuildMI ( MBB , & Instr , Instr . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , NewVPR ) . addReg ( LastVPTReg ) ; addUnpredicatedMveVpredNOp ( VPNot ) ; Instr . getOperand ( PIdx + <NUM_LIT> ) . setReg ( NewVPR ) ; if ( MRI -> use_empty ( VPR ) ) { DeadInstructions . insert ( Copy ) ; if ( MRI -> hasOneUse ( GPR ) ) DeadInstructions . insert ( MRI -> getVRegDef ( GPR ) ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * VPNot << <STR_LIT> << Instr ) ;


<s> if ( ! STI . isThumb2 ( ) || ! STI . hasLOB ( ) ) return false ; TII = static_cast < const Thumb2InstrInfo * > ( STI . getInstrInfo ( ) ) ; MRI = & Fn . getRegInfo ( ) ; MachineLoopInfo * MLI = & getAnalysis < MachineLoopInfo > ( ) ; MachineDominatorTree * DT = & getAnalysis < MachineDominatorTree > ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << Fn . getName ( ) << '\n' ) ; bool Modified = false ; for ( MachineLoop * ML : MLI -> getBase ( ) . getLoopsInPreorder ( ) ) { Modified |= MergeLoopEnd ( ML ) ;

Modified |= ConvertTailPredLoop ( ML , DT ) ;


<s> MRI = & Fn . getRegInfo ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << Fn . getName ( ) << '\n' ) ; bool Modified = false ; for ( MachineBasicBlock & MBB : Fn ) Modified |= ReplaceVCMPsByVPNOTs ( MBB ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return Modified ;


<s> } if ( ! HasVCTP || MI . getOpcode ( ) != <STR_LIT> ) continue ; MachineInstrBuilder MIBuilder = BuildMI ( MBB , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( <STR_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; ( void ) MIBuilder ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; dbgs ( ) << <STR_LIT> ; MIBuilder . getInstr ( ) -> dump ( ) ) ; DeadInstructions . push_back ( & MI ) ; }

for ( MachineInstr * DeadInstruction : DeadInstructions ) DeadInstruction -> eraseFromParent ( ) ;


<s> FunctionPass * llvm :: createMVEVPTOptimisationsPass ( ) { return new MVEVPTOptimisations ( ) ;


<s> StringRef getPassName ( ) const override { return <STR_LIT> ;


<s> MVEVPTOptimisations ( ) : MachineFunctionPass ( ID ) {


<s> if ( VCCRValue && Iter -> getOpcode ( ) == <STR_LIT> && Iter -> findRegisterUseOperandIdx ( VCCRValue ) != - <NUM_LIT> ) { if ( ! MoveVPNOTBeforeFirstUser ( MBB , Iter , VCCRValue ) ) continue ; OppositeVCCRValue = Dst ; ++ Iter ; break ; } VCCRValue = Dst ; } if ( Iter == End ) break ; assert ( VCCRValue && OppositeVCCRValue && <STR_LIT> <STR_LIT> ) ; assert ( VCCRValue != OppositeVCCRValue && <STR_LIT> ) ; Register LastVPNOTResult = OppositeVCCRValue ; for ( ; Iter != End ; ++ Iter ) { bool IsInteresting = false ; if ( MachineOperand * MO = Iter -> findRegisterUseOperand ( VCCRValue ) ) { IsInteresting = true ; if ( Iter -> getOpcode ( ) == <STR_LIT> ) { Register Result = Iter -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MRI -> replaceRegWith ( Result , LastVPNOTResult ) ; DeadInstructions . push_back ( & * Iter ) ; Modified = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Result ) << <STR_LIT> << printReg ( LastVPNOTResult ) << <STR_LIT> ) ; } else { MachineInstr & VPNOT = ReplaceRegisterUseWithVPNOT ( MBB , * Iter , * MO , LastVPNOTResult ) ; Modified = true ; LastVPNOTResult = VPNOT . getOperand ( <NUM_LIT> ) . getReg ( ) ; std :: swap ( VCCRValue , OppositeVCCRValue ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( VCCRValue ) << <STR_LIT> << printReg ( LastVPNOTResult ) << <STR_LIT> << * Iter ) ; } } else { if ( MachineOperand * MO = Iter -> findRegisterUseOperand ( OppositeVCCRValue ) ) { IsInteresting = true ; if ( LastVPNOTResult != OppositeVCCRValue ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( OppositeVCCRValue ) << <STR_LIT> << printReg ( LastVPNOTResult ) << <STR_LIT> ; Iter -> dump ( ) ) ; MO -> setReg ( LastVPNOTResult ) ; Modified = true ; } MO -> setIsKill ( false ) ; } if ( Iter -> getOpcode ( ) == <STR_LIT> && getVPTInstrPredicate ( * Iter ) == <STR_LIT> ) { Register VPNOTOperand = Iter -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( VPNOTOperand == LastVPNOTResult || VPNOTOperand == OppositeVCCRValue ) { IsInteresting = true ; std :: swap ( VCCRValue , OppositeVCCRValue ) ; LastVPNOTResult = Iter -> getOperand ( <NUM_LIT> ) . getReg ( ) ; } }


<s> MachineInstr & MVEVPTOptimisations :: ReplaceRegisterUseWithVPNOT ( MachineBasicBlock & MBB , MachineInstr & Instr , MachineOperand & User , Register Target ) { Register NewResult = MRI -> createVirtualRegister ( MRI -> getRegClass ( Target ) ) ; MachineInstrBuilder MIBuilder = BuildMI ( MBB , & Instr , Instr . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addDef ( NewResult ) . addReg ( Target ) ;

addUnpredicatedMveVpredNOp ( MIBuilder ) ;