<s> IRBuilder :: CreateSectionData ( * m_pscommon_1 ) ; m_pscommon_2 = pBuilder . CreateSection ( <STR_LIT> , LDFileFormat :: Target , llvm :: ELF :: SHT_PROGBITS , llvm :: ELF :: SHF_ALLOC | llvm :: ELF :: SHF_WRITE , <NUM_LIT> ) ; IRBuilder :: CreateSectionData ( * m_pscommon_2 ) ; m_pscommon_4 = pBuilder . CreateSection ( <STR_LIT> , LDFileFormat :: Target , llvm :: ELF :: SHT_PROGBITS , llvm :: ELF :: SHF_ALLOC | llvm :: ELF :: SHF_WRITE , <NUM_LIT> ) ; IRBuilder :: CreateSectionData ( * m_pscommon_4 ) ; m_pscommon_8 = pBuilder . CreateSection ( <STR_LIT> , LDFileFormat :: Target , llvm :: ELF :: SHT_PROGBITS , llvm :: ELF :: SHF_ALLOC | llvm :: ELF :: SHF_WRITE , <NUM_LIT> ) ;

IRBuilder :: CreateSectionData ( * m_pscommon_8 ) ;


<s> bool <ISA_LIT>LDBackend :: initTargetStubs ( ) { if ( getStubFactory ( ) != NULL ) { getStubFactory ( ) -> addPrototype ( new <ISA_LIT>AbsoluteStub ( config ( ) . isCodeIndep ( ) ) ) ; return true ;


<s> pBuilder . AddSymbol < IRBuilder :: AsReferred , IRBuilder :: Resolve > ( <STR_LIT> , ResolveInfo :: Object , ResolveInfo :: Define , ResolveInfo :: Absolute , <NUM_LIT> , <NUM_LIT> , FragmentRef :: Null ( ) , ResolveInfo :: Hidden ) ; pBuilder . AddSymbol < IRBuilder :: AsReferred , IRBuilder :: Resolve > ( <STR_LIT> , ResolveInfo :: Object , ResolveInfo :: Define , ResolveInfo :: Absolute , <NUM_LIT> , <NUM_LIT> , FragmentRef :: Null ( ) , ResolveInfo :: Hidden ) ;


<s> void <ISA_LIT>LDBackend :: initTargetSymbols ( IRBuilder & pBuilder , Module & pModule ) { if ( config ( ) . codeGenType ( ) == LinkerConfig :: Object ) return ; m_pGOTSymbol = pBuilder . AddSymbol < IRBuilder :: AsReferred , IRBuilder :: Resolve > ( <STR_LIT> , ResolveInfo :: Object , ResolveInfo :: Define , ResolveInfo :: Local , <NUM_LIT> , <NUM_LIT> , FragmentRef :: Null ( ) , ResolveInfo :: Hidden ) ; m_psdabase = pBuilder . AddSymbol < IRBuilder :: AsReferred , IRBuilder :: Resolve > ( <STR_LIT> , ResolveInfo :: Object , ResolveInfo :: Define , ResolveInfo :: Absolute , <NUM_LIT> , <NUM_LIT> , FragmentRef :: Null ( ) , ResolveInfo :: Hidden ) ;


<s> return ~ ( ~ <NUM_LIT> << <NUM_LIT> ) ;


<s> void MCLDInitialize<ISA_LIT>LDBackend ( ) {

mcld :: TargetRegistry :: RegisterTargetLDBackend ( mcld :: The<ISA_LIT>Target , mcld :: create<ISA_LIT>LDBackend ) ;


<s> if ( ! m_psdata -> hasSectionData ( ) ) { sd = IRBuilder :: CreateSectionData ( * m_psdata ) ; m_psdata -> setSectionData ( sd ) ; } sd = m_psdata -> getSectionData ( ) ; MoveSectionDataAndSort ( * pInputSection . getSectionData ( ) , * sd ) ; } else { ObjectBuilder builder ( pModule ) ; builder . MergeSection ( pInputFile , pInputSection ) ; }


<s> AlignFragment * align = NULL ; if ( pFrom . getSection ( ) . align ( ) > <NUM_LIT> ) { unsigned int alignment = pFrom . getSection ( ) . align ( ) ; align = new AlignFragment ( alignment , <NUM_LIT> , <NUM_LIT> , alignment - <NUM_LIT> ) ; pFrom . getFragmentList ( ) . push_front ( align ) ; } if ( found ) to_list . splice ( fragInsert , pFrom . getFragmentList ( ) ) ;

else to_list . splice ( frag , pFrom . getFragmentList ( ) ) ;


<s> } AlignFragment * align = NULL ; if ( pFrom . getSection ( ) . align ( ) > <NUM_LIT> ) { unsigned int alignment = pFrom . getSection ( ) . align ( ) ; align = new AlignFragment ( alignment , <NUM_LIT> , <NUM_LIT> , alignment - <NUM_LIT> ) ; pFrom . getFragmentList ( ) . push_front ( align ) ; } if ( found ) to_list . splice ( fragInsert , pFrom . getFragmentList ( ) ) ; else to_list . splice ( frag , pFrom . getFragmentList ( ) ) ; uint32_t offset = <NUM_LIT> ; for ( frag = to_list . begin ( ) ; frag != fragEnd ; ++ frag ) { frag -> setOffset ( offset ) ; offset += frag -> size ( ) ; } pTo . getSection ( ) . setSize ( offset ) ; if ( pFrom . getSection ( ) . align ( ) > pTo . getSection ( ) . align ( ) ) pTo . getSection ( ) . setAlign ( pFrom . getSection ( ) . align ( ) ) ; if ( pFrom . getSection ( ) . flag ( ) > pTo . getSection ( ) . flag ( ) ) pTo . getSection ( ) . setFlag ( pFrom . getSection ( ) . flag ( ) ) ; return true ;


<s> uint32_t size = pSD . getSection ( ) . size ( ) ; if ( pSD . getSection ( ) . type ( ) == llvm :: ELF :: SHT_NOBITS ) { frag = new FillFragment ( <NUM_LIT> , <NUM_LIT> , size ) ; } else { llvm :: StringRef region = pInput . memArea ( ) -> request ( offset , size ) ; if ( region . size ( ) == <NUM_LIT> ) {

frag = new FillFragment ( <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) ;


<s> void <ISA_LIT>LDBackend :: setGOTSectionSize ( IRBuilder & pBuilder ) {

if ( LinkerConfig :: DynObj == config ( ) . codeGenType ( ) || m_pGOTPLT -> hasGOT1 ( ) || m_pGOTSymbol != NULL ) {


<s> ELFFileFormat * file_format = getOutputFormat ( ) ;

file_format -> getRelaDyn ( ) . setSize ( m_pRelaDyn -> numOfRelocs ( ) * getRelaEntrySize ( ) ) ;


<s> ELFFileFormat * file_format = getOutputFormat ( ) ; file_format -> getRelaPlt ( ) . setSize ( m_pRelaPLT -> numOfRelocs ( ) * getRelaEntrySize ( ) ) ;


<s> SectionData :: FragmentListType & to_list = pTo -> getFragmentList ( ) ; SectionData :: FragmentListType :: iterator fragTo , fragToEnd = to_list . end ( ) ; uint32_t offset = <NUM_LIT> ; for ( fragTo = to_list . begin ( ) ; fragTo != fragToEnd ; ++ fragTo ) { fragTo -> setOffset ( offset ) ; offset += fragTo -> size ( ) ; } pTo -> getSection ( ) . setSize ( offset ) ; SectionData :: FragmentListType & newlist = pTo -> getFragmentList ( ) ; for ( fragTo = newlist . begin ( ) , fragToEnd = newlist . end ( ) ;


<s> <ISA_LIT>LDBackend :: ~ <ISA_LIT>LDBackend ( ) { delete m_pRelocator ; delete m_pGOT ; delete m_pPLT ; delete m_pRelaDyn ;


<s> delete m_pRelaDyn ; delete m_pRelaPLT ; delete m_pDynamic ;


<s> void PolynomialMultiplyRecognize :: cleanupLoopBody ( BasicBlock * LoopB ) { for ( auto & I : * LoopB ) if ( Value * SV = SimplifyInstruction ( & I , DL , & TLI , & DT ) ) I . replaceAllUsesWith ( SV ) ;

for ( auto I = LoopB -> begin ( ) , N = I ; I != LoopB -> end ( ) ; I = N ) {


<s> return CV ; } ; for ( auto I = LoopB -> begin ( ) , E = LoopB -> end ( ) ; I != E ; ++ I ) { if ( isa < PHINode > ( I ) || ! Users . count ( & * I ) ) continue ; using namespace PatternMatch ; Value * V = nullptr ; if ( match ( & * I , m_LShr ( m_Value ( V ) , m_One ( ) ) ) ) { replaceAllUsesOfWithIn ( & * I , V , LoopB ) ; continue ; } for ( auto & J : I -> operands ( ) ) { Value * Op = J . get ( ) ; if ( ! isOperandShifted ( & * I , Op ) ) continue ; if ( Users . count ( Op ) ) continue ; if ( isa < ConstantInt > ( Op ) && cast < ConstantInt > ( Op ) -> isZero ( ) ) continue ; auto F = ShiftMap . find ( Op ) ; Value * W = ( F != ShiftMap . end ( ) ) ? F -> second : nullptr ; if ( W == nullptr ) { IRB . SetInsertPoint ( & * I ) ; Value * ShAmt = CIV , * ShVal = Op ; auto * VTy = cast < IntegerType > ( ShVal -> getType ( ) ) ; auto * ATy = cast < IntegerType > ( ShAmt -> getType ( ) ) ; if ( Late . count ( & * I ) ) ShVal = IRB . CreateShl ( Op , ConstantInt :: get ( VTy , <NUM_LIT> ) ) ; if ( VTy != ATy ) { if ( VTy -> getBitWidth ( ) < ATy -> getBitWidth ( ) ) ShVal = upcast ( CastMap , IRB , ShVal , ATy ) ; else ShAmt = upcast ( CastMap , IRB , ShAmt , VTy ) ; } W = IRB . CreateShl ( ShVal , ShAmt ) ; ShiftMap . insert ( std :: make_pair ( Op , W ) ) ; } I -> replaceUsesOfWith ( Op , W ) ; } } IRB . SetInsertPoint ( ExitB , ExitB -> getFirstInsertionPt ( ) ) ; for ( auto P = ExitB -> begin ( ) , Q = ExitB -> end ( ) ; P != Q ; ++ P ) { if ( ! isa < PHINode > ( P ) ) break ; auto * PN = cast < PHINode > ( P ) ; Value * U = PN -> getIncomingValueForBlock ( LoopB ) ; if ( ! Users . count ( U ) ) continue ;

Value * S = IRB . CreateLShr ( PN , ConstantInt :: get ( PN -> getType ( ) , IterCount ) ) ;


<s> uint64_t SizeInBits = DL -> getTypeSizeInBits ( SI -> getValueOperand ( ) -> getType ( ) ) ; assert ( ( ( SizeInBits & <NUM_LIT> ) || ( SizeInBits >> <NUM_LIT> ) == <NUM_LIT> ) && <STR_LIT> ) ;

return ( unsigned ) SizeInBits >> <NUM_LIT> ;


<s> unsigned <ISA_LIT>LoopIdiomRecognize :: getStoreSizeInBytes ( StoreInst * SI ) { uint64_t SizeInBits = DL -> getTypeSizeInBits ( SI -> getValueOperand ( ) -> getType ( ) ) ; assert ( ( ( SizeInBits & <NUM_LIT> ) || ( SizeInBits >> <NUM_LIT> ) == <NUM_LIT> ) && <STR_LIT> ) ; return ( unsigned ) SizeInBits >> <NUM_LIT> ;


<s> bool PolynomialMultiplyRecognize :: highBitsAreZero ( Value * V , unsigned IterCount ) { auto * T = dyn_cast < IntegerType > ( V -> getType ( ) ) ; if ( ! T ) return false ; unsigned BW = T -> getBitWidth ( ) ; APInt K0 ( BW , <NUM_LIT> ) , K1 ( BW , <NUM_LIT> ) ; computeKnownBits ( V , K0 , K1 , DL ) ; return K0 . countLeadingOnes ( ) >= IterCount ;


<s> auto * T = dyn_cast < IntegerType > ( V -> getType ( ) ) ; if ( ! T ) return false ; unsigned BW = T -> getBitWidth ( ) ; APInt K0 ( BW , <NUM_LIT> ) , K1 ( BW , <NUM_LIT> ) ; computeKnownBits ( V , K0 , K1 , DL ) ; return K0 . countLeadingOnes ( ) >= IterCount ;


<s> unsigned StoreSize = getStoreSizeInBytes ( SI ) ; if ( StoreSize != unsigned ( std :: abs ( Stride ) ) ) return false ; LoadInst * LI = dyn_cast < LoadInst > ( SI -> getValueOperand ( ) ) ; if ( ! LI || ! LI -> isSimple ( ) ) return false ; Value * LoadPtr = LI -> getPointerOperand ( ) ; auto * LoadEv = dyn_cast < SCEVAddRecExpr > ( SE -> getSCEV ( LoadPtr ) ) ; if ( ! LoadEv || LoadEv -> getLoop ( ) != CurLoop || ! LoadEv -> isAffine ( ) ) return false ;

if ( StoreEv -> getOperand ( <NUM_LIT> ) != LoadEv -> getOperand ( <NUM_LIT> ) ) return false ;


<s> bool <ISA_LIT>LoopIdiomRecognize :: isLegalStore ( Loop * CurLoop , StoreInst * SI ) { if ( ! ( SI -> isVolatile ( ) && <ISA_LIT>VolatileMemcpy ) && ! SI -> isSimple ( ) ) return false ; Value * StoredVal = SI -> getValueOperand ( ) ; Value * StorePtr = SI -> getPointerOperand ( ) ; uint64_t SizeInBits = DL -> getTypeSizeInBits ( StoredVal -> getType ( ) ) ; if ( ( SizeInBits & <NUM_LIT> ) || ( SizeInBits >> <NUM_LIT> ) != <NUM_LIT> ) return false ; auto * StoreEv = dyn_cast < SCEVAddRecExpr > ( SE -> getSCEV ( StorePtr ) ) ; if ( ! StoreEv || StoreEv -> getLoop ( ) != CurLoop || ! StoreEv -> isAffine ( ) ) return false ;

int Stride = getSCEVStride ( StoreEv ) ;


<s> if ( auto * I = dyn_cast < Instruction > ( V ) ) { switch ( I -> getOpcode ( ) ) { case Instruction :: And : case Instruction :: Or : case Instruction :: Xor : case Instruction :: LShr : case Instruction :: Select : case Instruction :: ICmp : case Instruction :: PHI : return true ; } } return false ;


<s> if ( const SCEVConstant * BECst = dyn_cast < SCEVConstant > ( BECount ) ) AccessSize = ( BECst -> getValue ( ) -> getZExtValue ( ) + <NUM_LIT> ) * StoreSize ; MemoryLocation StoreLoc ( Ptr , AccessSize ) ;

for ( auto * B : L -> blocks ( ) ) for ( auto & I : * B ) if ( Ignored . count ( & I ) == <NUM_LIT> && ( AA . getModRefInfo ( & I , StoreLoc ) & Access ) ) return true ;


<s> ParsedValues ( ) : M ( nullptr ) , P ( nullptr ) , Q ( nullptr ) , R ( nullptr ) , X ( nullptr ) , Res ( nullptr ) , IterCount ( <NUM_LIT> ) , Left ( false ) , Inv ( false ) {


<s> StringRef PP = ( PV . M ? <STR_LIT> : <STR_LIT> ) ; if ( ! PV . Inv ) dbgs ( ) << <STR_LIT> << PP << <STR_LIT> ; else dbgs ( ) << <STR_LIT> << PP << <STR_LIT> << PP << <STR_LIT> ; dbgs ( ) << <STR_LIT> << * PV . Res << <STR_LIT> << * PV . P << <STR_LIT> ; if ( PV . M ) dbgs ( ) << <STR_LIT> << * PV . M << <STR_LIT> ; dbgs ( ) << <STR_LIT> << * PV . Q << <STR_LIT> ; dbgs ( ) << <STR_LIT> << PV . IterCount << <STR_LIT> ; } ) ; BasicBlock :: iterator At ( EntryB -> getTerminator ( ) ) ; Value * PM = generate ( At , PV ) ; if ( PM == nullptr ) return false ;

if ( PM -> getType ( ) != PV . Res -> getType ( ) ) PM = IRBuilder < > ( & * At ) . CreateIntCast ( PM , PV . Res -> getType ( ) , false ) ;


<s> if ( ExitB == nullptr ) return false ; BasicBlock * EntryB = CurLoop -> getLoopPreheader ( ) ; if ( EntryB == nullptr ) return false ; unsigned IterCount = <NUM_LIT> ; const SCEV * CT = SE . getBackedgeTakenCount ( CurLoop ) ; if ( isa < SCEVCouldNotCompute > ( CT ) ) return false ; if ( auto * CV = dyn_cast < SCEVConstant > ( CT ) ) IterCount = CV -> getValue ( ) -> getZExtValue ( ) + <NUM_LIT> ; Value * CIV = getCountIV ( LoopB ) ; ParsedValues PV ; PV . IterCount = IterCount ; auto CouldBePmpy = [ this , LoopB , EntryB , CIV , & PV ] ( bool PreScan ) -> std :: function < bool ( Instruction & I ) > { return [ this , LoopB , EntryB , CIV , & PV , PreScan ] ( Instruction & I ) -> bool { if ( auto * SelI = dyn_cast < SelectInst > ( & I ) ) return scanSelect ( SelI , LoopB , EntryB , CIV , PV , PreScan ) ; return false ; } ; } ; auto PreF = std :: find_if ( LoopB -> begin ( ) , LoopB -> end ( ) , CouldBePmpy ( true ) ) ; if ( PreF == LoopB -> end ( ) ) return false ;


<s> } ; if ( ! std :: all_of ( ExitBlocks . begin ( ) , ExitBlocks . end ( ) , DominatedByBB ) ) return false ; bool MadeChange = false ; SmallVector < StoreInst * , <NUM_LIT> > Stores ;

collectStores ( CurLoop , BB , Stores ) ;


<s> return DT -> dominates ( BB , EB ) ; } ; if ( ! std :: all_of ( ExitBlocks . begin ( ) , ExitBlocks . end ( ) , DominatedByBB ) ) return false ; bool MadeChange = false ; SmallVector < StoreInst * , <NUM_LIT> > Stores ; collectStores ( CurLoop , BB , Stores ) ; for ( auto & SI : Stores ) MadeChange |= processCopyingStore ( CurLoop , SI , BECount ) ;


<s> StringRef Name = L -> getHeader ( ) -> getParent ( ) -> getName ( ) ; if ( Name == <STR_LIT> || Name == <STR_LIT> || Name == <STR_LIT> ) return false ; AA = & getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; DL = & L -> getHeader ( ) -> getModule ( ) -> getDataLayout ( ) ; DT = & getAnalysis < DominatorTreeWrapperPass > ( ) . getDomTree ( ) ; LF = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; TLI = & getAnalysis < TargetLibraryInfoWrapperPass > ( ) . getTLI ( * L -> getHeader ( ) -> getParent ( ) ) ; SE = & getAnalysis < ScalarEvolutionWrapperPass > ( ) . getSE ( ) ;

HasMemcpy = TLI -> has ( LibFunc_memcpy ) ;


<s> for ( auto * B : L -> blocks ( ) ) for ( auto & I : * B ) if ( Ignored . count ( & I ) == <NUM_LIT> && isModOrRefSet ( intersectModRef ( AA . getModRefInfo ( & I , StoreLoc ) , Access ) ) ) return true ; return false ;


<s> static bool mayLoopAccessLocation ( Value * Ptr , ModRefInfo Access , Loop * L , const SCEV * BECount , unsigned StoreSize , AliasAnalysis & AA , SmallPtrSetImpl < Instruction * > & Ignored ) { uint64_t AccessSize = MemoryLocation :: UnknownSize ; if ( const SCEVConstant * BECst = dyn_cast < SCEVConstant > ( BECount ) ) AccessSize = ( BECst -> getValue ( ) -> getZExtValue ( ) + <NUM_LIT> ) * StoreSize ; MemoryLocation StoreLoc ( Ptr , AccessSize ) ;

for ( auto * B : L -> blocks ( ) ) for ( auto & I : * B ) if ( Ignored . count ( & I ) == <NUM_LIT> && isModOrRefSet ( intersectModRef ( AA . getModRefInfo ( & I , StoreLoc ) , Access ) ) ) return true ;


<s> unsigned DestBW = DestTy -> getBitWidth ( ) ; if ( PHINode * P = dyn_cast < PHINode > ( In ) ) { unsigned N = P -> getNumIncomingValues ( ) ; for ( unsigned i = <NUM_LIT> ; i != N ; ++ i ) { BasicBlock * InB = P -> getIncomingBlock ( i ) ; if ( InB == LoopB ) continue ; Value * InV = P -> getIncomingValue ( i ) ; IntegerType * Ty = cast < IntegerType > ( InV -> getType ( ) ) ; if ( Ty != P -> getType ( ) ) { assert ( Ty -> getBitWidth ( ) < DestBW ) ; InV = IRBuilder < > ( InB -> getTerminator ( ) ) . CreateZExt ( InV , DestTy ) ; P -> setIncomingValue ( i , InV ) ; } } } else if ( ZExtInst * Z = dyn_cast < ZExtInst > ( In ) ) { Value * Op = Z -> getOperand ( <NUM_LIT> ) ; if ( Op -> getType ( ) == Z -> getType ( ) ) Z -> replaceAllUsesWith ( Op ) ; Z -> eraseFromParent ( ) ; return ; } if ( TruncInst * T = dyn_cast < TruncInst > ( In ) ) { IntegerType * TruncTy = cast < IntegerType > ( OrigTy ) ; Value * Mask = ConstantInt :: get ( DestTy , ( <NUM_LIT> << TruncTy -> getBitWidth ( ) ) - <NUM_LIT> ) ; Value * And = IRBuilder < > ( In ) . CreateAnd ( T -> getOperand ( <NUM_LIT> ) , Mask ) ;


<s> } for ( Instruction & In : * LoopB ) if ( ! In . isTerminator ( ) && ! isPromotableTo ( & In , DestTy ) ) return false ; std :: vector < Instruction * > LoopIns ; std :: transform ( LoopB -> begin ( ) , LoopB -> end ( ) , std :: back_inserter ( LoopIns ) , [ ] ( Instruction & In ) { return & In ; } ) ; for ( Instruction * In : LoopIns ) promoteTo ( In , DestTy , LoopB ) ; Instruction * EndI = ExitB -> getFirstNonPHI ( ) ; BasicBlock :: iterator End = EndI ? EndI -> getIterator ( ) : ExitB -> end ( ) ; for ( auto I = ExitB -> begin ( ) ; I != End ; ++ I ) { PHINode * P = dyn_cast < PHINode > ( I ) ; if ( ! P ) break ; Type * Ty0 = P -> getIncomingValue ( <NUM_LIT> ) -> getType ( ) ; Type * PTy = P -> getType ( ) ; if ( PTy != Ty0 ) {


<s> if ( P . getNumIncomingValues ( ) != <NUM_LIT> ) return false ; assert ( P . getIncomingBlock ( <NUM_LIT> ) == LoopB ) ; IntegerType * T = dyn_cast < IntegerType > ( P . getType ( ) ) ; if ( ! T || T -> getBitWidth ( ) > DestBW ) return false ; } for ( Instruction & In : * LoopB ) if ( ! In . isTerminator ( ) && ! isPromotableTo ( & In , DestTy ) ) return false ; std :: vector < Instruction * > LoopIns ; std :: transform ( LoopB -> begin ( ) , LoopB -> end ( ) , std :: back_inserter ( LoopIns ) , [ ] ( Instruction & In ) { return & In ; } ) ; for ( Instruction * In : LoopIns ) promoteTo ( In , DestTy , LoopB ) ; Instruction * EndI = ExitB -> getFirstNonPHI ( ) ; BasicBlock :: iterator End = EndI ? EndI -> getIterator ( ) : ExitB -> end ( ) ;

for ( auto I = ExitB -> begin ( ) ; I != End ; ++ I ) {


<s> for ( auto & In : * B ) { if ( isa < BranchInst > ( In ) || isa < DbgInfoIntrinsic > ( In ) ) continue ; if ( ! Worklist . count ( & In ) && In . mayHaveSideEffects ( ) ) return false ; for ( auto K : In . users ( ) ) { Instruction * UseI = dyn_cast < Instruction > ( K ) ; if ( ! UseI ) continue ; BasicBlock * UseB = UseI -> getParent ( ) ;

if ( LF -> getLoopFor ( UseB ) != L ) return false ;


<s> SmallSet < BasicBlock * , <NUM_LIT> > LoopBlocks ; for ( auto * B : L -> blocks ( ) ) LoopBlocks . insert ( B ) ; SetVector < Instruction * > Worklist ( Insts . begin ( ) , Insts . end ( ) ) ; for ( unsigned i = <NUM_LIT> ; i < Worklist . size ( ) ; ++ i ) { Instruction * In = Worklist [ i ] ; for ( auto I = In -> op_begin ( ) , E = In -> op_end ( ) ; I != E ; ++ I ) { Instruction * OpI = dyn_cast < Instruction > ( I ) ; if ( ! OpI ) continue ;

BasicBlock * PB = OpI -> getParent ( ) ;


<s> return new <ISA_LIT>LoopIdiomRecognizeLegacyPass ( ) ;


<s> Pass * llvm :: create<ISA_LIT>LoopIdiomPass ( ) {

return new <ISA_LIT>LoopIdiomRecognizeLegacyPass ( ) ;


<s> Module * M = At -> getParent ( ) -> getParent ( ) -> getParent ( ) ; Function * PMF = <STR_LIT> ( M , <STR_LIT> ) ; Value * P = PV . P , * Q = PV . Q , * P0 = P ; unsigned IC = PV . IterCount ; if ( PV . M != nullptr ) P0 = P = B . CreateXor ( P , PV . M ) ; auto * BMI = ConstantInt :: get ( P -> getType ( ) , APInt :: getLowBitsSet ( <NUM_LIT> , IC ) ) ; if ( PV . IterCount != <NUM_LIT> ) P = B . CreateAnd ( P , BMI ) ; if ( PV . Inv ) { auto * QI = dyn_cast < ConstantInt > ( PV . Q ) ; assert ( QI && QI -> getBitWidth ( ) <= <NUM_LIT> ) ;

unsigned M = ( <NUM_LIT> << PV . IterCount ) - <NUM_LIT> ;


<s> AU . addRequiredID ( LCSSAID ) ; AU . addRequired < AAResultsWrapperPass > ( ) ;

AU . addRequired < ScalarEvolutionWrapperPass > ( ) ;


<s> AU . addRequired < AAResultsWrapperPass > ( ) ; AU . addRequired < ScalarEvolutionWrapperPass > ( ) ; AU . addRequired < DominatorTreeWrapperPass > ( ) ;

AU . addRequired < TargetLibraryInfoWrapperPass > ( ) ;


<s> BasicBlock * PB = ( * PI == BB ) ? * std :: next ( PI ) : * PI ; for ( auto I = BB -> begin ( ) , E = BB -> end ( ) ; I != E && isa < PHINode > ( I ) ; ++ I ) { auto * PN = cast < PHINode > ( I ) ; Value * InitV = PN -> getIncomingValueForBlock ( PB ) ; if ( ! isa < ConstantInt > ( InitV ) || ! cast < ConstantInt > ( InitV ) -> isZero ( ) ) continue ; Value * IterV = PN -> getIncomingValueForBlock ( BB ) ; auto * BO = dyn_cast < BinaryOperator > ( IterV ) ; if ( ! BO ) continue ; if ( BO -> getOpcode ( ) != Instruction :: Add ) continue ; Value * IncV = nullptr ; if ( BO -> getOperand ( <NUM_LIT> ) == PN ) IncV = BO -> getOperand ( <NUM_LIT> ) ; else if ( BO -> getOperand ( <NUM_LIT> ) == PN ) IncV = BO -> getOperand ( <NUM_LIT> ) ; if ( IncV == nullptr ) continue ;

if ( auto * T = dyn_cast < ConstantInt > ( IncV ) ) if ( T -> getZExtValue ( ) == <NUM_LIT> ) return PN ;


<s> explicit <ISA_LIT>LoopIdiomRecognize ( AliasAnalysis * AA , DominatorTree * DT , LoopInfo * LF , const TargetLibraryInfo * TLI , ScalarEvolution * SE ) : AA ( AA ) , DT ( DT ) , LF ( LF ) , TLI ( TLI ) , SE ( SE ) {


<s> LocationSize AccessSize = LocationSize :: afterPointer ( ) ; if ( const SCEVConstant * BECst = dyn_cast < SCEVConstant > ( BECount ) ) AccessSize = LocationSize :: precise ( ( BECst -> getValue ( ) -> getZExtValue ( ) + <NUM_LIT> ) * StoreSize ) ; MemoryLocation StoreLoc ( Ptr , AccessSize ) ; for ( auto * B : L -> blocks ( ) ) for ( auto & I : * B ) if ( Ignored . count ( & I ) == <NUM_LIT> && isModOrRefSet ( intersectModRef ( AA . getModRefInfo ( & I , StoreLoc ) , Access ) ) ) return true ;


<s> if ( ! S . count ( V ) ) { Q . push_back ( V ) ;


<s> PreservedAnalyses <ISA_LIT>LoopIdiomRecognitionPass :: run ( Loop & L , LoopAnalysisManager & AM , LoopStandardAnalysisResults & AR , LPMUpdater & U ) {

return <ISA_LIT>LoopIdiomRecognize ( & AR . AA , & AR . DT , & AR . LI , & AR . TLI , & AR . SE ) . run ( & L ) ? getLoopPassPreservedAnalyses ( ) : PreservedAnalyses :: all ( ) ;


<s> auto * AA = & getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; auto * DT = & getAnalysis < DominatorTreeWrapperPass > ( ) . getDomTree ( ) ;

auto * LF = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ;


<s> auto * TLI = & getAnalysis < TargetLibraryInfoWrapperPass > ( ) . getTLI ( * L -> getHeader ( ) -> getParent ( ) ) ; auto * SE = & getAnalysis < ScalarEvolutionWrapperPass > ( ) . getSE ( ) ;

return <ISA_LIT>LoopIdiomRecognize ( AA , DT , LF , TLI , SE ) . run ( L ) ;


<s> bool PolynomialMultiplyRecognize :: isPromotableTo ( Value * Val , IntegerType * DestTy ) { IntegerType * T = dyn_cast < IntegerType > ( Val -> getType ( ) ) ; if ( ! T || T -> getBitWidth ( ) > DestTy -> getBitWidth ( ) ) return false ; if ( T -> getBitWidth ( ) == DestTy -> getBitWidth ( ) ) return true ; Instruction * In = dyn_cast < Instruction > ( Val ) ; if ( ! In ) return true ; switch ( In -> getOpcode ( ) ) { case Instruction :: PHI : case Instruction :: ZExt : case Instruction :: And : case Instruction :: Or : case Instruction :: Xor : case Instruction :: LShr :


<s> case Instruction :: And : case Instruction :: Or : case Instruction :: Xor : case Instruction :: LShr : case Instruction :: Select : return true ; case Instruction :: ICmp : if ( CmpInst * CI = cast < CmpInst > ( In ) ) return CI -> isEquality ( ) || CI -> isUnsigned ( ) ; llvm_unreachable ( <STR_LIT> ) ; case Instruction :: Add : return In -> hasNoSignedWrap ( ) && In -> hasNoUnsignedWrap ( ) ; } return false ;


<s> if ( InB == LoopB ) continue ; Value * InV = P -> getIncomingValue ( i ) ; IntegerType * Ty = cast < IntegerType > ( InV -> getType ( ) ) ; if ( Ty != P -> getType ( ) ) { assert ( Ty -> getBitWidth ( ) < DestBW ) ; InV = IRBuilder < > ( InB -> getTerminator ( ) ) . CreateZExt ( InV , DestTy ) ; P -> setIncomingValue ( i , InV ) ; } } } else if ( ZExtInst * Z = dyn_cast < ZExtInst > ( In ) ) { Value * Op = Z -> getOperand ( <NUM_LIT> ) ; if ( Op -> getType ( ) == Z -> getType ( ) ) Z -> replaceAllUsesWith ( Op ) ; Z -> eraseFromParent ( ) ; return ; } for ( unsigned i = <NUM_LIT> , n = In -> getNumOperands ( ) ; i != n ; ++ i ) { if ( ConstantInt * CI = dyn_cast < ConstantInt > ( In -> getOperand ( i ) ) ) if ( CI -> getType ( ) -> getBitWidth ( ) < DestBW ) In -> setOperand ( i , ConstantInt :: get ( DestTy , CI -> getZExtValue ( ) ) ) ; }


<s> if ( ! P ) break ; if ( P -> getNumIncomingValues ( ) != <NUM_LIT> ) return false ; assert ( P -> getIncomingBlock ( <NUM_LIT> ) == LoopB ) ; IntegerType * T = dyn_cast < IntegerType > ( P -> getType ( ) ) ; if ( ! T || T -> getBitWidth ( ) > DestBW ) return false ; } for ( Instruction & In : * LoopB ) if ( ! In . isTerminator ( ) && ! isPromotableTo ( & In , DestTy ) ) return false ; std :: vector < Instruction * > LoopIns ; std :: transform ( LoopB -> begin ( ) , LoopB -> end ( ) , std :: back_inserter ( LoopIns ) , [ ] ( Instruction & In ) { return & In ; } ) ; for ( Instruction * In : LoopIns ) promoteTo ( In , DestTy , LoopB ) ; Instruction * EndI = ExitB -> getFirstNonPHI ( ) ;

BasicBlock :: iterator End = EndI ? EndI -> getIterator ( ) : ExitB -> end ( ) ;


<s> if ( const SCEVConstant * BECst = dyn_cast < SCEVConstant > ( BECount ) ) AccessSize = ( BECst -> getValue ( ) -> getZExtValue ( ) + <NUM_LIT> ) * StoreSize ; MemoryLocation StoreLoc ( Ptr , AccessSize ) ;


<s> MemoryLocation StoreLoc ( Ptr , AccessSize ) ; for ( auto * B : L -> blocks ( ) ) for ( auto & I : * B ) if ( Ignored . count ( & I ) == <NUM_LIT> && isModOrRefSet ( intersectModRef ( AA . getModRefInfo ( & I , StoreLoc ) , Access ) ) ) return true ; return false ;


FoundScan = scanSelect ( SelI , LoopB , EntryB , CIV , PV , false ) ;


<s> if ( FoundScan ) break ; } assert ( FoundScan ) ; DEBUG ( { StringRef PP = ( PV . M ? <STR_LIT> : <STR_LIT> ) ; if ( ! PV . Inv ) dbgs ( ) << <STR_LIT> << PP << <STR_LIT> ; else dbgs ( ) << <STR_LIT> << PP << <STR_LIT> << PP << <STR_LIT> ; dbgs ( ) << <STR_LIT> << * PV . Res << <STR_LIT> << * PV . P << <STR_LIT> ; if ( PV . M ) dbgs ( ) << <STR_LIT> << * PV . M << <STR_LIT> ; dbgs ( ) << <STR_LIT> << * PV . Q << <STR_LIT> ; dbgs ( ) << <STR_LIT> << PV . IterCount << <STR_LIT> ; } ) ; BasicBlock :: iterator At ( EntryB -> getTerminator ( ) ) ; Value * PM = generate ( At , PV ) ; if ( PM == nullptr ) return false ; if ( PM -> getType ( ) != PV . Res -> getType ( ) ) PM = IRBuilder < > ( & * At ) . CreateIntCast ( PM , PV . Res -> getType ( ) , false ) ; PV . Res -> replaceAllUsesWith ( PM ) ; PV . Res -> eraseFromParent ( ) ;


<s> unsigned Count = <NUM_LIT> ; const unsigned Limit = <NUM_LIT> ; while ( ! Q . empty ( ) ) { if ( Count ++ >= Limit ) break ; Instruction * U = dyn_cast < Instruction > ( Q . pop_front_val ( ) ) ; if ( ! U || U -> getParent ( ) || ! C . Used . count ( U ) ) continue ; bool Changed = false ;


<s> if ( Count ++ >= Limit ) break ; Instruction * U = dyn_cast < Instruction > ( Q . pop_front_val ( ) ) ; if ( ! U || U -> getParent ( ) || ! C . Used . count ( U ) ) continue ; bool Changed = false ; for ( Rule & R : Rules ) { Value * W = R ( U , C . Ctx ) ; if ( ! W ) continue ; Changed = true ; C . record ( W ) ; C . replace ( U , W ) ; Q . push_back ( C . Root ) ; break ; } if ( ! Changed ) { for ( Value * Op : U -> operands ( ) ) Q . push_back ( Op ) ; } } assert ( Count < Limit && <STR_LIT> ) ;


<s> for ( auto * B : L -> blocks ( ) ) for ( auto & I : * B ) if ( Ignored . count ( & I ) == <NUM_LIT> && isModOrRefSet ( intersectModRef ( AA . getModRefInfo ( & I , StoreLoc ) , Access ) ) ) return true ;


<s> } if ( ! PV . Left ) { if ( ! promoteTypes ( LoopB , ExitB ) ) return false ; if ( ! convertShiftsToLeft ( LoopB , ExitB , IterCount ) ) return false ; cleanupLoopBody ( LoopB ) ; } bool FoundScan = false ; for ( Instruction & In : * LoopB ) { SelectInst * SelI = dyn_cast < SelectInst > ( & In ) ; if ( ! SelI ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << * SelI << '\n' ) ; FoundScan = scanSelect ( SelI , LoopB , EntryB , CIV , PV , false ) ; if ( FoundScan ) break ; } assert ( FoundScan ) ; DEBUG ( { StringRef PP = ( PV . M ? <STR_LIT> : <STR_LIT> ) ; if ( ! PV . Inv ) dbgs ( ) << <STR_LIT> << PP << <STR_LIT> ; else dbgs ( ) << <STR_LIT> << PP << <STR_LIT> << PP << <STR_LIT> ; dbgs ( ) << <STR_LIT> << * PV . Res << <STR_LIT> << * PV . P << <STR_LIT> ; if ( PV . M ) dbgs ( ) << <STR_LIT> << * PV . M << <STR_LIT> ; dbgs ( ) << <STR_LIT> << * PV . Q << <STR_LIT> ; dbgs ( ) << <STR_LIT> << PV . IterCount << <STR_LIT> ; } ) ; BasicBlock :: iterator At ( EntryB -> getTerminator ( ) ) ; Value * PM = generate ( At , PV ) ;


<s> if ( ! SelI ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << * SelI << '\n' ) ; FoundScan = scanSelect ( SelI , LoopB , EntryB , CIV , PV , false ) ; if ( FoundScan ) break ; } assert ( FoundScan ) ; DEBUG ( { StringRef PP = ( PV . M ? <STR_LIT> : <STR_LIT> ) ; if ( ! PV . Inv ) dbgs ( ) << <STR_LIT> << PP << <STR_LIT> ; else dbgs ( ) << <STR_LIT> << PP << <STR_LIT> << PP << <STR_LIT> ; dbgs ( ) << <STR_LIT> << * PV . Res << <STR_LIT> << * PV . P << <STR_LIT> ; if ( PV . M ) dbgs ( ) << <STR_LIT> << * PV . M << <STR_LIT> ; dbgs ( ) << <STR_LIT> << * PV . Q << <STR_LIT> ; dbgs ( ) << <STR_LIT> << PV . IterCount << <STR_LIT> ; } ) ; BasicBlock :: iterator At ( EntryB -> getTerminator ( ) ) ; Value * PM = generate ( At , PV ) ; if ( PM == nullptr ) return false ;

if ( PM -> getType ( ) != PV . Res -> getType ( ) ) PM = IRBuilder < > ( & * At ) . CreateIntCast ( PM , PV . Res -> getType ( ) , false ) ;


<s> while ( ! Q . empty ( ) ) { if ( Count ++ >= Limit ) break ; Instruction * U = dyn_cast < Instruction > ( Q . pop_front_val ( ) ) ; if ( ! U || U -> getParent ( ) || ! C . Used . count ( U ) ) continue ; bool Changed = false ; for ( Rule & R : Rules ) { Value * W = R ( U , C . Ctx ) ; if ( ! W ) continue ; Changed = true ; C . record ( W ) ; C . replace ( U , W ) ; Q . push_back ( C . Root ) ; break ; } if ( ! Changed ) {

for ( Value * Op : U -> operands ( ) ) Q . push_back ( Op ) ;


<s> Value * Simplifier :: simplify ( Context & C ) { WorkListType Q ; Q . push_back ( C . Root ) ; unsigned Count = <NUM_LIT> ; const unsigned Limit = SimplifyLimit ; while ( ! Q . empty ( ) ) { if ( Count ++ >= Limit ) break ; Instruction * U = dyn_cast < Instruction > ( Q . pop_front_val ( ) ) ; if ( ! U || U -> getParent ( ) || ! C . Used . count ( U ) ) continue ; bool Changed = false ; for ( Rule & R : Rules ) { Value * W = R ( U , C . Ctx ) ; if ( ! W ) continue ; Changed = true ; C . record ( W ) ;


<s> void addRule ( StringRef N , const Rule :: FuncType & F ) {


<s> for ( unsigned J = <NUM_LIT> ; J < I ; ++ J ) First . insert ( Cycle [ J ] ) ; ValueSeq & Second = IsE ? Early : Late ; Second . insert ( Cycle [ I ] ) ; for ( ++ I ; I < N ; ++ I ) {


<s> return true ; } if ( UseI -> getNumOperands ( ) == <NUM_LIT> ) return true ; bool AE = true , AL = true ; for ( auto & I : UseI -> operands ( ) ) { if ( Early . count ( & * I ) ) AL = false ; else if ( Late . count ( & * I ) ) AE = false ; } if ( AE && AL ) return true ; if ( ! AE && ! AL ) return false ; assert ( AE != AL ) ; if ( AE ) Early . insert ( UseI ) ; else Late . insert ( UseI ) ; return true ;


<s> void <ISA_LIT>LoopIdiomRecognize :: collectStores ( Loop * CurLoop , BasicBlock * BB , SmallVectorImpl < StoreInst * > & Stores ) { Stores . clear ( ) ; for ( Instruction & I : * BB ) if ( StoreInst * SI = dyn_cast < StoreInst > ( & I ) ) if ( isLegalStore ( CurLoop , SI ) ) Stores . push_back ( SI ) ;


<s> Stores . clear ( ) ; for ( Instruction & I : * BB ) if ( StoreInst * SI = dyn_cast < StoreInst > ( & I ) ) if ( isLegalStore ( CurLoop , SI ) ) Stores . push_back ( SI ) ;


<s> case Instruction :: Or : case Instruction :: Xor : case Instruction :: LShr : case Instruction :: Shl : case Instruction :: Select : case Instruction :: ICmp : case Instruction :: PHI :


<s> case Instruction :: Xor : case Instruction :: LShr : case Instruction :: Shl : case Instruction :: Select :


<s> Context ( Instruction * Exp ) : Ctx ( Exp -> getParent ( ) -> getParent ( ) -> getContext ( ) ) {


<s> SetVector < Instruction * > Worklist ( Insts . begin ( ) , Insts . end ( ) ) ; for ( unsigned i = <NUM_LIT> ; i < Worklist . size ( ) ; ++ i ) { Instruction * In = Worklist [ i ] ; for ( auto I = In -> op_begin ( ) , E = In -> op_end ( ) ; I != E ; ++ I ) { Instruction * OpI = dyn_cast < Instruction > ( I ) ; if ( ! OpI ) continue ; BasicBlock * PB = OpI -> getParent ( ) ;

if ( ! LoopBlocks . count ( PB ) ) continue ;


<s> if ( ! OpI ) continue ; BasicBlock * PB = OpI -> getParent ( ) ; if ( ! LoopBlocks . count ( PB ) ) continue ; Worklist . insert ( OpI ) ; } } for ( auto * B : L -> blocks ( ) ) { for ( auto & In : * B ) { if ( isa < BranchInst > ( In ) || isa < DbgInfoIntrinsic > ( In ) ) continue ; if ( ! Worklist . count ( & In ) && In . mayHaveSideEffects ( ) ) return false ; for ( const auto & K : In . users ( ) ) { Instruction * UseI = dyn_cast < Instruction > ( K ) ; if ( ! UseI ) continue ; BasicBlock * UseB = UseI -> getParent ( ) ; if ( LF -> getLoopFor ( UseB ) != L ) return false ;


<s> Pass * llvm :: create<ISA_LIT>LoopIdiomPass ( ) { return new <ISA_LIT>LoopIdiomRecognize ( ) ;


<s> bool IsPhi = isa < PHINode > ( I ) ; if ( IsPhi && HadPhi ) return false ; HadPhi |= IsPhi ; if ( Cycle . count ( I ) ) return false ; Cycle . insert ( I ) ; if ( findCycle ( I , In , Cycle ) ) break ; Cycle . remove ( I ) ; }


<s> auto * QI = dyn_cast < ConstantInt > ( PV . Q ) ; assert ( QI && QI -> getBitWidth ( ) <= <NUM_LIT> ) ; unsigned M = ( <NUM_LIT> << PV . IterCount ) - <NUM_LIT> ; unsigned Tmp = ( QI -> getZExtValue ( ) | <NUM_LIT> ) & M ; unsigned QV = getInverseMxN ( Tmp ) & M ; auto * QVI = ConstantInt :: get ( QI -> getType ( ) , QV ) ; P = B . CreateCall ( PMF , { P , QVI } ) ; P = B . CreateTrunc ( P , QI -> getType ( ) ) ; if ( IC != <NUM_LIT> ) P = B . CreateAnd ( P , BMI ) ; } Value * R = B . CreateCall ( PMF , { P , Q } ) ; if ( PV . M != nullptr ) R = B . CreateXor ( R , B . CreateIntCast ( P0 , R -> getType ( ) , false ) ) ; return R ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < LoopInfoWrapperPass > ( ) ; AU . addRequiredID ( LoopSimplifyID ) ; AU . addRequiredID ( LCSSAID ) ; AU . addRequired < AAResultsWrapperPass > ( ) ; AU . addPreserved < AAResultsWrapperPass > ( ) ; AU . addRequired < ScalarEvolutionWrapperPass > ( ) ; AU . addRequired < DominatorTreeWrapperPass > ( ) ; AU . addRequired < TargetLibraryInfoWrapperPass > ( ) ; AU . addPreserved < TargetLibraryInfoWrapperPass > ( ) ;


<s> AU . addRequired < ScalarEvolutionWrapperPass > ( ) ; AU . addRequired < DominatorTreeWrapperPass > ( ) ;


<s> C [ <NUM_LIT> ] = <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) { unsigned T = <NUM_LIT> ; for ( unsigned j = <NUM_LIT> ; j < i ; ++ j ) T = T ^ ( C [ j ] & Q [ i - j ] ) ; C [ i ] = T ; } unsigned QV = <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) if ( C [ i ] ) QV |= ( <NUM_LIT> << i ) ;


<s> return IntegerType :: get ( Ctx , <NUM_LIT> ) ;


<s> IntegerType * getPmpyType ( ) const { LLVMContext & Ctx = CurLoop -> getHeader ( ) -> getParent ( ) -> getContext ( ) ; return IntegerType :: get ( Ctx , <NUM_LIT> ) ;


<s> int <ISA_LIT>LoopIdiomRecognize :: getSCEVStride ( const SCEVAddRecExpr * S ) { if ( const SCEVConstant * SC = dyn_cast < SCEVConstant > ( S -> getOperand ( <NUM_LIT> ) ) ) return SC -> getAPInt ( ) . getSExtValue ( ) ;


<s> int <ISA_LIT>LoopIdiomRecognize :: getSCEVStride ( const SCEVAddRecExpr * S ) {

if ( const SCEVConstant * SC = dyn_cast < SCEVConstant > ( S -> getOperand ( <NUM_LIT> ) ) ) return SC -> getAPInt ( ) . getSExtValue ( ) ;


<s> if ( const auto SI = dyn_cast < const ConstantInt > ( I -> getOperand ( <NUM_LIT> ) ) ) return SI -> getZExtValue ( ) > <NUM_LIT> ; return false ; case Instruction :: Or : case Instruction :: Xor : return hasZeroSignBit ( I -> getOperand ( <NUM_LIT> ) ) && hasZeroSignBit ( I -> getOperand ( <NUM_LIT> ) ) ; case Instruction :: And : return hasZeroSignBit ( I -> getOperand ( <NUM_LIT> ) ) || hasZeroSignBit ( I -> getOperand ( <NUM_LIT> ) ) ; }


<s> KnownBits Known ( T -> getBitWidth ( ) ) ; computeKnownBits ( V , Known , DL ) ; return Known . countMinLeadingZeros ( ) >= IterCount ;


<s> if ( ! T ) return false ; KnownBits Known ( T -> getBitWidth ( ) ) ;


<s> if ( ! ( SI -> isVolatile ( ) && <ISA_LIT>VolatileMemcpy ) && ! SI -> isSimple ( ) ) return false ; Value * StoredVal = SI -> getValueOperand ( ) ; Value * StorePtr = SI -> getPointerOperand ( ) ; uint64_t SizeInBits = DL -> getTypeSizeInBits ( StoredVal -> getType ( ) ) ; if ( ( SizeInBits & <NUM_LIT> ) || ( SizeInBits >> <NUM_LIT> ) != <NUM_LIT> ) return false ; auto * StoreEv = dyn_cast < SCEVAddRecExpr > ( SE -> getSCEV ( StorePtr ) ) ; if ( ! StoreEv || StoreEv -> getLoop ( ) != CurLoop || ! StoreEv -> isAffine ( ) ) return false ; int Stride = getSCEVStride ( StoreEv ) ; if ( Stride == <NUM_LIT> ) return false ; unsigned StoreSize = DL -> getTypeStoreSize ( SI -> getValueOperand ( ) -> getType ( ) ) ; if ( StoreSize != unsigned ( std :: abs ( Stride ) ) ) return false ; LoadInst * LI = dyn_cast < LoadInst > ( SI -> getValueOperand ( ) ) ; if ( ! LI || ! LI -> isSimple ( ) ) return false ; Value * LoadPtr = LI -> getPointerOperand ( ) ; auto * LoadEv = dyn_cast < SCEVAddRecExpr > ( SE -> getSCEV ( LoadPtr ) ) ;

if ( ! LoadEv || LoadEv -> getLoop ( ) != CurLoop || ! LoadEv -> isAffine ( ) ) return false ;


<s> bool PolynomialMultiplyRecognize :: isOperandShifted ( Instruction * I , Value * Op ) {


if ( Opc == Instruction :: Shl || Opc == Instruction :: LShr ) return Op != I -> getOperand ( <NUM_LIT> ) ;


<s> if ( ! T || T -> getBitWidth ( ) > DestTy -> getBitWidth ( ) ) return false ; if ( T -> getBitWidth ( ) == DestTy -> getBitWidth ( ) ) return true ; Instruction * In = dyn_cast < Instruction > ( Val ) ; if ( ! In ) return true ; switch ( In -> getOpcode ( ) ) { case Instruction :: PHI : case Instruction :: ZExt :


<s> case Instruction :: Xor : case Instruction :: LShr : case Instruction :: Select : case Instruction :: ICmp : case Instruction :: PHI : case Instruction :: ZExt : return true ; }


<s> Value * T = nullptr ; if ( match ( ShouldXoredV , m_Xor ( m_Value ( Y ) , m_Value ( Z ) ) ) ) { if ( ShouldSameV == Y ) T = Z ; else if ( ShouldSameV == Z ) T = Y ; else return false ; R = ShouldSameV ; } else if ( match ( ShouldSameV , m_Zero ( ) ) ) { if ( ! SelI -> hasOneUse ( ) ) return false ; T = ShouldXoredV ; Value * U = * SelI -> user_begin ( ) ; if ( ! match ( U , m_Xor ( m_Specific ( SelI ) , m_Value ( R ) ) ) && ! match ( U , m_Xor ( m_Value ( R ) , m_Specific ( SelI ) ) ) ) return false ; } else return false ;

if ( ! match ( T , m_Shl ( m_Value ( Q ) , m_Specific ( CIV ) ) ) && ! match ( T , m_Shl ( m_ZExt ( m_Value ( Q ) ) , m_ZExt ( m_Specific ( CIV ) ) ) ) ) return false ;


<s> if ( match ( CondV , m_ICmp ( P , m_Value ( C ) , m_Zero ( ) ) ) || match ( CondV , m_ICmp ( P , m_Zero ( ) , m_Value ( C ) ) ) ) { if ( P != CmpInst :: ICMP_EQ && P != CmpInst :: ICMP_NE ) return false ; TrueIfZero = ( P == CmpInst :: ICMP_EQ ) ; } else if ( match ( CondV , m_ICmp ( P , m_Value ( C ) , m_One ( ) ) ) || match ( CondV , m_ICmp ( P , m_One ( ) , m_Value ( C ) ) ) ) { if ( P != CmpInst :: ICMP_EQ && P != CmpInst :: ICMP_NE ) return false ; TrueIfZero = ( P == CmpInst :: ICMP_NE ) ; } else return false ; Value * X = nullptr ; if ( ! match ( C , m_And ( m_Value ( X ) , m_One ( ) ) ) && ! match ( C , m_And ( m_One ( ) , m_Value ( X ) ) ) ) return false ; Value * R = nullptr , * Q = nullptr ; if ( TrueIfZero ) { if ( ! match ( TrueV , m_LShr ( m_Value ( R ) , m_One ( ) ) ) ) return false ; if ( ! match ( FalseV , m_Xor ( m_Specific ( TrueV ) , m_Value ( Q ) ) ) && ! match ( FalseV , m_Xor ( m_Value ( Q ) , m_Specific ( TrueV ) ) ) ) return false ;


<s> if ( TrueIfZero ) { if ( ! match ( TrueV , m_LShr ( m_Value ( R ) , m_One ( ) ) ) ) return false ; if ( ! match ( FalseV , m_Xor ( m_Specific ( TrueV ) , m_Value ( Q ) ) ) && ! match ( FalseV , m_Xor ( m_Value ( Q ) , m_Specific ( TrueV ) ) ) ) return false ; } else { if ( ! match ( FalseV , m_LShr ( m_Value ( R ) , m_One ( ) ) ) ) return false ; if ( ! match ( TrueV , m_Xor ( m_Specific ( FalseV ) , m_Value ( Q ) ) ) && ! match ( TrueV , m_Xor ( m_Value ( Q ) , m_Specific ( FalseV ) ) ) ) return false ; } PV . X = X ; PV . Q = Q ;


<s> MemoryLocation StoreLoc ( Ptr , AccessSize ) ;

for ( auto * B : L -> blocks ( ) ) for ( auto & I : * B ) if ( Ignored . count ( & I ) == <NUM_LIT> && isModOrRefSet ( intersectModRef ( AA . getModRefInfo ( & I , StoreLoc ) , Access ) ) ) return true ;


<s> PE ( const Simplifier :: Context & c , Value * v = nullptr ) : C ( c ) , V ( v ) {


<s> explicit PolynomialMultiplyRecognize ( Loop * loop , const DataLayout & dl , const DominatorTree & dt , const TargetLibraryInfo & tli , ScalarEvolution & se ) : CurLoop ( loop ) , DL ( dl ) , DT ( dt ) , TLI ( tli ) , SE ( se ) {


<s> Value * V = Q . front ( ) ; Q . pop_front ( ) ; S . erase ( V ) ;


<s> Z -> eraseFromParent ( ) ; return ; } if ( TruncInst * T = dyn_cast < TruncInst > ( In ) ) { IntegerType * TruncTy = cast < IntegerType > ( OrigTy ) ; Value * Mask = ConstantInt :: get ( DestTy , ( <NUM_LIT> << TruncTy -> getBitWidth ( ) ) - <NUM_LIT> ) ; Value * And = IRBuilder < > ( In ) . CreateAnd ( T -> getOperand ( <NUM_LIT> ) , Mask ) ; T -> replaceAllUsesWith ( And ) ; T -> eraseFromParent ( ) ; return ; } for ( unsigned i = <NUM_LIT> , n = In -> getNumOperands ( ) ; i != n ; ++ i ) {

if ( ConstantInt * CI = dyn_cast < ConstantInt > ( In -> getOperand ( i ) ) ) if ( CI -> getType ( ) -> getBitWidth ( ) < DestBW ) In -> setOperand ( i , ConstantInt :: get ( DestTy , CI -> getZExtValue ( ) ) ) ;


<s> } } else if ( ZExtInst * Z = dyn_cast < ZExtInst > ( In ) ) { Value * Op = Z -> getOperand ( <NUM_LIT> ) ; if ( Op -> getType ( ) == Z -> getType ( ) ) Z -> replaceAllUsesWith ( Op ) ; Z -> eraseFromParent ( ) ; return ; } if ( TruncInst * T = dyn_cast < TruncInst > ( In ) ) { IntegerType * TruncTy = cast < IntegerType > ( OrigTy ) ; Value * Mask = ConstantInt :: get ( DestTy , ( <NUM_LIT> << TruncTy -> getBitWidth ( ) ) - <NUM_LIT> ) ; Value * And = IRBuilder < > ( In ) . CreateAnd ( T -> getOperand ( <NUM_LIT> ) , Mask ) ; T -> replaceAllUsesWith ( And ) ; T -> eraseFromParent ( ) ; return ; } for ( unsigned i = <NUM_LIT> , n = In -> getNumOperands ( ) ; i != n ; ++ i ) {


<s> if ( ! ExitB || ( ExitB -> getSinglePredecessor ( ) != LoopB ) ) return false ; IntegerType * DestTy = getPmpyType ( ) ; unsigned DestBW = DestTy -> getBitWidth ( ) ; for ( PHINode & P : ExitB -> phis ( ) ) { if ( P . getNumIncomingValues ( ) != <NUM_LIT> ) return false ; assert ( P . getIncomingBlock ( <NUM_LIT> ) == LoopB ) ; IntegerType * T = dyn_cast < IntegerType > ( P . getType ( ) ) ; if ( ! T || T -> getBitWidth ( ) > DestBW ) return false ; } for ( Instruction & In : * LoopB ) if ( ! In . isTerminator ( ) && ! isPromotableTo ( & In , DestTy ) ) return false ; std :: vector < Instruction * > LoopIns ; std :: transform ( LoopB -> begin ( ) , LoopB -> end ( ) , std :: back_inserter ( LoopIns ) , [ ] ( Instruction & In ) { return & In ; } ) ; for ( Instruction * In : LoopIns ) if ( ! In -> isTerminator ( ) ) promoteTo ( In , DestTy , LoopB ) ; Instruction * EndI = ExitB -> getFirstNonPHI ( ) ; BasicBlock :: iterator End = EndI ? EndI -> getIterator ( ) : ExitB -> end ( ) ; for ( auto I = ExitB -> begin ( ) ; I != End ; ++ I ) { PHINode * P = dyn_cast < PHINode > ( I ) ; if ( ! P ) break ; Type * Ty0 = P -> getIncomingValue ( <NUM_LIT> ) -> getType ( ) ; Type * PTy = P -> getType ( ) ; if ( PTy != Ty0 ) { assert ( Ty0 == DestTy ) ; P -> mutateType ( Ty0 ) ; Value * T = IRBuilder < > ( ExitB , End ) . CreateTrunc ( P , PTy ) ; P -> mutateType ( PTy ) ;


<s> if ( ! S . count ( V ) ) { Q . push_back ( V ) ; S . insert ( V ) ; }


<s> Q . push_back ( V ) ; S . insert ( V ) ;


<s> bool PolynomialMultiplyRecognize :: recognize ( ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * CurLoop << '\n' ) ; BasicBlock * LoopB = CurLoop -> getHeader ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * LoopB ) ; if ( LoopB != CurLoop -> getLoopLatch ( ) ) return false ; BasicBlock * ExitB = CurLoop -> getExitBlock ( ) ; if ( ExitB == nullptr ) return false ; BasicBlock * EntryB = CurLoop -> getLoopPreheader ( ) ; if ( EntryB == nullptr ) return false ; unsigned IterCount = <NUM_LIT> ; const SCEV * CT = SE . getBackedgeTakenCount ( CurLoop ) ; if ( isa < SCEVCouldNotCompute > ( CT ) ) return false ; if ( auto * CV = dyn_cast < SCEVConstant > ( CT ) ) IterCount = CV -> getValue ( ) -> getZExtValue ( ) + <NUM_LIT> ; Value * CIV = getCountIV ( LoopB ) ; ParsedValues PV ; Simplifier PreSimp ; PV . IterCount = IterCount ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * CIV << <STR_LIT> << IterCount << '\n' ) ; setupPreSimplifier ( PreSimp ) ; bool FoundPreScan = false ; auto FeedsPHI = [ LoopB ] ( const Value * V ) -> bool { for ( const Value * U : V -> users ( ) ) { if ( const auto * P = dyn_cast < const PHINode > ( U ) ) if ( P -> getParent ( ) == LoopB ) return true ; } return false ; } ; for ( Instruction & In : * LoopB ) { SelectInst * SI = dyn_cast < SelectInst > ( & In ) ; if ( ! SI || ! FeedsPHI ( SI ) ) continue ; Simplifier :: Context C ( SI ) ; Value * T = PreSimp . simplify ( C ) ; SelectInst * SelI = ( T && isa < SelectInst > ( T ) ) ? cast < SelectInst > ( T ) : SI ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << PE ( C , SelI ) << '\n' ) ; if ( scanSelect ( SelI , LoopB , EntryB , CIV , PV , true ) ) {


<s> ++ UI ; if ( auto * II = dyn_cast < Instruction > ( TheUse . getUser ( ) ) ) if ( BB == II -> getParent ( ) ) II -> replaceUsesOfWith ( I , J ) ;


<s> for ( auto UI = I -> user_begin ( ) , UE = I -> user_end ( ) ; UI != UE ; ) { Use & TheUse = UI . getUse ( ) ; ++ UI ; if ( auto * II = dyn_cast < Instruction > ( TheUse . getUser ( ) ) ) if ( BB == II -> getParent ( ) ) II -> replaceUsesOfWith ( I , J ) ;


<s> Rule ( StringRef N , FuncType F ) : Name ( N ) , Fn ( F ) {


<s> for ( auto * BB : L -> getBlocks ( ) ) { if ( LF -> getLoopFor ( BB ) != L ) continue ; Changed |= runOnLoopBlock ( L , BB , BECount , ExitBlocks ) ; } return Changed ;


<s> bool <ISA_LIT>LoopIdiomRecognize :: runOnLoopBlock ( Loop * CurLoop , BasicBlock * BB , const SCEV * BECount , SmallVectorImpl < BasicBlock * > & ExitBlocks ) { auto DominatedByBB = [ this , BB ] ( BasicBlock * EB ) -> bool { return DT -> dominates ( BB , EB ) ; } ; if ( ! all_of ( ExitBlocks , DominatedByBB ) ) return false ; bool MadeChange = false ; SmallVector < StoreInst * , <NUM_LIT> > Stores ; collectStores ( CurLoop , BB , Stores ) ;

for ( auto & SI : Stores ) MadeChange |= processCopyingStore ( CurLoop , SI , BECount ) ;


<s> if ( ! RPhi ) return false ; if ( SelI != RPhi -> getIncomingValueForBlock ( LoopB ) ) return false ; PV . Res = SelI ; if ( CurLoop -> isLoopInvariant ( PV . X ) ) { PV . P = PV . X ; PV . Inv = false ; } else { PV . Inv = true ; if ( PV . X != PV . R ) { Value * Var = nullptr , * Inv = nullptr , * X1 = nullptr , * X2 = nullptr ; if ( ! match ( PV . X , m_Xor ( m_Value ( X1 ) , m_Value ( X2 ) ) ) ) return false ; auto * I1 = dyn_cast < Instruction > ( X1 ) ; auto * I2 = dyn_cast < Instruction > ( X2 ) ; if ( ! I1 || I1 -> getParent ( ) != LoopB ) { Var = X2 ; Inv = X1 ; } else if ( ! I2 || I2 -> getParent ( ) != LoopB ) { Var = X1 ; Inv = X2 ; } else return false ; if ( Var != PV . R ) return false ; PV . M = Inv ; } Value * EntryP = RPhi -> getIncomingValueForBlock ( PrehB ) ; PV . P = EntryP ; } return true ; } if ( matchRightShift ( SelI , PV ) ) { if ( PV . Inv && ! isa < ConstantInt > ( PV . Q ) ) return false ; if ( PreScan ) return true ; return false ; }


<s> if ( ! Xor || ! C0 ) return nullptr ; if ( Xor -> getOpcode ( ) != Instruction :: Xor ) return nullptr ; Instruction * And0 = dyn_cast < Instruction > ( Xor -> getOperand ( <NUM_LIT> ) ) ; Instruction * And1 = dyn_cast < Instruction > ( Xor -> getOperand ( <NUM_LIT> ) ) ; if ( ! And0 || And0 -> getOpcode ( ) != Instruction :: And ) std :: swap ( And0 , And1 ) ;

ConstantInt * C1 = dyn_cast < ConstantInt > ( And0 -> getOperand ( <NUM_LIT> ) ) ;


<s> IRBuilder < > B ( Ctx ) ; return B . CreateBinOp ( cast < BinaryOperator > ( T ) -> getOpcode ( ) , B . CreateZExt ( T -> getOperand ( <NUM_LIT> ) , I -> getType ( ) ) , B . CreateZExt ( T -> getOperand ( <NUM_LIT> ) , I -> getType ( ) ) ) ; } ) ; S . addRule ( <STR_LIT> , [ ] ( Instruction * I , LLVMContext & Ctx ) -> Value * { if ( I -> getOpcode ( ) != Instruction :: Xor ) return nullptr ; Instruction * And0 = dyn_cast < Instruction > ( I -> getOperand ( <NUM_LIT> ) ) ; Instruction * And1 = dyn_cast < Instruction > ( I -> getOperand ( <NUM_LIT> ) ) ; if ( ! And0 || ! And1 ) return nullptr ; if ( And0 -> getOpcode ( ) != Instruction :: And || And1 -> getOpcode ( ) != Instruction :: And ) return nullptr ; if ( And0 -> getOperand ( <NUM_LIT> ) != And1 -> getOperand ( <NUM_LIT> ) ) return nullptr ; IRBuilder < > B ( Ctx ) ; return B . CreateAnd ( B . CreateXor ( And0 -> getOperand ( <NUM_LIT> ) , And1 -> getOperand ( <NUM_LIT> ) ) , And0 -> getOperand ( <NUM_LIT> ) ) ; } ) ; S . addRule ( <STR_LIT> , [ ] ( Instruction * I , LLVMContext & Ctx ) -> Value * { BinaryOperator * BO = dyn_cast < BinaryOperator > ( I ) ; if ( ! BO ) return nullptr ; Instruction :: BinaryOps Op = BO -> getOpcode ( ) ; if ( SelectInst * Sel = dyn_cast < SelectInst > ( BO -> getOperand ( <NUM_LIT> ) ) ) { IRBuilder < > B ( Ctx ) ; Value * X = Sel -> getTrueValue ( ) , * Y = Sel -> getFalseValue ( ) ; Value * Z = BO -> getOperand ( <NUM_LIT> ) ; return B . CreateSelect ( Sel -> getCondition ( ) , B . CreateBinOp ( Op , X , Z ) , B . CreateBinOp ( Op , Y , Z ) ) ; } if ( SelectInst * Sel = dyn_cast < SelectInst > ( BO -> getOperand ( <NUM_LIT> ) ) ) { IRBuilder < > B ( Ctx ) ;

Value * X = BO -> getOperand ( <NUM_LIT> ) ;


<s> Q . push_back ( C . Root ) ; unsigned Count = <NUM_LIT> ; const unsigned Limit = SimplifyLimit ; while ( ! Q . empty ( ) ) { if ( Count ++ >= Limit ) break ; Instruction * U = dyn_cast < Instruction > ( Q . pop_front_val ( ) ) ; if ( ! U || U -> getParent ( ) || ! C . Used . count ( U ) ) continue ; bool Changed = false ; for ( Rule & R : Rules ) { Value * W = R . Fn ( U , C . Ctx ) ; if ( ! W ) continue ; Changed = true ; C . record ( W ) ;


<s> int getFirstNamedArgFrameIndex ( ) {


<s> int getRegSavedAreaStartFrameIndex ( ) { return RegSavedAreaStartFrameIndex ;


<s> return RegSavedAreaStartFrameIndex ;


<s> unsigned getStackAlignBasePhysReg ( ) const { return StackAlignBasePhysReg ;


<s> unsigned getStackAlignBaseVReg ( ) const {


<s> <ISA_LIT>MachineFunctionInfo ( MachineFunction & MF ) {


<s> void setFirstNamedArgFrameIndex ( int v ) {


<s> void setLastNamedArgFrameIndex ( int v ) {


<s> void setRegSavedAreaStartFrameIndex ( int v ) {


<s> void setStackAlignBasePhysReg ( unsigned R ) { StackAlignBasePhysReg = R ;


<s> void setStackAlignBasePhysReg ( unsigned R ) {


<s> void setStackAlignBaseVReg ( unsigned R ) { StackAlignBaseVReg = R ;


<s> <ISA_LIT>MachineFunctionInfo ( MachineFunction & MF ) : SRetReturnReg ( <NUM_LIT> ) , HasClobberLR ( <NUM_LIT> ) {


<s> <ISA_LIT>MachineFunctionInfo ( MachineFunction & MF ) : SRetReturnReg ( <NUM_LIT> ) , StackAlignBaseVReg ( <NUM_LIT> ) , StackAlignBasePhysReg ( <NUM_LIT> ) , HasClobberLR ( <NUM_LIT> ) , HasEHReturn ( false ) {


<s> <ISA_LIT>MachineFunctionInfo ( MachineFunction & MF ) : SRetReturnReg ( <NUM_LIT> ) , StackAlignBaseReg ( <NUM_LIT> ) , HasClobberLR ( <NUM_LIT> ) , HasEHReturn ( false ) {


<s> void setStackAlignBaseVReg ( unsigned R ) { StackAlignBaseReg = R ;


<s> void addAllocaAdjustInst ( MachineInstr * MI ) {


<s> AllocaAdjustInsts . push_back ( MI ) ;


<s> void <ISA_LIT>MachineFunctionInfo :: anchor ( ) {


<s> const std :: vector < MachineInstr * > & getAllocaAdjustInsts ( ) { return AllocaAdjustInsts ;


<s> unsigned getSRetReturnReg ( ) const {


<s> <ISA_LIT>MachineFunctionInfo ( MachineFunction & MF ) : SRetReturnReg ( <NUM_LIT> ) , HasClobberLR ( <NUM_LIT> ) , HasEHReturn ( false ) {


<s> bool isEndPacket ( const MachineInstr * MI ) const {

return ( PacketInfo . count ( MI ) && ( PacketInfo . find ( MI ) -> second & <STR_LIT> ) ) ;


<s> bool isStartPacket ( const MachineInstr * MI ) const { return ( PacketInfo . count ( MI ) && ( PacketInfo . find ( MI ) -> second & <STR_LIT> ) ) ;


<s> bool isStartPacket ( const MachineInstr * MI ) const {


<s> void setHasClobberLR ( bool v ) { HasClobberLR = v ;


<s> void setSRetReturnReg ( unsigned Reg ) { SRetReturnReg = Reg ;


<s> SUnit & Pred = * I -> getSUnit ( ) ; if ( ! Pred . isScheduled ) { if ( OnlyAvailablePred && OnlyAvailablePred != & Pred ) return nullptr ; OnlyAvailablePred = & Pred ;


<s> I != E ; ++ I ) { SUnit & Succ = * I -> getSUnit ( ) ; if ( ! Succ . isScheduled ) {

if ( OnlyAvailableSucc && OnlyAvailableSucc != & Succ ) return nullptr ;


<s> DAG = static_cast < VLIWMachineScheduler * > ( dag ) ; SchedModel = DAG -> getSchedModel ( ) ; Top . init ( DAG , SchedModel ) ; Bot . init ( DAG , SchedModel ) ; const InstrItineraryData * Itin = DAG -> getSchedModel ( ) -> getInstrItineraries ( ) ; const TargetSubtargetInfo & STI = DAG -> MF . getSubtarget ( ) ; const TargetInstrInfo * TII = STI . getInstrInfo ( ) ; delete Top . HazardRec ; delete Bot . HazardRec ; Top . HazardRec = TII -> CreateTargetMIHazardRecognizer ( Itin , DAG ) ; Bot . HazardRec = TII -> CreateTargetMIHazardRecognizer ( Itin , DAG ) ; delete Top . ResourceModel ; delete Bot . ResourceModel ;

Top . ResourceModel = new VLIWResourceModel ( STI , DAG -> getSchedModel ( ) ) ;


<s> const TargetSubtargetInfo & STI = DAG -> MF . getSubtarget ( ) ; const TargetInstrInfo * TII = STI . getInstrInfo ( ) ; delete Top . HazardRec ; delete Bot . HazardRec ; Top . HazardRec = TII -> CreateTargetMIHazardRecognizer ( Itin , DAG ) ; Bot . HazardRec = TII -> CreateTargetMIHazardRecognizer ( Itin , DAG ) ; delete Top . ResourceModel ; delete Bot . ResourceModel ; Top . ResourceModel = new VLIWResourceModel ( STI , DAG -> getSchedModel ( ) ) ;

Bot . ResourceModel = new VLIWResourceModel ( STI , DAG -> getSchedModel ( ) ) ;


<s> break ; } for ( unsigned i = <NUM_LIT> , e = Packet . size ( ) ; i != e ; ++ i ) { if ( Packet [ i ] -> Succs . size ( ) == <NUM_LIT> ) continue ;

for ( SUnit :: const_succ_iterator I = Packet [ i ] -> Succs . begin ( ) , E = Packet [ i ] -> Succs . end ( ) ; I != E ; ++ I ) {


<s> if ( SUnit * SU = Top . pickOnlyChoice ( ) ) { IsTopNode = true ; return SU ; } SchedCandidate BotCand ; CandResult BotResult = pickNodeFromQueue ( Bot . Available , DAG -> getBotRPTracker ( ) , BotCand ) ; assert ( BotResult != NoCand && <STR_LIT> ) ; if ( BotResult == SingleExcess || BotResult == SingleCritical ) { IsTopNode = false ; return BotCand . SU ; } SchedCandidate TopCand ;

CandResult TopResult = pickNodeFromQueue ( Top . Available , DAG -> getTopRPTracker ( ) , TopCand ) ;


<s> void VLIWMachineScheduler :: postprocessDAG ( ) { SUnit * LastSequentialCall = nullptr ; for ( unsigned su = <NUM_LIT> , e = SUnits . size ( ) ; su != e ; ++ su ) { if ( SUnits [ su ] . getInstr ( ) -> isCall ( ) ) LastSequentialCall = & ( SUnits [ su ] ) ; else if ( SUnits [ su ] . getInstr ( ) -> isCompare ( ) && LastSequentialCall ) SUnits [ su ] . addPred ( SDep ( LastSequentialCall , SDep :: Barrier ) ) ; }


<s> SUnit * LastSequentialCall = nullptr ; for ( unsigned su = <NUM_LIT> , e = SUnits . size ( ) ; su != e ; ++ su ) { if ( SUnits [ su ] . getInstr ( ) -> isCall ( ) ) LastSequentialCall = & ( SUnits [ su ] ) ; else if ( SUnits [ su ] . getInstr ( ) -> isCompare ( ) && LastSequentialCall ) SUnits [ su ] . addPred ( SDep ( LastSequentialCall , SDep :: Barrier ) ) ;


<s> for ( SUnit :: succ_iterator I = SU -> Preds . begin ( ) , E = SU -> Preds . end ( ) ; I != E ; ++ I ) {

unsigned PredReadyCycle = I -> getSUnit ( ) -> TopReadyCycle ;


<s> return false ; } if ( ! isResourceAvailable ( SU ) ) { ResourcesModel -> clearResources ( ) ; Packet . clear ( ) ; TotalPackets ++ ; startNewCycle = true ; } switch ( SU -> getInstr ( ) -> getOpcode ( ) ) { default : ResourcesModel -> reserveResources ( SU -> getInstr ( ) ) ; break ; case TargetOpcode :: EXTRACT_SUBREG : case TargetOpcode :: INSERT_SUBREG : case TargetOpcode :: SUBREG_TO_REG : case TargetOpcode :: REG_SEQUENCE : case TargetOpcode :: IMPLICIT_DEF : case TargetOpcode :: KILL : case TargetOpcode :: CFI_INSTRUCTION : case TargetOpcode :: EH_LABEL : case TargetOpcode :: COPY :


<s> TotalPackets ++ ; startNewCycle = true ; } switch ( SU -> getInstr ( ) -> getOpcode ( ) ) { default : ResourcesModel -> reserveResources ( SU -> getInstr ( ) ) ; break ; case TargetOpcode :: EXTRACT_SUBREG : case TargetOpcode :: INSERT_SUBREG : case TargetOpcode :: SUBREG_TO_REG : case TargetOpcode :: REG_SEQUENCE : case TargetOpcode :: IMPLICIT_DEF : case TargetOpcode :: KILL : case TargetOpcode :: CFI_INSTRUCTION : case TargetOpcode :: EH_LABEL : case TargetOpcode :: COPY : case TargetOpcode :: INLINEASM : break ; } Packet . push_back ( SU ) ; DEBUG ( dbgs ( ) << <STR_LIT> << TotalPackets << <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = Packet . size ( ) ; i != e ; ++ i ) { DEBUG ( dbgs ( ) << <STR_LIT> << i << <STR_LIT> ) ; DEBUG ( dbgs ( ) << Packet [ i ] -> NodeNum << <STR_LIT> ) ; DEBUG ( Packet [ i ] -> getInstr ( ) -> dump ( ) ) ; }

if ( Packet . size ( ) >= SchedModel -> getIssueWidth ( ) ) {


<s> postprocessDAG ( ) ; SmallVector < SUnit * , <NUM_LIT> > TopRoots , BotRoots ; findRootsAndBiasEdges ( TopRoots , BotRoots ) ; SchedImpl -> initialize ( this ) ; DEBUG ( unsigned maxH = <NUM_LIT> ; for ( unsigned su = <NUM_LIT> , e = SUnits . size ( ) ; su != e ; ++ su ) if ( SUnits [ su ] . getHeight ( ) > maxH ) maxH = SUnits [ su ] . getHeight ( ) ; dbgs ( ) << <STR_LIT> << maxH << <STR_LIT> ; ) ; DEBUG ( unsigned maxD = <NUM_LIT> ; for ( unsigned su = <NUM_LIT> , e = SUnits . size ( ) ; su != e ; ++ su ) if ( SUnits [ su ] . getDepth ( ) > maxD ) maxD = SUnits [ su ] . getDepth ( ) ; dbgs ( ) << <STR_LIT> << maxD << <STR_LIT> ; ) ; DEBUG ( for ( unsigned su = <NUM_LIT> , e = SUnits . size ( ) ; su != e ; ++ su ) SUnits [ su ] . dumpAll ( this ) ) ; initQueues ( TopRoots , BotRoots ) ; bool IsTopNode = false ; while ( true ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; SUnit * SU = SchedImpl -> pickNode ( IsTopNode ) ;


<s> DEBUG ( dbgs ( ) << <STR_LIT> << BB -> getNumber ( ) << <STR_LIT> << BB -> getName ( ) << <STR_LIT> << BB -> getParent ( ) -> getFunction ( ) -> getName ( ) << <STR_LIT> << MLI -> getLoopDepth ( BB ) << <STR_LIT> ) ; buildDAGWithRegPressure ( ) ; postprocessDAG ( ) ; SmallVector < SUnit * , <NUM_LIT> > TopRoots , BotRoots ; findRootsAndBiasEdges ( TopRoots , BotRoots ) ; SchedImpl -> initialize ( this ) ; DEBUG ( unsigned maxH = <NUM_LIT> ; for ( unsigned su = <NUM_LIT> , e = SUnits . size ( ) ; su != e ; ++ su ) if ( SUnits [ su ] . getHeight ( ) > maxH ) maxH = SUnits [ su ] . getHeight ( ) ; dbgs ( ) << <STR_LIT> << maxH << <STR_LIT> ; ) ; DEBUG ( unsigned maxD = <NUM_LIT> ; for ( unsigned su = <NUM_LIT> , e = SUnits . size ( ) ; su != e ; ++ su ) if ( SUnits [ su ] . getDepth ( ) > maxD ) maxD = SUnits [ su ] . getDepth ( ) ; dbgs ( ) << <STR_LIT> << maxD << <STR_LIT> ; ) ; DEBUG ( for ( unsigned su = <NUM_LIT> , e = SUnits . size ( ) ; su != e ; ++ su ) SUnits [ su ] . dumpAll ( this ) ) ; initQueues ( TopRoots , BotRoots ) ; bool IsTopNode = false ; while ( true ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; SUnit * SU = SchedImpl -> pickNode ( IsTopNode ) ; if ( ! SU ) break ; if ( ! checkSchedLimit ( ) ) break ;


<s> } else { ResCount += ( SU -> getDepth ( ) * ScaleTwo ) ; if ( Bot . ResourceModel -> isResourceAvailable ( SU ) ) ResCount <<= FactorOne ; } unsigned NumNodesBlocking = <NUM_LIT> ; if ( Q . getID ( ) == TopQID ) { for ( SUnit :: const_succ_iterator I = SU -> Succs . begin ( ) , E = SU -> Succs . end ( ) ; I != E ; ++ I ) if ( getSingleUnscheduledPred ( I -> getSUnit ( ) ) == SU ) ++ NumNodesBlocking ;


<s> void ConvergingVLIWScheduler :: traceCandidate ( const char * Label , const ReadyQueue & Q , SUnit * SU , PressureChange P ) { dbgs ( ) << Label << <STR_LIT> << Q . getName ( ) << <STR_LIT> ; if ( P . isValid ( ) ) dbgs ( ) << DAG -> TRI -> getRegPressureSetName ( P . getPSet ( ) ) << <STR_LIT> << P . getUnitInc ( ) << <STR_LIT> ; else dbgs ( ) << <STR_LIT> ;


<s> dbgs ( ) << Label << <STR_LIT> << Q . getName ( ) << <STR_LIT> ; if ( P . isValid ( ) ) dbgs ( ) << DAG -> TRI -> getRegPressureSetName ( P . getPSet ( ) ) << <STR_LIT> << P . getUnitInc ( ) << <STR_LIT> ;


<s> void <ISA_LIT>CallMutation :: apply ( ScheduleDAGInstrs * DAG ) { SUnit * LastSequentialCall = nullptr ; unsigned VRegHoldingRet = <NUM_LIT> ; unsigned RetRegister ; SUnit * LastUseOfRet = nullptr ; auto & TRI = * DAG -> MF . getSubtarget ( ) . getRegisterInfo ( ) ; auto & HII = * DAG -> MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) ; for ( unsigned su = <NUM_LIT> , e = DAG -> SUnits . size ( ) ; su != e ; ++ su ) { if ( DAG -> SUnits [ su ] . getInstr ( ) -> isCall ( ) ) LastSequentialCall = & DAG -> SUnits [ su ] ; else if ( DAG -> SUnits [ su ] . getInstr ( ) -> isCompare ( ) && LastSequentialCall ) DAG -> SUnits [ su ] . addPred ( SDep ( LastSequentialCall , SDep :: Barrier ) ) ; else if ( SchedPredsCloser && LastSequentialCall && su > <NUM_LIT> && su < e - <NUM_LIT> && shouldTFRICallBind ( HII , DAG -> SUnits [ su ] , DAG -> SUnits [ su + <NUM_LIT> ] ) ) DAG -> SUnits [ su ] . addPred ( SDep ( & DAG -> SUnits [ su - <NUM_LIT> ] , SDep :: Barrier ) ) ; else if ( SchedRetvalOptimization ) { const MachineInstr * MI = DAG -> SUnits [ su ] . getInstr ( ) ; if ( MI -> isCopy ( ) && ( MI -> readsRegister ( <STR_LIT> , & TRI ) || MI -> readsRegister ( <STR_LIT> , & TRI ) ) ) { VRegHoldingRet = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ;

RetRegister = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> case TargetOpcode :: INSERT_SUBREG : case TargetOpcode :: SUBREG_TO_REG : case TargetOpcode :: REG_SEQUENCE : case TargetOpcode :: IMPLICIT_DEF : case TargetOpcode :: COPY : case TargetOpcode :: INLINEASM : break ; } MachineFunction & MF = * SU -> getInstr ( ) -> getParent ( ) -> getParent ( ) ; auto & QII = * MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) ; for ( unsigned i = <NUM_LIT> , e = Packet . size ( ) ; i != e ; ++ i ) { if ( Packet [ i ] -> Succs . size ( ) == <NUM_LIT> ) continue ; if ( QII . mayBeCurLoad ( * Packet [ i ] -> getInstr ( ) ) ) continue ;


<s> static inline bool isSingleUnscheduledPred ( SUnit * SU , SUnit * SU2 ) { if ( SU -> NumPredsLeft == <NUM_LIT> ) return false ; for ( auto & Pred : SU -> Preds ) { if ( ! Pred . getSUnit ( ) -> isScheduled && ( Pred . getSUnit ( ) != SU2 ) ) return false ;


<s> if ( SU -> NumPredsLeft == <NUM_LIT> ) return false ; for ( auto & Pred : SU -> Preds ) { if ( ! Pred . getSUnit ( ) -> isScheduled && ( Pred . getSUnit ( ) != SU2 ) ) return false ; } return true ;


<s> static inline bool isSingleUnscheduledSucc ( SUnit * SU , SUnit * SU2 ) { if ( SU -> NumSuccsLeft == <NUM_LIT> ) return false ;


if ( ! Succ . getSUnit ( ) -> isScheduled && ( Succ . getSUnit ( ) != SU2 ) ) return false ;


<s> if ( DAG -> top ( ) == DAG -> bottom ( ) ) { assert ( Top . Available . empty ( ) && Top . Pending . empty ( ) && Bot . Available . empty ( ) && Bot . Pending . empty ( ) && <STR_LIT> ) ; return nullptr ; } SUnit * SU ; if ( llvm :: ForceTopDown ) { SU = Top . pickOnlyChoice ( ) ; if ( ! SU ) { SchedCandidate TopCand ; CandResult TopResult = pickNodeFromQueue ( Top . Available , DAG -> getTopRPTracker ( ) , TopCand ) ; assert ( TopResult != NoCand && <STR_LIT> ) ; ( void ) TopResult ; SU = TopCand . SU ; } IsTopNode = true ; } else if ( llvm :: ForceBottomUp ) { SU = Bot . pickOnlyChoice ( ) ; if ( ! SU ) { SchedCandidate BotCand ; CandResult BotResult = pickNodeFromQueue ( Bot . Available , DAG -> getBotRPTracker ( ) , BotCand ) ; assert ( BotResult != NoCand && <STR_LIT> ) ; ( void ) BotResult ; SU = BotCand . SU ; } IsTopNode = false ; } else { SU = pickNodeBidrectional ( IsTopNode ) ; } if ( SU -> isTopReady ( ) ) Top . removeReady ( SU ) ; if ( SU -> isBottomReady ( ) ) Bot . removeReady ( SU ) ;

DEBUG ( dbgs ( ) << <STR_LIT> << ( IsTopNode ? <STR_LIT> : <STR_LIT> ) << <STR_LIT> << ( IsTopNode ? Top . CurrCycle : Bot . CurrCycle ) << '\n' ;


<s> return SU ; } if ( SUnit * SU = Top . pickOnlyChoice ( ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode = true ; return SU ; } SchedCandidate BotCand ; CandResult BotResult = pickNodeFromQueue ( Bot . Available , DAG -> getBotRPTracker ( ) , BotCand ) ; assert ( BotResult != NoCand && <STR_LIT> ) ; if ( BotResult == SingleExcess || BotResult == SingleCritical ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode = false ; return BotCand . SU ; } SchedCandidate TopCand ; CandResult TopResult = pickNodeFromQueue ( Top . Available , DAG -> getTopRPTracker ( ) , TopCand ) ; assert ( TopResult != NoCand && <STR_LIT> ) ; if ( TopResult == SingleExcess || TopResult == SingleCritical ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode = true ; return TopCand . SU ; } if ( BotResult == SingleMax ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode = false ; return BotCand . SU ; } if ( TopResult == SingleMax ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode = true ; return TopCand . SU ; } if ( TopCand . SCost > BotCand . SCost ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode = true ; return TopCand . SU ; } DEBUG ( dbgs ( ) << <STR_LIT> ) ;


<s> IsTopNode = true ; return SU ; } SchedCandidate BotCand ; CandResult BotResult = pickNodeFromQueue ( Bot . Available , DAG -> getBotRPTracker ( ) , BotCand ) ; assert ( BotResult != NoCand && <STR_LIT> ) ; if ( BotResult == SingleExcess || BotResult == SingleCritical ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode = false ; return BotCand . SU ; } SchedCandidate TopCand ; CandResult TopResult = pickNodeFromQueue ( Top . Available , DAG -> getTopRPTracker ( ) , TopCand ) ; assert ( TopResult != NoCand && <STR_LIT> ) ; if ( TopResult == SingleExcess || TopResult == SingleCritical ) {


<s> Candidate . RPDelta = RPDelta ; Candidate . SCost = CurrentCost ; FoundCandidate = BestCost ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ; } else if ( InstrLatency > CandLatency && ! TopUseShorterTie ) { Candidate . SU = * I ; Candidate . RPDelta = RPDelta ; Candidate . SCost = CurrentCost ; FoundCandidate = BestCost ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ; } } else if ( Q . getID ( ) == BotQID && CurrentCost == Candidate . SCost ) { if ( InstrLatency < CandLatency && BotUseShorterTie ) { Candidate . SU = * I ; Candidate . RPDelta = RPDelta ; Candidate . SCost = CurrentCost ; FoundCandidate = BestCost ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ; } else if ( InstrLatency > CandLatency && ! BotUseShorterTie ) { Candidate . SU = * I ; Candidate . RPDelta = RPDelta ; Candidate . SCost = CurrentCost ; FoundCandidate = BestCost ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ; } } } if ( CurrentCost == Candidate . SCost ) { if ( ( Q . getID ( ) == TopQID && ( * I ) -> Succs . size ( ) > Candidate . SU -> Succs . size ( ) ) || ( Q . getID ( ) == BotQID && ( * I ) -> Preds . size ( ) < Candidate . SU -> Preds . size ( ) ) ) { DEBUG ( traceCandidate ( <STR_LIT> , Q , * I , CurrentCost ) ) ;


<s> dbgs ( ) << <STR_LIT> << Q . getName ( ) << <STR_LIT> ; for ( ReadyQueue :: iterator I = Q . begin ( ) , E = Q . end ( ) ; I != E ; ++ I ) { RegPressureDelta RPDelta ; TempTracker . getMaxPressureDelta ( ( * I ) -> getInstr ( ) , RPDelta , DAG -> getRegionCriticalPSets ( ) , DAG -> getRegPressure ( ) . MaxSetPressure ) ; std :: stringstream dbgstr ; dbgstr << <STR_LIT> << std :: setw ( <NUM_LIT> ) << ( * I ) -> NodeNum << <STR_LIT> ; dbgs ( ) << dbgstr . str ( ) ; SchedulingCost ( Q , * I , Candidate , RPDelta , true ) ; dbgs ( ) << <STR_LIT> ; ( * I ) -> getInstr ( ) -> dump ( ) ; } dbgs ( ) << <STR_LIT> ;


<s> dbgstr << <STR_LIT> << std :: setw ( <NUM_LIT> ) << ( * I ) -> NodeNum << <STR_LIT> ; dbgs ( ) << dbgstr . str ( ) ; SchedulingCost ( Q , * I , Candidate , RPDelta , true ) ; dbgs ( ) << <STR_LIT> ; ( * I ) -> getInstr ( ) -> dump ( ) ; } dbgs ( ) << <STR_LIT> ;


<s> I != E ; ++ I ) { unsigned SuccReadyCycle = I -> getSUnit ( ) -> BotReadyCycle ; unsigned MinLatency = I -> getLatency ( ) ; Bot . MaxMinLatency = std :: max ( MinLatency , Bot . MaxMinLatency ) ; if ( SU -> BotReadyCycle < SuccReadyCycle + MinLatency ) SU -> BotReadyCycle = SuccReadyCycle + MinLatency ;


<s> if ( SU -> TopReadyCycle < PredReadyCycle + MinLatency ) SU -> TopReadyCycle = PredReadyCycle + MinLatency ; } Top . releaseNode ( SU , SU -> TopReadyCycle ) ;


<s> for ( const SDep & PI : SU -> Preds ) { unsigned PredReadyCycle = PI . getSUnit ( ) -> TopReadyCycle ; unsigned MinLatency = PI . getLatency ( ) ; Top . MaxMinLatency = std :: max ( MinLatency , Top . MaxMinLatency ) ; if ( SU -> TopReadyCycle < PredReadyCycle + MinLatency ) SU -> TopReadyCycle = PredReadyCycle + MinLatency ; }

Top . releaseNode ( SU , SU -> TopReadyCycle ) ;


<s> default : ResourcesModel -> reserveResources ( * SU -> getInstr ( ) ) ; break ; case TargetOpcode :: EXTRACT_SUBREG : case TargetOpcode :: INSERT_SUBREG : case TargetOpcode :: SUBREG_TO_REG : case TargetOpcode :: REG_SEQUENCE : case TargetOpcode :: IMPLICIT_DEF : case TargetOpcode :: KILL : case TargetOpcode :: CFI_INSTRUCTION : case TargetOpcode :: EH_LABEL : case TargetOpcode :: COPY : case TargetOpcode :: INLINEASM : break ; } Packet . push_back ( SU ) ; DEBUG ( dbgs ( ) << <STR_LIT> << TotalPackets << <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = Packet . size ( ) ; i != e ; ++ i ) { DEBUG ( dbgs ( ) << <STR_LIT> << i << <STR_LIT> ) ; DEBUG ( dbgs ( ) << Packet [ i ] -> NodeNum << <STR_LIT> ) ; DEBUG ( Packet [ i ] -> getInstr ( ) -> dump ( ) ) ; } if ( Packet . size ( ) >= SchedModel -> getIssueWidth ( ) ) { ResourcesModel -> clearResources ( ) ; savePacket ( ) ; Packet . clear ( ) ; TotalPackets ++ ;


<s> void VLIWResourceModel :: savePacket ( ) { OldPacket = Packet ;


<s> void VLIWResourceModel :: savePacket ( ) {


<s> SU -> BotReadyCycle = Bot . CurrCycle ; Bot . bumpNode ( SU ) ; }


<s> initQueues ( TopRoots , BotRoots ) ; bool IsTopNode = false ; while ( true ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; SUnit * SU = SchedImpl -> pickNode ( IsTopNode ) ; if ( ! SU ) break ; if ( ! checkSchedLimit ( ) ) break ; scheduleMI ( SU , IsTopNode ) ; updateQueues ( SU , IsTopNode ) ; SchedImpl -> schedNode ( SU , IsTopNode ) ; } assert ( CurrentTop == CurrentBottom && <STR_LIT> ) ; placeDebugValues ( ) ; DEBUG ( { unsigned BBNum = begin ( ) -> getParent ( ) -> getNumber ( ) ; dbgs ( ) << <STR_LIT> << BBNum << <STR_LIT> ;


<s> if ( Inst1 . getInstr ( ) -> getOpcode ( ) != <STR_LIT> ) return false ;

unsigned Type = HII . getType ( * Inst2 . getInstr ( ) ) ;


<s> I != E ; ++ I ) { SUnit & Pred = * I -> getSUnit ( ) ; if ( ! Pred . isScheduled ) { if ( OnlyAvailablePred && OnlyAvailablePred != & Pred ) return <NUM_LIT> ; OnlyAvailablePred = & Pred ; } } return OnlyAvailablePred ;


<s> for ( SUnit :: const_pred_iterator I = SU -> Preds . begin ( ) , E = SU -> Preds . end ( ) ; I != E ; ++ I ) { SUnit & Pred = * I -> getSUnit ( ) ; if ( ! Pred . isScheduled ) { if ( OnlyAvailablePred && OnlyAvailablePred != & Pred ) return <NUM_LIT> ; OnlyAvailablePred = & Pred ; } } return OnlyAvailablePred ;


<s> for ( SUnit :: const_succ_iterator I = SU -> Succs . begin ( ) , E = SU -> Succs . end ( ) ; I != E ; ++ I ) { SUnit & Succ = * I -> getSUnit ( ) ;


<s> void ConvergingVLIWScheduler :: initialize ( ScheduleDAGMI * dag ) { DAG = static_cast < VLIWMachineScheduler * > ( dag ) ; SchedModel = DAG -> getSchedModel ( ) ; TRI = DAG -> TRI ; Top . init ( DAG , SchedModel ) ; Bot . init ( DAG , SchedModel ) ;

const InstrItineraryData * Itin = DAG -> getSchedModel ( ) -> getInstrItineraries ( ) ;


<s> void ConvergingVLIWScheduler :: initialize ( ScheduleDAGMI * dag ) { DAG = static_cast < VLIWMachineScheduler * > ( dag ) ; SchedModel = DAG -> getSchedModel ( ) ; TRI = DAG -> TRI ; Top . init ( DAG , SchedModel ) ; Bot . init ( DAG , SchedModel ) ; const InstrItineraryData * Itin = DAG -> getSchedModel ( ) -> getInstrItineraries ( ) ; const TargetMachine & TM = DAG -> MF . getTarget ( ) ; delete Top . HazardRec ; delete Bot . HazardRec ; Top . HazardRec = TM . getInstrInfo ( ) -> CreateTargetMIHazardRecognizer ( Itin , DAG ) ; Bot . HazardRec = TM . getInstrInfo ( ) -> CreateTargetMIHazardRecognizer ( Itin , DAG ) ; Top . ResourceModel = new VLIWResourceModel ( TM , DAG -> getSchedModel ( ) ) ;

Bot . ResourceModel = new VLIWResourceModel ( TM , DAG -> getSchedModel ( ) ) ;


<s> SUnit * ConvergingVLIWScheduler :: pickNode ( bool & IsTopNode ) { if ( DAG -> top ( ) == DAG -> bottom ( ) ) { assert ( Top . Available . empty ( ) && Top . Pending . empty ( ) && Bot . Available . empty ( ) && Bot . Pending . empty ( ) && <STR_LIT> ) ; return NULL ; } SUnit * SU ; if ( llvm :: ForceTopDown ) { SU = Top . pickOnlyChoice ( ) ; if ( ! SU ) { SchedCandidate TopCand ; CandResult TopResult = pickNodeFromQueue ( Top . Available , DAG -> getTopRPTracker ( ) , TopCand ) ; assert ( TopResult != NoCand && <STR_LIT> ) ; ( void ) TopResult ; SU = TopCand . SU ; } IsTopNode = true ; } else if ( llvm :: ForceBottomUp ) { SU = Bot . pickOnlyChoice ( ) ; if ( ! SU ) { SchedCandidate BotCand ; CandResult BotResult = pickNodeFromQueue ( Bot . Available , DAG -> getBotRPTracker ( ) , BotCand ) ; assert ( BotResult != NoCand && <STR_LIT> ) ; ( void ) BotResult ; SU = BotCand . SU ; } IsTopNode = false ; } else { SU = pickNodeBidrectional ( IsTopNode ) ; }

if ( SU -> isTopReady ( ) ) Top . removeReady ( SU ) ;


<s> else if ( SUnits [ su ] . getInstr ( ) -> isCompare ( ) && LastSequentialCall ) SUnits [ su ] . addPred ( SDep ( LastSequentialCall , SDep :: Barrier ) ) ;


<s> else if ( SUnits [ su ] . getInstr ( ) -> isCompare ( ) && LastSequentialCall ) SUnits [ su ] . addPred ( SDep ( LastSequentialCall , SDep :: Barrier ) ) ; }


<s> unsigned SuccReadyCycle = I -> getSUnit ( ) -> BotReadyCycle ; unsigned MinLatency = I -> getMinLatency ( ) ; Bot . MaxMinLatency = std :: max ( MinLatency , Bot . MaxMinLatency ) ; if ( SU -> BotReadyCycle < SuccReadyCycle + MinLatency ) SU -> BotReadyCycle = SuccReadyCycle + MinLatency ; }

Bot . releaseNode ( SU , SU -> BotReadyCycle ) ;


<s> for ( SUnit :: succ_iterator I = SU -> Succs . begin ( ) , E = SU -> Succs . end ( ) ; I != E ; ++ I ) { unsigned SuccReadyCycle = I -> getSUnit ( ) -> BotReadyCycle ; unsigned MinLatency = I -> getMinLatency ( ) ;

Bot . MaxMinLatency = std :: max ( MinLatency , Bot . MaxMinLatency ) ;


<s> void ConvergingVLIWScheduler :: releaseTopNode ( SUnit * SU ) { if ( SU -> isScheduled ) return ; for ( SUnit :: succ_iterator I = SU -> Preds . begin ( ) , E = SU -> Preds . end ( ) ; I != E ; ++ I ) { unsigned PredReadyCycle = I -> getSUnit ( ) -> TopReadyCycle ;

unsigned MinLatency = I -> getMinLatency ( ) ;


<s> dbgs ( ) << <STR_LIT> << maxH << <STR_LIT> ; ) ; DEBUG ( unsigned maxD = <NUM_LIT> ; for ( unsigned su = <NUM_LIT> , e = SUnits . size ( ) ; su != e ; ++ su ) if ( SUnits [ su ] . getDepth ( ) > maxD ) maxD = SUnits [ su ] . getDepth ( ) ; dbgs ( ) << <STR_LIT> << maxD << <STR_LIT> ; ) ; DEBUG ( for ( unsigned su = <NUM_LIT> , e = SUnits . size ( ) ; su != e ; ++ su ) SUnits [ su ] . dumpAll ( this ) ) ; initQueues ( TopRoots , BotRoots ) ;


<s> for ( SUnit :: const_pred_iterator I = SU -> Preds . begin ( ) , E = SU -> Preds . end ( ) ; I != E ; ++ I ) if ( getSingleUnscheduledSucc ( I -> getSUnit ( ) ) == SU ) ++ NumNodesBlocking ; } ResCount += ( NumNodesBlocking * ScaleTwo ) ; ResCount -= ( Delta . Excess . UnitIncrease * PriorityThree ) ; ResCount -= ( Delta . CriticalMax . UnitIncrease * PriorityThree ) ; DEBUG ( if ( verbose ) dbgs ( ) << <STR_LIT> << ResCount << <STR_LIT> ) ; return ResCount ;


<s> ResCount += ( SU -> getHeight ( ) * ScaleTwo ) ; if ( Top . ResourceModel -> isResourceAvailable ( SU ) ) ResCount <<= FactorOne ; } else { ResCount += ( SU -> getDepth ( ) * ScaleTwo ) ; if ( Bot . ResourceModel -> isResourceAvailable ( SU ) ) ResCount <<= FactorOne ; } unsigned NumNodesBlocking = <NUM_LIT> ; if ( Q . getID ( ) == TopQID ) { for ( SUnit :: const_succ_iterator I = SU -> Succs . begin ( ) , E = SU -> Succs . end ( ) ;

I != E ; ++ I ) if ( getSingleUnscheduledPred ( I -> getSUnit ( ) ) == SU ) ++ NumNodesBlocking ;


<s> if ( P . isValid ( ) ) dbgs ( ) << TRI -> getRegPressureSetName ( P . PSetID ) << <STR_LIT> << P . UnitIncrease << <STR_LIT> ; else dbgs ( ) << <STR_LIT> ; SU -> dump ( DAG ) ;


<s> void ConvergingVLIWScheduler :: traceCandidate ( const char * Label , const ReadyQueue & Q , SUnit * SU , PressureElement P ) { dbgs ( ) << Label << <STR_LIT> << Q . getName ( ) << <STR_LIT> ; if ( P . isValid ( ) ) dbgs ( ) << TRI -> getRegPressureSetName ( P . PSetID ) << <STR_LIT> << P . UnitIncrease << <STR_LIT> ;


<s> if ( SUd -> Succs . size ( ) == <NUM_LIT> ) return false ; if ( QII . mayBeCurLoad ( * SUd -> getInstr ( ) ) ) return false ; if ( QII . canExecuteInBundle ( * SUd -> getInstr ( ) , * SUu -> getInstr ( ) ) ) return false ; for ( const auto & S : SUd -> Succs ) { if ( S . isCtrl ( ) ) continue ; if ( S . getSUnit ( ) == SUu && S . getLatency ( ) > <NUM_LIT> ) return true ; } return false ;


<s> for ( const auto & S : SUd -> Succs ) { if ( S . isCtrl ( ) ) continue ; if ( S . getSUnit ( ) == SUu && S . getLatency ( ) > <NUM_LIT> ) return true ; } return false ;


<s> LLVM_DEBUG ( if ( verbose ) { std :: stringstream dbgstr ; dbgstr << <STR_LIT> << std :: setw ( <NUM_LIT> ) << ResCount << <STR_LIT> ;


<s> Top . init ( DAG , SchedModel ) ; Bot . init ( DAG , SchedModel ) ; const InstrItineraryData * Itin = DAG -> getSchedModel ( ) -> getInstrItineraries ( ) ; const TargetSubtargetInfo & STI = DAG -> MF . getSubtarget ( ) ; const TargetInstrInfo * TII = STI . getInstrInfo ( ) ; delete Top . HazardRec ; delete Bot . HazardRec ; Top . HazardRec = TII -> CreateTargetMIHazardRecognizer ( Itin , DAG ) ; Bot . HazardRec = TII -> CreateTargetMIHazardRecognizer ( Itin , DAG ) ; delete Top . ResourceModel ; delete Bot . ResourceModel ; Top . ResourceModel = new VLIWResourceModel ( STI , DAG -> getSchedModel ( ) ) ; Bot . ResourceModel = new VLIWResourceModel ( STI , DAG -> getSchedModel ( ) ) ; const std :: vector < unsigned > & MaxPressure = DAG -> getRegPressure ( ) . MaxSetPressure ; HighPressureSets . assign ( MaxPressure . size ( ) , <NUM_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = MaxPressure . size ( ) ; i < e ; ++ i ) { unsigned Limit = DAG -> getRegClassInfo ( ) -> getRegPressureSetLimit ( i ) ; HighPressureSets [ i ] = ( ( float ) MaxPressure [ i ] > ( ( float ) Limit * RPThreshold ) ) ; } assert ( ( ! ForceTopDown || ! ForceBottomUp ) && <STR_LIT> ) ;


<s> switch ( SU -> getInstr ( ) -> getOpcode ( ) ) { default : if ( ! ResourcesModel -> canReserveResources ( * SU -> getInstr ( ) ) ) return false ; break ; case TargetOpcode :: EXTRACT_SUBREG : case TargetOpcode :: INSERT_SUBREG : case TargetOpcode :: SUBREG_TO_REG : case TargetOpcode :: REG_SEQUENCE : case TargetOpcode :: IMPLICIT_DEF : case TargetOpcode :: COPY : case TargetOpcode :: INLINEASM :


<s> IsTopNode = false ; return BotCand . SU ; } SchedCandidate TopCand ; CandResult TopResult = pickNodeFromQueue ( Top , DAG -> getTopRPTracker ( ) , TopCand ) ; assert ( TopResult != NoCand && <STR_LIT> ) ; if ( TopResult == SingleExcess || TopResult == SingleCritical ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode = true ; return TopCand . SU ; } if ( BotResult == SingleMax ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode = false ; return BotCand . SU ;


<s> if ( ( Q . getID ( ) == TopQID && ( * I ) -> NodeNum < Candidate . SU -> NodeNum ) || ( Q . getID ( ) == BotQID && ( * I ) -> NodeNum > Candidate . SU -> NodeNum ) ) { LLVM_DEBUG ( traceCandidate ( <STR_LIT> , Q , * I , CurrentCost ) ) ; Candidate . SU = * I ; Candidate . RPDelta = RPDelta ; Candidate . SCost = CurrentCost ; FoundCandidate = NodeOrder ; } continue ; } if ( CurrentCost > Candidate . SCost ) { LLVM_DEBUG ( traceCandidate ( <STR_LIT> , Q , * I , CurrentCost ) ) ; Candidate . SU = * I ; Candidate . RPDelta = RPDelta ; Candidate . SCost = CurrentCost ; FoundCandidate = BestCost ; continue ; } unsigned CurrWeak = getWeakLeft ( * I , ( Q . getID ( ) == TopQID ) ) ; unsigned CandWeak = getWeakLeft ( Candidate . SU , ( Q . getID ( ) == TopQID ) ) ; if ( CurrWeak != CandWeak ) { if ( CurrWeak < CandWeak ) { LLVM_DEBUG ( traceCandidate ( <STR_LIT> , Q , * I , CurrentCost ) ) ; Candidate . SU = * I ; Candidate . RPDelta = RPDelta ; Candidate . SCost = CurrentCost ; FoundCandidate = Weak ; } continue ; } if ( CurrentCost == Candidate . SCost && Zone . isLatencyBound ( * I ) ) { unsigned CurrSize , CandSize ; if ( Q . getID ( ) == TopQID ) { CurrSize = ( * I ) -> Succs . size ( ) ; CandSize = Candidate . SU -> Succs . size ( ) ; } else { CurrSize = ( * I ) -> Preds . size ( ) ; CandSize = Candidate . SU -> Preds . size ( ) ; } if ( CurrSize > CandSize ) { LLVM_DEBUG ( traceCandidate ( <STR_LIT> , Q , * I , CurrentCost ) ) ; Candidate . SU = * I ; Candidate . RPDelta = RPDelta ; Candidate . SCost = CurrentCost ; FoundCandidate = BestCost ; } if ( CurrSize != CandSize ) continue ; } if ( UseNewerCandidate && CurrentCost == Candidate . SCost ) { if ( ( Q . getID ( ) == TopQID && ( * I ) -> NodeNum < Candidate . SU -> NodeNum ) || ( Q . getID ( ) == BotQID && ( * I ) -> NodeNum > Candidate . SU -> NodeNum ) ) { LLVM_DEBUG ( traceCandidate ( <STR_LIT> , Q , * I , CurrentCost ) ) ; Candidate . SU = * I ; Candidate . RPDelta = RPDelta ; Candidate . SCost = CurrentCost ; FoundCandidate = NodeOrder ; continue ; } } if ( FoundCandidate == NoCand ) continue ;


<s> if ( HighPressureSets [ P . getPSet ( ) ] ) return ( isBotUp ? P . getUnitInc ( ) : - P . getUnitInc ( ) ) ;


<s> for ( SUnit :: succ_iterator I = SU -> Succs . begin ( ) , E = SU -> Succs . end ( ) ; I != E ; ++ I ) { unsigned SuccReadyCycle = I -> getSUnit ( ) -> BotReadyCycle ;

unsigned MinLatency = I -> getLatency ( ) ;


<s> void ConvergingVLIWScheduler :: releaseBottomNode ( SUnit * SU ) { assert ( SU -> getInstr ( ) && <STR_LIT> ) ; for ( SUnit :: succ_iterator I = SU -> Succs . begin ( ) , E = SU -> Succs . end ( ) ; I != E ; ++ I ) { unsigned SuccReadyCycle = I -> getSUnit ( ) -> BotReadyCycle ; unsigned MinLatency = I -> getLatency ( ) ; Bot . MaxMinLatency = std :: max ( MinLatency , Bot . MaxMinLatency ) ;

if ( SU -> BotReadyCycle < SuccReadyCycle + MinLatency ) SU -> BotReadyCycle = SuccReadyCycle + MinLatency ;


<s> for ( const SDep & PI : SU -> Preds ) { unsigned PredReadyCycle = PI . getSUnit ( ) -> TopReadyCycle ; unsigned MinLatency = PI . getLatency ( ) ; Top . MaxMinLatency = std :: max ( MinLatency , Top . MaxMinLatency ) ;

if ( SU -> TopReadyCycle < PredReadyCycle + MinLatency ) SU -> TopReadyCycle = PredReadyCycle + MinLatency ;


<s> unsigned PredReadyCycle = PI . getSUnit ( ) -> TopReadyCycle ; unsigned MinLatency = PI . getLatency ( ) ; Top . MaxMinLatency = std :: max ( MinLatency , Top . MaxMinLatency ) ; if ( SU -> TopReadyCycle < PredReadyCycle + MinLatency ) SU -> TopReadyCycle = PredReadyCycle + MinLatency ;


<s> case TargetOpcode :: CFI_INSTRUCTION : case TargetOpcode :: EH_LABEL : case TargetOpcode :: COPY : case TargetOpcode :: INLINEASM : case TargetOpcode :: INLINEASM_BR : break ; } Packet . push_back ( SU ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << TotalPackets << <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = Packet . size ( ) ; i != e ; ++ i ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << i << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << Packet [ i ] -> NodeNum << <STR_LIT> ) ; LLVM_DEBUG ( Packet [ i ] -> getInstr ( ) -> dump ( ) ) ; } return startNewCycle ;


<s> SU -> TopReadyCycle = Top . CurrCycle ; } else { Bot . bumpNode ( SU ) ; SU -> BotReadyCycle = Bot . CurrCycle ; }


<s> void ConvergingVLIWScheduler :: schedNode ( SUnit * SU , bool IsTopNode ) { if ( IsTopNode ) {


<s> if ( P . isValid ( ) ) dbgs ( ) << DAG -> TRI -> getRegPressureSetName ( P . getPSet ( ) ) << <STR_LIT> << P . getUnitInc ( ) << <STR_LIT> ; else dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ;


<s> dbgs ( ) << Label << <STR_LIT> << Q . getName ( ) << <STR_LIT> ; if ( P . isValid ( ) ) dbgs ( ) << DAG -> TRI -> getRegPressureSetName ( P . getPSet ( ) ) << <STR_LIT> << P . getUnitInc ( ) << <STR_LIT> ; else dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ;


<s> delete Top . ResourceModel ; delete Bot . ResourceModel ; Top . ResourceModel = new VLIWResourceModel ( STI , DAG -> getSchedModel ( ) ) ; Bot . ResourceModel = new VLIWResourceModel ( STI , DAG -> getSchedModel ( ) ) ;

assert ( ( ! ForceTopDown || ! ForceBottomUp ) && <STR_LIT> ) ;


<s> if ( DAG -> top ( ) == DAG -> bottom ( ) ) { assert ( Top . Available . empty ( ) && Top . Pending . empty ( ) && Bot . Available . empty ( ) && Bot . Pending . empty ( ) && <STR_LIT> ) ; return nullptr ; } SUnit * SU ; if ( ForceTopDown ) { SU = Top . pickOnlyChoice ( ) ; if ( ! SU ) { SchedCandidate TopCand ; CandResult TopResult = pickNodeFromQueue ( Top . Available , DAG -> getTopRPTracker ( ) , TopCand ) ; assert ( TopResult != NoCand && <STR_LIT> ) ; ( void ) TopResult ; SU = TopCand . SU ; } IsTopNode = true ; } else if ( ForceBottomUp ) { SU = Bot . pickOnlyChoice ( ) ; if ( ! SU ) { SchedCandidate BotCand ; CandResult BotResult = pickNodeFromQueue ( Bot . Available , DAG -> getBotRPTracker ( ) , BotCand ) ; assert ( BotResult != NoCand && <STR_LIT> ) ;


<s> void init ( VLIWMachineScheduler * dag , const TargetSchedModel * smodel ) { DAG = dag ; SchedModel = smodel ; IssueCount = <NUM_LIT> ;


<s> bool isInPacket ( SUnit * SU ) const {


<s> Packet . clear ( ) ; OldPacket . resize ( SchedModel -> getIssueWidth ( ) ) ; OldPacket . clear ( ) ; ResourcesModel -> clearResources ( ) ;


<s> assert ( ResourcesModel && <STR_LIT> ) ; Packet . resize ( SchedModel -> getIssueWidth ( ) ) ; Packet . clear ( ) ;

OldPacket . resize ( SchedModel -> getIssueWidth ( ) ) ;


<s> } ResCount += ( NumNodesBlocking * ScaleTwo ) ; DEBUG ( if ( verbose ) { std :: stringstream dbgstr ; dbgstr << <STR_LIT> << std :: setw ( <NUM_LIT> ) << NumNodesBlocking << <STR_LIT> ; dbgs ( ) << dbgstr . str ( ) ; } ) ; if ( ! IgnoreBBRegPressure ) { ResCount -= ( Delta . Excess . getUnitInc ( ) * PriorityOne ) ; ResCount -= ( Delta . CriticalMax . getUnitInc ( ) * PriorityOne ) ; ResCount -= ( Delta . CurrentMax . getUnitInc ( ) * PriorityTwo ) ; DEBUG ( if ( verbose ) { dbgs ( ) << <STR_LIT> << Delta . Excess . getUnitInc ( ) << <STR_LIT> << Delta . CriticalMax . getUnitInc ( ) << <STR_LIT> << Delta . CurrentMax . getUnitInc ( ) << <STR_LIT> ; } ) ; } auto & QST = DAG -> MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; auto & QII = * QST . getInstrInfo ( ) ; if ( SU -> isInstr ( ) && QII . mayBeCurLoad ( * SU -> getInstr ( ) ) ) { if ( Q . getID ( ) == TopQID && Top . ResourceModel -> isResourceAvailable ( SU ) ) { ResCount += PriorityTwo ; DEBUG ( if ( verbose ) dbgs ( ) << <STR_LIT> ) ; } else if ( Q . getID ( ) == BotQID && Bot . ResourceModel -> isResourceAvailable ( SU ) ) { ResCount += PriorityTwo ; DEBUG ( if ( verbose ) dbgs ( ) << <STR_LIT> ) ; } } if ( Q . getID ( ) == TopQID ) { for ( const SDep & PI : SU -> Preds ) { if ( ! PI . getSUnit ( ) -> getInstr ( ) -> isPseudo ( ) && PI . isAssignedRegDep ( ) && PI . getLatency ( ) == <NUM_LIT> && Top . ResourceModel -> isInPacket ( PI . getSUnit ( ) ) ) { ResCount += PriorityThree ; DEBUG ( if ( verbose ) dbgs ( ) << <STR_LIT> ) ;


<s> for ( const SDep & PI : SU -> Preds ) { if ( ! PI . getSUnit ( ) -> getInstr ( ) -> isPseudo ( ) && PI . isAssignedRegDep ( ) && PI . getLatency ( ) == <NUM_LIT> && Top . ResourceModel -> isInPacket ( PI . getSUnit ( ) ) ) { ResCount += PriorityThree ; DEBUG ( if ( verbose ) dbgs ( ) << <STR_LIT> ) ; } } } else { for ( const SDep & SI : SU -> Succs ) { if ( ! SI . getSUnit ( ) -> getInstr ( ) -> isPseudo ( ) && SI . isAssignedRegDep ( ) && SI . getLatency ( ) == <NUM_LIT> && Bot . ResourceModel -> isInPacket ( SI . getSUnit ( ) ) ) { ResCount += PriorityThree ; DEBUG ( if ( verbose ) dbgs ( ) << <STR_LIT> ) ; } } } if ( QII . isV60VectorInstruction ( Instr ) ) { if ( Q . getID ( ) == TopQID ) { for ( auto J : Top . ResourceModel -> OldPacket ) if ( QII . producesStall ( * J -> getInstr ( ) , Instr ) ) ResCount -= PriorityOne ; } else { for ( auto J : Bot . ResourceModel -> OldPacket ) if ( QII . producesStall ( Instr , * J -> getInstr ( ) ) ) ResCount -= PriorityOne ; } } if ( CheckEarlyAvail ) { if ( Q . getID ( ) == TopQID ) { for ( const auto & PI : SU -> Preds ) { if ( PI . getLatency ( ) > <NUM_LIT> && Top . ResourceModel -> isInPacket ( PI . getSUnit ( ) ) ) { ResCount -= PriorityOne ; DEBUG ( if ( verbose ) dbgs ( ) << <STR_LIT> ) ; } } } else { for ( const auto & SI : SU -> Succs ) { if ( SI . getLatency ( ) > <NUM_LIT> && Bot . ResourceModel -> isInPacket ( SI . getSUnit ( ) ) ) { ResCount -= PriorityOne ; DEBUG ( if ( verbose ) dbgs ( ) << <STR_LIT> ) ; } }


<s> bool <ISA_LIT>CallMutation :: shouldTFRICallBind ( const <ISA_LIT>InstrInfo & HII , const SUnit & Inst1 , const SUnit & Inst2 ) const { if ( Inst1 . getInstr ( ) -> getOpcode ( ) != <STR_LIT> ) return false ; if ( HII . getType ( * Inst2 . getInstr ( ) ) == <STR_LIT> ) return true ; return false ;


<s> SchedModel ( SM ) , TotalPackets ( <NUM_LIT> ) { ResourcesModel = TM . getSubtargetImpl ( ) -> getInstrInfo ( ) -> CreateTargetScheduleState ( * TM . getSubtargetImpl ( ) ) ; assert ( ResourcesModel && <STR_LIT> ) ; Packet . resize ( SchedModel -> getIssueWidth ( ) ) ;


<s> case TargetOpcode :: COPY : case TargetOpcode :: INLINEASM : break ; } MachineBasicBlock * MBB = SU -> getInstr ( ) -> getParent ( ) ; auto & QST = MBB -> getParent ( ) -> getSubtarget < <ISA_LIT>Subtarget > ( ) ; const auto & QII = * QST . getInstrInfo ( ) ; if ( IsTop ) { for ( unsigned i = <NUM_LIT> , e = Packet . size ( ) ; i != e ; ++ i ) if ( hasDependence ( Packet [ i ] , SU , QII ) ) return false ; } else {

for ( unsigned i = <NUM_LIT> , e = Packet . size ( ) ; i != e ; ++ i ) if ( hasDependence ( SU , Packet [ i ] , QII ) ) return false ;


<s> case TargetOpcode :: COPY : case TargetOpcode :: INLINEASM : break ; } Packet . push_back ( SU ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << TotalPackets << <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = Packet . size ( ) ; i != e ; ++ i ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << i << <STR_LIT> ) ;

LLVM_DEBUG ( dbgs ( ) << Packet [ i ] -> NodeNum << <STR_LIT> ) ;


<s> break ; case TargetOpcode :: EXTRACT_SUBREG : case TargetOpcode :: INSERT_SUBREG : case TargetOpcode :: SUBREG_TO_REG : case TargetOpcode :: REG_SEQUENCE : case TargetOpcode :: IMPLICIT_DEF : case TargetOpcode :: KILL : case TargetOpcode :: CFI_INSTRUCTION : case TargetOpcode :: EH_LABEL : case TargetOpcode :: COPY : case TargetOpcode :: INLINEASM : break ; } Packet . push_back ( SU ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << TotalPackets << <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = Packet . size ( ) ; i != e ; ++ i ) {

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << i << <STR_LIT> ) ;


DAG ( <NUM_LIT> ) , TRI ( <NUM_LIT> ) , Top ( TopQID , <STR_LIT> ) , Bot ( BotQID , <STR_LIT> ) {


<s> VLIWResourceModel * <ISA_LIT>ConvergingVLIWScheduler :: createVLIWResourceModel ( const TargetSubtargetInfo & STI , const TargetSchedModel * SchedModel ) const { return new <ISA_LIT>VLIWResourceModel ( STI , SchedModel ) ;


<s> if ( QII -> mayBeCurLoad ( * SUd -> getInstr ( ) ) ) return false ; if ( QII -> canExecuteInBundle ( * SUd -> getInstr ( ) , * SUu -> getInstr ( ) ) ) return false ; return VLIWResourceModel :: hasDependence ( SUd , SUu ) ;


<s> SchedCandidate ( ) : SU ( NULL ) , SCost ( <NUM_LIT> ) {


<s> if ( ! SU || SU -> isScheduled ) return ResCount ; auto & QST = DAG -> MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; auto & QII = * QST . getInstrInfo ( ) ; if ( SU -> isInstr ( ) && QII . mayBeCurLoad ( * SU -> getInstr ( ) ) ) { if ( Q . getID ( ) == TopQID && Top . ResourceModel -> isResourceAvailable ( SU , true ) ) { ResCount += PriorityTwo ;

LLVM_DEBUG ( if ( verbose ) dbgs ( ) << <STR_LIT> ) ;


<s> Packet . resize ( InstrItins -> SchedModel -> IssueWidth ) ; Packet . clear ( ) ; ResourcesModel -> clearResources ( ) ;


<s> VLIWResourceModel ( const TargetMachine & TM ) : InstrItins ( TM . getInstrItineraryData ( ) ) , TotalPackets ( <NUM_LIT> ) { ResourcesModel = TM . getInstrInfo ( ) -> CreateTargetScheduleState ( & TM , NULL ) ; assert ( ResourcesModel && <STR_LIT> ) ; Packet . resize ( InstrItins -> SchedModel -> IssueWidth ) ; Packet . clear ( ) ;


<s> SchedModel ( SM ) , TotalPackets ( <NUM_LIT> ) {

ResourcesModel = TM . getInstrInfo ( ) -> CreateTargetScheduleState ( & TM , nullptr ) ;


<s> DAG ( <NUM_LIT> ) , SchedModel ( <NUM_LIT> ) , Available ( ID , Name + <STR_LIT> ) , Pending ( ID << ConvergingVLIWScheduler :: LogMaxQID , Name + <STR_LIT> ) , CheckPending ( false ) , HazardRec ( <NUM_LIT> ) , ResourceModel ( <NUM_LIT> ) , CurrCycle ( <NUM_LIT> ) , IssueCount ( <NUM_LIT> ) , MinReadyCycle ( UINT_MAX ) , MaxMinLatency ( <NUM_LIT> ) {


<s> VLIWSchedBoundary ( unsigned ID , const Twine & Name ) :

DAG ( <NUM_LIT> ) , SchedModel ( <NUM_LIT> ) , Available ( ID , Name + <STR_LIT> ) , Pending ( ID << ConvergingVLIWScheduler :: LogMaxQID , Name + <STR_LIT> ) , CheckPending ( false ) , HazardRec ( <NUM_LIT> ) , ResourceModel ( <NUM_LIT> ) , CurrCycle ( <NUM_LIT> ) , IssueCount ( <NUM_LIT> ) , MinReadyCycle ( UINT_MAX ) , MaxMinLatency ( <NUM_LIT> ) {


<s> case TargetOpcode :: EXTRACT_SUBREG : case TargetOpcode :: INSERT_SUBREG : case TargetOpcode :: SUBREG_TO_REG : case TargetOpcode :: REG_SEQUENCE : case TargetOpcode :: IMPLICIT_DEF : case TargetOpcode :: COPY : case TargetOpcode :: INLINEASM : break ; } MachineFunction & MF = * SU -> getInstr ( ) -> getParent ( ) -> getParent ( ) ; auto & QII = * MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) ; for ( unsigned i = <NUM_LIT> , e = Packet . size ( ) ; i != e ; ++ i ) { if ( Packet [ i ] -> Succs . size ( ) == <NUM_LIT> ) continue ; if ( QII . mayBeCurLoad ( Packet [ i ] -> getInstr ( ) ) ) continue ; for ( SUnit :: const_succ_iterator I = Packet [ i ] -> Succs . begin ( ) , E = Packet [ i ] -> Succs . end ( ) ; I != E ; ++ I ) { if ( I -> isCtrl ( ) ) continue ; if ( I -> getSUnit ( ) == SU ) return false ; } }


<s> DEBUG ( dbgs ( ) << <STR_LIT> << CandLatency << <STR_LIT> << InstrLatency << <STR_LIT> << * MI << * CandI << <STR_LIT> ) ; if ( Q . getID ( ) == TopQID && CurrentCost == Candidate . SCost ) { if ( InstrLatency < CandLatency && TopUseShorterTie ) { Candidate . SU = * I ; Candidate . RPDelta = RPDelta ; Candidate . SCost = CurrentCost ; FoundCandidate = BestCost ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ; } else if ( InstrLatency > CandLatency && ! TopUseShorterTie ) { Candidate . SU = * I ; Candidate . RPDelta = RPDelta ; Candidate . SCost = CurrentCost ; FoundCandidate = BestCost ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ; } } else if ( Q . getID ( ) == BotQID && CurrentCost == Candidate . SCost ) { if ( InstrLatency < CandLatency && BotUseShorterTie ) { Candidate . SU = * I ; Candidate . RPDelta = RPDelta ; Candidate . SCost = CurrentCost ; FoundCandidate = BestCost ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ; } else if ( InstrLatency > CandLatency && ! BotUseShorterTie ) {


<s> } auto & QST = DAG -> MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; auto & QII = * QST . getInstrInfo ( ) ; if ( SU -> isInstr ( ) && QII . mayBeCurLoad ( SU -> getInstr ( ) ) ) { if ( Q . getID ( ) == TopQID && Top . ResourceModel -> isResourceAvailable ( SU ) ) { ResCount += PriorityTwo ; DEBUG ( if ( verbose ) dbgs ( ) << <STR_LIT> ) ; } else if ( Q . getID ( ) == BotQID && Bot . ResourceModel -> isResourceAvailable ( SU ) ) { ResCount += PriorityTwo ; DEBUG ( if ( verbose ) dbgs ( ) << <STR_LIT> ) ; } } if ( Q . getID ( ) == TopQID ) { for ( const SDep & PI : SU -> Preds ) { if ( ! PI . getSUnit ( ) -> getInstr ( ) -> isPseudo ( ) && PI . isAssignedRegDep ( ) && PI . getLatency ( ) == <NUM_LIT> && Top . ResourceModel -> isInPacket ( PI . getSUnit ( ) ) ) { ResCount += PriorityThree ; DEBUG ( if ( verbose ) dbgs ( ) << <STR_LIT> ) ; } } } else { for ( const SDep & SI : SU -> Succs ) { if ( ! SI . getSUnit ( ) -> getInstr ( ) -> isPseudo ( ) && SI . isAssignedRegDep ( ) && SI . getLatency ( ) == <NUM_LIT> && Bot . ResourceModel -> isInPacket ( SI . getSUnit ( ) ) ) { ResCount += PriorityThree ; DEBUG ( if ( verbose ) dbgs ( ) << <STR_LIT> ) ; } } } if ( QII . isV60VectorInstruction ( Instr ) ) { if ( Q . getID ( ) == TopQID ) { for ( auto J : Top . ResourceModel -> OldPacket ) if ( QII . producesStall ( J -> getInstr ( ) , Instr ) ) ResCount -= PriorityOne ; } else { for ( auto J : Bot . ResourceModel -> OldPacket ) if ( QII . producesStall ( Instr , J -> getInstr ( ) ) ) ResCount -= PriorityOne ; } } if ( CheckEarlyAvail ) { if ( Q . getID ( ) == TopQID ) { for ( const auto & PI : SU -> Preds ) { if ( PI . getLatency ( ) > <NUM_LIT> && Top . ResourceModel -> isInPacket ( PI . getSUnit ( ) ) ) { ResCount -= PriorityOne ; DEBUG ( if ( verbose ) dbgs ( ) << <STR_LIT> ) ; } } } else {


<s> ConvergingVLIWScheduler ( ) : Top ( TopQID , <STR_LIT> ) , Bot ( BotQID , <STR_LIT> ) {


<s> int getBBSize ( ) { return BB -> size ( ) ;


<s> RegisterClassInfo * getRegClassInfo ( ) { return RegClassInfo ;


<s> SchedModel = smodel ; CurrCycle = <NUM_LIT> ; IssueCount = <NUM_LIT> ; CriticalPathLength = DAG -> getBBSize ( ) / SchedModel -> getIssueWidth ( ) ; if ( DAG -> getBBSize ( ) < <NUM_LIT> ) CriticalPathLength >>= <NUM_LIT> ; else {


<s> Top . DAG = DAG ; Bot . DAG = DAG ; const TargetMachine & TM = DAG -> MF . getTarget ( ) ; const InstrItineraryData * Itin = TM . getInstrItineraryData ( ) ; Top . HazardRec = TM . getInstrInfo ( ) -> CreateTargetMIHazardRecognizer ( Itin , DAG ) ;

Bot . HazardRec = TM . getInstrInfo ( ) -> CreateTargetMIHazardRecognizer ( Itin , DAG ) ;


<s> bool isLatencyBound ( SUnit * SU ) { if ( CurrCycle >= CriticalPathLength ) return true ; unsigned PathLength = isTop ( ) ? SU -> getHeight ( ) : SU -> getDepth ( ) ; return CriticalPathLength - CurrCycle <= PathLength ;


<s> void VLIWMachineScheduler :: postprocessDAG ( ) { SUnit * LastSequentialCall = NULL ; for ( unsigned su = <NUM_LIT> , e = SUnits . size ( ) ; su != e ; ++ su ) { if ( SUnits [ su ] . getInstr ( ) -> isCall ( ) ) LastSequentialCall = & ( SUnits [ su ] ) ; else if ( SUnits [ su ] . getInstr ( ) -> isCompare ( ) && LastSequentialCall ) SUnits [ su ] . addPred ( SDep ( LastSequentialCall , SDep :: Order , <NUM_LIT> , <NUM_LIT> , false ) ) ;


return Top . ResourceModel -> getTotalPackets ( ) + Bot . ResourceModel -> getTotalPackets ( ) ;


<s> } if ( ! isResourceAvailable ( SU ) ) { ResourcesModel -> clearResources ( ) ; Packet . clear ( ) ; TotalPackets ++ ; startNewCycle = true ; } switch ( SU -> getInstr ( ) -> getOpcode ( ) ) { default : ResourcesModel -> reserveResources ( SU -> getInstr ( ) ) ; break ; case TargetOpcode :: EXTRACT_SUBREG : case TargetOpcode :: INSERT_SUBREG : case TargetOpcode :: SUBREG_TO_REG : case TargetOpcode :: REG_SEQUENCE : case TargetOpcode :: IMPLICIT_DEF : case TargetOpcode :: KILL : case TargetOpcode :: PROLOG_LABEL : case TargetOpcode :: EH_LABEL : case TargetOpcode :: COPY : case TargetOpcode :: INLINEASM : break ; } Packet . push_back ( SU ) ;

DEBUG ( dbgs ( ) << <STR_LIT> << TotalPackets << <STR_LIT> ) ;