<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return SplitHvxPairOp ( Op , DAG ) ; } } switch ( Opc ) { default : break ; case <STR_LIT> : return LowerHvxBuildVector ( Op , DAG ) ; case <STR_LIT> : return LowerHvxConcatVectors ( Op , DAG ) ; case <STR_LIT> : return LowerHvxInsertSubvector ( Op , DAG ) ; case <STR_LIT> : return LowerHvxInsertElement ( Op , DAG ) ; case <STR_LIT> : return LowerHvxExtractSubvector ( Op , DAG ) ; case <STR_LIT> : return LowerHvxExtractElement ( Op , DAG ) ; case <STR_LIT> : return LowerHvxAnyExt ( Op , DAG ) ; case <STR_LIT> : return LowerHvxSignExt ( Op , DAG ) ; case <STR_LIT> : return LowerHvxZeroExt ( Op ,


<s> MOp1 = MF . getMachineMemOperand ( MMO , HwLen , HwLen ) ; } unsigned MemOpc = BN -> getOpcode ( ) ; SDValue NewOp ; if ( MemOpc == <STR_LIT> ) { SDValue Load0 = DAG . getLoad ( SingleTy , dl , Chain , Base0 , MOp0 ) ; SDValue Load1 = DAG . getLoad ( SingleTy , dl , Chain , Base1 , MOp1 ) ; NewOp = DAG . getMergeValues ( { DAG . getNode ( <STR_LIT> , dl , MemTy , Load0 , Load1 ) , DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Load0 . getValue ( <NUM_LIT> ) , Load1 . getValue ( <NUM_LIT> ) ) } , dl ) ; } else { assert ( MemOpc ==


<s> LSBaseSDNode * BN = cast < LSBaseSDNode > ( Op . getNode ( ) ) ; assert ( BN -> isUnindexed ( ) ) ; MVT MemTy = BN -> getMemoryVT ( ) . getSimpleVT ( ) ; if ( ! isHvxPairTy ( MemTy ) ) return Op ; const SDLoc & dl ( Op ) ; unsigned HwLen = Subtarget . getVectorLength ( ) ; MVT SingleTy = typeSplit ( MemTy ) . first ; SDValue Chain = BN -> getChain ( ) ; SDValue Base0 = BN -> getBasePtr ( ) ; SDValue Base1 = DAG . getMemBasePlusOffset ( Base0 , HwLen , dl ) ; MachineMemOperand * MOp0 = nullptr , * MOp1 = nullptr ; if ( MachineMemOperand * MMO = BN -> getMemOperand ( ) ) { MachineFunction & MF = DAG . getMachineFunction ( ) ; MOp0 = MF . getMachineMemOperand ( MMO , <NUM_LIT> , HwLen


<s> SDValue SizeNode = DAG . getConstant ( Flags . getByValSize ( ) , dl , <STR_LIT> ) ; return DAG . getMemcpy ( Chain , dl , Dst , Src , SizeNode , Flags . getByValAlign ( )

, false , false , false , MachinePointerInfo ( ) , MachinePointerInfo ( ) ) ;


<s> return DAG . getMemcpy ( Chain , dl , Dst , Src , SizeNode , Flags . getByValAlign ( ) , false , false , false ,

MachinePointerInfo ( ) , MachinePointerInfo ( ) ) ;


<s> if ( VT . getSimpleVT ( ) == <STR_LIT> ) return DAG . getNode ( <STR_LIT> , dl , VT , Val ) ; if ( VT . getSimpleVT ( ) == <STR_LIT> ) return DAG . getNode ( <STR_LIT> , dl , VT , Val ) ; return SDValue


<s> if ( VT . getSimpleVT ( ) == <STR_LIT> ) return DAG . getNode ( <STR_LIT> , dl , VT , Val ) ; return SDValue (


<s> switch ( MI -> getOpcode ( ) ) { case <STR_LIT> : { MachineFunction * MF = BB


<s> switch ( MI -> getOpcode ( ) ) { case <STR_LIT> : { MachineFunction * MF = BB -> getParent ( ) ; auto * FuncInfo = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; FuncInfo -> addAllocaAdjustInst ( MI ) ; return BB ; } default : llvm_unreachable ( <STR_LIT>


<s> case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> :


<s> SDValue <ISA_LIT>TargetLowering :: LowerBlockAddress ( SDValue Op , SelectionDAG & DAG ) const { const BlockAddress * BA = cast < BlockAddressSDNode > ( Op ) -> getBlockAddress ( ) ; SDValue BA_SD = DAG . getTargetBlockAddress ( BA , <STR_LIT> ) ; SDLoc dl ( Op ) ; return DAG . getNode ( <STR_LIT> , dl , getPointerTy ( DAG . getDataLayout ( ) ) ,


<s> LowerBR_JT ( SDValue Op , SelectionDAG & DAG ) const { SDValue Chain = Op . getOperand ( <NUM_LIT> ) ; SDValue Table = Op . getOperand ( <NUM_LIT> ) ; SDValue Index = Op . getOperand ( <NUM_LIT> ) ; SDLoc dl ( Op ) ; JumpTableSDNode * JT = cast < JumpTableSDNode > ( Table ) ; unsigned JTI = JT -> getIndex ( ) ; MachineFunction & MF = DAG . getMachineFunction ( ) ; const MachineJumpTableInfo * MJTI = MF . getJumpTableInfo ( ) ; SDValue TargetJT = DAG . getTargetJumpTable ( JT -> getIndex ( ) , <STR_LIT> ) ; const std :: vector < MachineJumpTableEntry > & JTE = MJTI


<s> unsigned SplatBits = APSplatBits . getZExtValue ( ) ; int32_t SextVal = ( ( int32_t ) ( SplatBits << ( <NUM_LIT> - SplatBitSize ) ) >> ( <NUM_LIT> - SplatBitSize ) ) ; return createSplat ( DAG , dl , VT , DAG . getConstant ( SextVal , dl , <STR_LIT> ) ) ; } if ( VT . getSimpleVT ( ) == <STR_LIT> ) { SDValue V0 = BVN -> getOperand ( <NUM_LIT> ) ; SDValue V1 = BVN -> getOperand ( <NUM_LIT> ) ; if ( V0 . getOpcode ( ) == <STR_LIT> ) V0 = DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ; if ( V1 . getOpcode ( ) == <STR_LIT> ) V1 = DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ; ConstantSDNode * C0 = dyn_cast < ConstantSDNode > ( V0 ) ; ConstantSDNode * C1 = dyn_cast < ConstantSDNode > ( V1 ) ; if ( ! C0 || ! C1 ) return DAG . getNode ( <STR_LIT> , dl , VT , V1 , V0 ) ; if ( isInt < <NUM_LIT> > ( C0 -> getSExtValue ( ) ) || isInt < <NUM_LIT> > ( C1 -> getSExtValue ( ) ) ) return DAG . getNode ( <STR_LIT> , dl , VT , V1 , V0 ) ; } if ( VT . getSimpleVT ( ) == <STR_LIT> ) { for ( unsigned i = <NUM_LIT> , e = NElts ; i != e ; ++ i ) { if ( BVN -> getOperand ( i ) . getOpcode ( ) == <STR_LIT> ) continue ; ConstantSDNode * Cst = dyn_cast < ConstantSDNode > ( BVN -> getOperand ( i ) ) ; if ( ! Cst ) { SDValue pack = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , BVN -> getOperand ( <NUM_LIT> ) , BVN -> getOperand ( <NUM_LIT> ) ) ; return DAG . getTargetExtractSubreg ( <STR_LIT> , dl , <STR_LIT> , pack ) ; } } } uint64_t Res = <NUM_LIT> ; unsigned EltSize = Size / NElts ; SDValue


<s> EVT VecVT = Vec . getValueType ( ) ; SDValue Width = DAG . getConstant ( VecVT . getSizeInBits ( ) , dl , <STR_LIT> ) ; SDValue Shifted = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Width , DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ) ; SDValue ConstVal = DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ; ConstantSDNode * W = dyn_cast < ConstantSDNode > ( Width ) ; ConstantSDNode * S = dyn_cast < ConstantSDNode > ( Shifted ) ; if ( ( VecVT . getSimpleVT ( ) == <STR_LIT> ) && ( NElts == <NUM_LIT> ) && W && S ) { if ( ( W -> getZExtValue ( ) == <NUM_LIT> ) && ( ( S -> getZExtValue ( ) >> <NUM_LIT> ) == <NUM_LIT> ) ) { SDValue Vec0 = Op . getOperand ( <NUM_LIT> ) ; SDValue Combined = DAG . getNode ( <STR_LIT> , dl , VT , Vec0 , Vec ) ; return DAG . getNode ( <STR_LIT> , dl


<s> SDValue Chain = Op . getOperand ( <NUM_LIT> ) ; SDValue Size = Op . getOperand ( <NUM_LIT> ) ; SDValue Align = Op . getOperand ( <NUM_LIT> ) ; SDLoc dl ( Op ) ; ConstantSDNode * AlignConst = dyn_cast < ConstantSDNode > ( Align ) ; assert ( AlignConst && <STR_LIT> ) ; unsigned A = AlignConst -> getSExtValue ( ) ; auto & HFI = * Subtarget . getFrameLowering ( ) ; if ( A == <NUM_LIT> ) A = HFI . getStackAlignment ( ) ; DEBUG (


<s> MachineFunction & MF = DAG . getMachineFunction ( ) ; MachineFrameInfo * MFI = MF . getFrameInfo ( ) ; MachineRegisterInfo & RegInfo = MF . getRegInfo ( ) ; auto & FuncInfo = * MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( CallConv , isVarArg , DAG . getMachineFunction ( ) , ArgLocs , * DAG . getContext ( ) ) ; CCInfo . AnalyzeFormalArguments ( Ins , CC_<ISA_LIT> ) ; SmallVector < SDValue , <NUM_LIT> > MemOps ; for ( unsigned i = <NUM_LIT> , e = ArgLocs . size ( ) ; i != e ; ++ i ) { CCValAssign & VA = ArgLocs [ i ] ; <STR_LIT> Flags = Ins [ i ] . Flags ; unsigned ObjSize ; unsigned StackLocation ; int FI ; if ( ( VA . isRegLoc ( ) && ! Flags . isByVal ( ) ) || ( VA . isRegLoc ( ) && Flags . isByVal ( ) && Flags . getByValSize ( ) > <NUM_LIT> ) ) { EVT RegVT = VA . getLocVT ( ) ; if ( RegVT == <STR_LIT> || RegVT == <STR_LIT> || RegVT == <STR_LIT> || RegVT == <STR_LIT> ) { unsigned VReg = RegInfo . createVirtualRegister ( & <STR_LIT> ) ; RegInfo . addLiveIn ( VA . getLocReg ( ) , VReg ) ; InVals . push_back ( DAG . getCopyFromReg ( Chain , dl , VReg , RegVT ) ) ; } else if ( RegVT == <STR_LIT> || RegVT == <STR_LIT> ) { unsigned VReg = RegInfo . createVirtualRegister ( & <STR_LIT> ) ; RegInfo . addLiveIn ( VA . getLocReg ( ) , VReg ) ; InVals . push_back ( DAG . getCopyFromReg ( Chain , dl , VReg , RegVT ) ) ; } else { assert ( <NUM_LIT> ) ; } } else if ( VA . isRegLoc ( ) && Flags . isByVal ( ) && Flags . getByValSize ( ) <= <NUM_LIT> ) { assert ( <NUM_LIT> && <STR_LIT> ) ; } else { assert ( VA . isMemLoc ( ) ) ; if ( Flags . isByVal ( ) ) { ObjSize = Flags . getByValSize ( ) ; } else { ObjSize = VA . getLocVT ( ) . getStoreSizeInBits ( ) >> <NUM_LIT> ; } StackLocation = HEXAGON_LRFP_SIZE + VA


<s> static SDValue LowerVECTOR_SHUFFLE ( SDValue Op , SelectionDAG & DAG ) { const ShuffleVectorSDNode * SVN = cast < ShuffleVectorSDNode > ( Op ) ; SDValue V1 = Op . getOperand ( <NUM_LIT> ) ; SDValue V2 = Op . getOperand ( <NUM_LIT> ) ; SDLoc dl ( Op ) ; EVT VT = Op . getValueType ( ) ; if ( V2 . getOpcode ( ) == <STR_LIT> ) V2 = V1 ; if ( SVN -> isSplat ( ) ) { int Lane = SVN -> getSplatIndex ( ) ; if ( Lane == - <NUM_LIT> ) Lane = <NUM_LIT> ; if ( Lane == <NUM_LIT> && V1 . getOpcode ( ) == <STR_LIT> ) return createSplat ( DAG , dl , VT , V1 . getOperand ( <NUM_LIT> ) ) ; if ( Lane == <NUM_LIT> && V1 . getOpcode ( ) == <STR_LIT> && ! isa < ConstantSDNode > ( V1 . getOperand ( <NUM_LIT> ) ) ) { bool IsScalarToVector = true ; for ( unsigned i = <NUM_LIT> , e = V1 . getNumOperands ( ) ; i != e ; ++ i ) if ( V1 . getOperand ( i ) . getOpcode ( ) != <STR_LIT> ) { IsScalarToVector = false ; break ; } if ( IsScalarToVector ) return createSplat ( DAG , dl , VT , V1 . getOperand


<s> setOperationAction ( <STR_LIT> , VT . getSimpleVT ( ) , Promote ) ; AddPromotedToType ( <STR_LIT> , VT . getSimpleVT ( ) , PromotedLdStVT .


<s> TargetLowering :: AtomicExpansionKind <ISA_LIT>TargetLowering :: shouldExpandAtomicLoadInIR ( LoadInst * LI ) const { return LI -> getType ( ) -> getPrimitiveSizeInBits ( ) > <NUM_LIT> ? AtomicExpansionKind ::


<s> assert ( <NUM_LIT> && <STR_LIT> ) ; } return new <ISA_LIT>LDBackend ( pConfig , new <ISA_LIT>GNUInfo


<s> } else { m_pGOTSymbol = pBuilder . AddSymbol < IRBuilder :: Force , IRBuilder :: Resolve > ( <STR_LIT> , ResolveInfo :: Object , ResolveInfo :: Define , ResolveInfo :: Local , <NUM_LIT> , <NUM_LIT> , FragmentRef :: Create ( pFrag

, <NUM_LIT> ) , ResolveInfo :: Hidden ) ;


<s> setGOTSectionSize ( pBuilder ) ; if ( m_pPLT -> hasPLT1 ( ) ) m_pPLT -> finalizeSectionSize ( ) ; if ( ! m_pRelaDyn -> empty ( ) ) { assert ( ! config ( ) . isCodeStatic ( ) && <STR_LIT> ) ; setRelaDynSize ( ) ; } if ( ! m_pRelaPLT ->


<s> if ( m_pPLT -> hasPLT1 ( ) ) m_pPLT -> finalizeSectionSize ( ) ; if ( ! m_pRelaDyn -> empty ( ) ) { assert ( ! config ( ) . isCodeStatic ( ) && <STR_LIT> ) ; setRelaDynSize ( ) ; } if ( ! m_pRelaPLT -> empty ( ) ) { assert ( ! config ( ) . isCodeStatic (


<s> } } } } std :: vector < Fragment * > invalid_frags ; pFinished = true ; for ( BranchIslandFactory :: iterator island = getBRIslandFactory ( ) -> begin ( ) , island_end = getBRIslandFactory ( ) -> end ( ) ; island != island_end ; ++ island ) { if ( ( * island ) . size ( ) > stubGroupSize ( ) ) { error ( diag :: err_no_space_to_place_stubs ) << stubGroupSize ( ) ; return false ; } if ( ( * island ) . numOfStubs ( ) == <NUM_LIT> ) { continue ; } Fragment * exit = & * ( * island ) . end ( ) ; if ( exit == & * ( * island ) . begin ( ) -> getParent ( ) -> end ( ) ) { continue ; } if ( ( ( * island ) . offset ( ) + ( * island ) . size ( ) ) > exit -> getOffset ( ) ) { if ( invalid_frags . empty ( ) || ( invalid_frags . back ( ) -> getParent ( ) != ( * island ) . getParent ( ) ) ) { invalid_frags . push_back ( exit ) ; pFinished = false ; } continue ; } } for ( auto it = invalid_frags . begin ( ) , ie = invalid_frags . end ( ) ; it != ie ; ++ it ) { Fragment * invalid = * it ; while ( invalid != NULL ) { invalid -> setOffset ( invalid -> getPrevNode ( ) -> getOffset ( ) + invalid -> getPrevNode ( ) -> size ( ) ) ; invalid = invalid -> getNextNode ( ) ; } } if ( isRelaxed ) { SectionData * prev = NULL ; for ( BranchIslandFactory :: iterator island = getBRIslandFactory ( ) -> begin ( ) , island_end = getBRIslandFactory ( ) -> end ( ) ; island != island_end ; ++ island ) { SectionData * sd = ( * island ) . begin ( ) -> getParent ( ) ; if ( ( * island ) . numOfStubs


<s> const <ISA_LIT>ELFDynamic & <ISA_LIT>LDBackend :: dynamic ( ) const { assert ( m_pDynamic != NULL ) ; return *


<s> uint64_t RegionSize = <NUM_LIT> ; for ( <ISA_LIT>GOTPLT :: iterator it = m_pGOTPLT -> begin ( ) , ie = m_pGOTPLT -> end ( ) ; it != ie ; ++ it , ++ buffer ) { got = & ( llvm :: cast < <ISA_LIT>GOTEntry > ( ( * it ) ) ) ; * buffer = static_cast < uint32_t > ( got -> getValue ( ) ) ; RegionSize += EntrySize ; } return RegionSize


<s> uint32_t * buffer = reinterpret_cast < uint32_t * > ( pRegion . begin ( ) ) ; <ISA_LIT>GOTEntry * got = <NUM_LIT> ; unsigned int EntrySize = <STR_LIT> ; uint64_t RegionSize = <NUM_LIT> ; for ( <ISA_LIT>GOTPLT :: iterator it = m_pGOTPLT -> begin


<s> bool <ISA_LIT>LDBackend :: finalizeTargetSymbols ( ) { if ( config ( ) . codeGenType ( ) == LinkerConfig :: Object ) return true ; if ( m_psdabase ) m_psdabase -> setValue ( m_psdata -> addr ( ) ) ; ELFSegmentFactory :: const_iterator edata = elfSegmentTable ( ) . find ( llvm :: ELF :: PT_LOAD , llvm :: ELF :: PF_W , llvm :: ELF :: PF_X ) ; if ( elfSegmentTable ( ) . end ( ) != edata ) { if ( f_pEData != NULL && ResolveInfo :: ThreadLocal != f_pEData -> type ( ) ) { f_pEData -> setValue ( ( * edata ) -> vaddr ( ) + ( * edata ) -> filesz ( ) ) ; } if ( f_p_EData != NULL && ResolveInfo :: ThreadLocal != f_p_EData -> type ( ) ) { f_p_EData -> setValue ( ( * edata ) -> vaddr ( ) +


<s> Relocation :: Type getCopyRelType ( ) const { return m_CopyRel


<s> assert ( m_pGOTPLT != NULL ) ; return *


<s> const <ISA_LIT>PLT & <ISA_LIT>LDBackend :: getPLT ( ) const { assert ( m_pPLT != NULL && <STR_LIT> ) ; return * m_pPLT


<s> const OutputRelocSection & <ISA_LIT>LDBackend :: getRelaDyn ( ) const { assert ( m_pRelaDyn != NULL && <STR_LIT> ) ; return *


<s> size_t getRelaEntrySize ( ) { return


<s> size_t getRelaEntrySize ( ) { return <NUM_LIT>


<s> assert ( m_pRelocator != NULL ) ; return m_pRelocator


<s> if ( shndx >= llvm :: ELF :: SHN_HEXAGON_SCOMMON && shndx <= llvm :: ELF :: SHN_HEXAGON_SCOMMON_8 ) return


<s> ResolveInfo :: Desc getSymDesc ( uint16_t shndx ) const { if ( shndx >= llvm :: ELF :: SHN_HEXAGON_SCOMMON && shndx <= llvm

:: ELF :: SHN_HEXAGON_SCOMMON_8 ) return ResolveInfo :: Common ;


<s> bool <ISA_LIT>LDBackend :: initBRIslandFactory ( ) { if ( m_pBRIslandFactory == NULL ) { m_pBRIslandFactory = new BranchIslandFactory ( maxFwdBranchOffset ( ) , maxBwdBranchOffset (


<s> bool <ISA_LIT>LDBackend :: initRelocator ( ) { if ( m_pRelocator ==


<s> bool <ISA_LIT>LDBackend :: initStubFactory ( ) { if ( m_pStubFactory == NULL


<s> m_pPLT = new <ISA_LIT>PLT ( plt , * m_pGOTPLT , config ( ) ) ; LDSection & relaplt = file_format -> getRelaPlt ( ) ; relaplt . setLink ( & plt ) ; m_pRelaPLT = new OutputRelocSection ( pModule , relaplt ) ; LDSection & reladyn = file_format -> getRelaDyn ( ) ; m_pRelaDyn = new OutputRelocSection ( pModule , reladyn ) ; } m_psdata = pBuilder . CreateSection ( <STR_LIT> , LDFileFormat :: Target , llvm :: ELF :: SHT_PROGBITS , llvm :: ELF :: SHF_ALLOC | llvm :: ELF :: SHF_WRITE , <NUM_LIT> * <NUM_LIT> ) ; m_pscommon_1 = pBuilder . CreateSection ( <STR_LIT> , LDFileFormat :: Target , llvm :: ELF :: SHT_PROGBITS , llvm :: ELF :: SHF_ALLOC | llvm :: ELF :: SHF_WRITE , <NUM_LIT> ) ; IRBuilder :: CreateSectionData ( *


<s> IRBuilder :: CreateSectionData ( * m_pscommon_1 ) ; m_pscommon_2 = pBuilder . CreateSection ( <STR_LIT> , LDFileFormat :: Target , llvm :: ELF :: SHT_PROGBITS , llvm :: ELF :: SHF_ALLOC | llvm :: ELF :: SHF_WRITE , <NUM_LIT> ) ; IRBuilder :: CreateSectionData ( * m_pscommon_2 ) ; m_pscommon_4 = pBuilder . CreateSection ( <STR_LIT> , LDFileFormat :: Target , llvm :: ELF :: SHT_PROGBITS , llvm :: ELF :: SHF_ALLOC | llvm :: ELF :: SHF_WRITE , <NUM_LIT> ) ; IRBuilder :: CreateSectionData ( * m_pscommon_4 ) ; m_pscommon_8 = pBuilder . CreateSection ( <STR_LIT> , LDFileFormat :: Target , llvm :: ELF :: SHT_PROGBITS , llvm :: ELF :: SHF_ALLOC | llvm :: ELF :: SHF_WRITE , <NUM_LIT> ) ; IRBuilder :: CreateSectionData (


<s> void <ISA_LIT>LDBackend :: initTargetSymbols ( IRBuilder & pBuilder , Module & pModule ) { if ( config ( ) . codeGenType ( ) == LinkerConfig :: Object ) return ; m_pGOTSymbol = pBuilder . AddSymbol < IRBuilder :: AsReferred , IRBuilder :: Resolve > ( <STR_LIT> , ResolveInfo :: Object , ResolveInfo :: Define , ResolveInfo :: Local , <NUM_LIT> , <NUM_LIT> , FragmentRef :: Null ( ) , ResolveInfo :: Hidden ) ; m_psdabase = pBuilder . AddSymbol < IRBuilder :: AsReferred , IRBuilder :: Resolve > ( <STR_LIT> , ResolveInfo :: Object , ResolveInfo :: Define , ResolveInfo :: Absolute , <NUM_LIT> , <NUM_LIT> , FragmentRef :: Null ( ) , ResolveInfo :: Hidden ) ; pBuilder . AddSymbol < IRBuilder :: AsReferred , IRBuilder :: Resolve > ( <STR_LIT> , ResolveInfo :: Object , ResolveInfo :: Define , ResolveInfo :: Absolute


<s> void MCLDInitialize<ISA_LIT>LDBackend ( ) { mcld :: TargetRegistry :: RegisterTargetLDBackend ( mcld :: The<ISA_LIT>Target ,


<s> mcld :: TargetRegistry :: RegisterTargetLDBackend ( mcld :: The<ISA_LIT>Target , mcld ::


<s> if ( ! m_psdata -> hasSectionData ( ) ) { sd = IRBuilder :: CreateSectionData ( * m_psdata ) ; m_psdata -> setSectionData ( sd ) ; } sd = m_psdata -> getSectionData ( ) ; MoveSectionDataAndSort ( * pInputSection . getSectionData ( ) , * sd ) ; } else { ObjectBuilder builder ( pModule ) ; builder . MergeSection ( pInputFile , pInputSection ) ; } return true


<s> AlignFragment * align = NULL ; if ( pFrom . getSection ( ) . align ( ) > <NUM_LIT> ) { unsigned int alignment = pFrom . getSection ( ) . align ( ) ; align = new AlignFragment ( alignment , <NUM_LIT> , <NUM_LIT> , alignment - <NUM_LIT> ) ; pFrom . getFragmentList ( ) . push_front ( align ) ; } if ( found ) to_list . splice ( fragInsert , pFrom . getFragmentList ( ) ) ; else to_list . splice ( frag , pFrom .


<s> uint32_t size = pSD . getSection ( ) . size ( ) ; if ( pSD . getSection ( ) . type ( ) == llvm :: ELF :: SHT_NOBITS ) { frag = new FillFragment ( <NUM_LIT> , <NUM_LIT> , size ) ; } else { llvm :: StringRef region = pInput . memArea ( ) -> request ( offset , size ) ; if ( region . size ( ) == <NUM_LIT> ) { frag = new FillFragment ( <NUM_LIT> ,


<s> void <ISA_LIT>LDBackend :: setGOTSectionSize ( IRBuilder & pBuilder ) { if ( LinkerConfig :: DynObj == config ( ) . codeGenType ( ) ||

m_pGOTPLT -> hasGOT1 ( ) || m_pGOTSymbol != NULL ) {


<s> ELFFileFormat * file_format = getOutputFormat ( ) ; file_format -> getRelaDyn ( ) . setSize ( m_pRelaDyn -> numOfRelocs ( ) * getRelaEntrySize (


<s> file_format -> getRelaPlt ( ) . setSize ( m_pRelaPLT -> numOfRelocs ( ) * getRelaEntrySize


<s> SectionData :: FragmentListType & to_list = pTo -> getFragmentList ( ) ; SectionData :: FragmentListType :: iterator fragTo , fragToEnd = to_list . end ( ) ; uint32_t offset = <NUM_LIT> ; for ( fragTo = to_list . begin ( ) ; fragTo != fragToEnd ; ++ fragTo ) { fragTo -> setOffset ( offset ) ; offset += fragTo -> size ( ) ; } pTo -> getSection ( ) . setSize ( offset ) ; SectionData :: FragmentListType & newlist = pTo -> getFragmentList ( ) ; for ( fragTo = newlist . begin ( ) , fragToEnd = newlist . end ( ) ; fragTo !=


<s> <ISA_LIT>LDBackend :: ~ <ISA_LIT>LDBackend ( ) { delete m_pRelocator ; delete m_pGOT ; delete m_pPLT ; delete m_pRelaDyn ; delete


<s> void PolynomialMultiplyRecognize :: cleanupLoopBody ( BasicBlock * LoopB ) { for ( auto & I : * LoopB ) if ( Value * SV = SimplifyInstruction ( & I , DL , & TLI , & DT ) ) I . replaceAllUsesWith ( SV ) ; for ( auto I = LoopB -> begin ( ) , N = I


<s> return CV ; } ; for ( auto I = LoopB -> begin ( ) , E = LoopB -> end ( ) ; I != E ; ++ I ) { if ( isa < PHINode > ( I ) || ! Users . count ( & * I ) ) continue ; using namespace PatternMatch ; Value * V = nullptr ; if ( match ( & * I , m_LShr ( m_Value ( V ) , m_One ( ) ) ) ) { replaceAllUsesOfWithIn ( & * I , V , LoopB ) ; continue ; } for ( auto & J : I -> operands ( ) ) { Value * Op = J . get ( ) ; if ( ! isOperandShifted ( & * I , Op ) ) continue ; if ( Users . count ( Op ) ) continue ; if ( isa < ConstantInt > ( Op ) && cast < ConstantInt > ( Op ) -> isZero ( ) ) continue ; auto F = ShiftMap . find ( Op ) ; Value * W = ( F != ShiftMap . end ( ) ) ? F -> second : nullptr ; if ( W == nullptr ) { IRB . SetInsertPoint ( & * I ) ; Value * ShAmt = CIV , * ShVal = Op ; auto * VTy = cast < IntegerType > ( ShVal -> getType ( ) ) ; auto * ATy = cast < IntegerType > ( ShAmt -> getType ( ) ) ; if ( Late . count ( & * I ) ) ShVal = IRB . CreateShl ( Op , ConstantInt :: get ( VTy , <NUM_LIT> ) ) ; if ( VTy != ATy ) { if ( VTy -> getBitWidth ( ) < ATy -> getBitWidth ( ) ) ShVal = upcast ( CastMap , IRB , ShVal , ATy ) ; else ShAmt = upcast ( CastMap , IRB , ShAmt , VTy ) ; } W = IRB . CreateShl ( ShVal , ShAmt ) ; ShiftMap . insert ( std :: make_pair ( Op , W ) ) ; } I -> replaceUsesOfWith ( Op , W ) ; } } IRB . SetInsertPoint ( ExitB , ExitB -> getFirstInsertionPt ( ) ) ; for ( auto P = ExitB -> begin ( ) , Q = ExitB -> end ( ) ; P != Q ; ++ P ) { if ( ! isa < PHINode > ( P ) ) break ; auto * PN = cast < PHINode > ( P ) ; Value * U = PN -> getIncomingValueForBlock ( LoopB ) ; if ( ! Users . count ( U ) ) continue ; Value * S = IRB . CreateLShr ( PN , ConstantInt :: get ( PN -> getType ( )


<s> uint64_t SizeInBits = DL -> getTypeSizeInBits ( SI -> getValueOperand ( ) -> getType ( ) ) ; assert ( ( ( SizeInBits & <NUM_LIT> ) || ( SizeInBits >> <NUM_LIT> ) == <NUM_LIT> ) && <STR_LIT> ) ; return ( unsigned ) SizeInBits


<s> unsigned StoreSize = getStoreSizeInBytes ( SI ) ; if ( StoreSize != unsigned ( std :: abs ( Stride ) ) ) return false ; LoadInst * LI = dyn_cast < LoadInst > ( SI -> getValueOperand ( ) ) ; if ( ! LI || ! LI -> isSimple ( ) ) return false ; Value * LoadPtr = LI -> getPointerOperand ( ) ; auto * LoadEv = dyn_cast < SCEVAddRecExpr > ( SE -> getSCEV ( LoadPtr ) ) ; if ( ! LoadEv || LoadEv -> getLoop ( ) != CurLoop || ! LoadEv -> isAffine ( ) ) return false ; if ( StoreEv -> getOperand ( <NUM_LIT> ) !=

LoadEv -> getOperand ( <NUM_LIT> ) ) return false ;


<s> bool <ISA_LIT>LoopIdiomRecognize :: isLegalStore ( Loop * CurLoop , StoreInst * SI ) { if ( ! ( SI -> isVolatile ( ) && <ISA_LIT>VolatileMemcpy ) && ! SI -> isSimple ( ) ) return false ; Value * StoredVal = SI -> getValueOperand ( ) ; Value * StorePtr = SI -> getPointerOperand ( ) ; uint64_t SizeInBits = DL -> getTypeSizeInBits ( StoredVal -> getType ( ) ) ; if ( ( SizeInBits & <NUM_LIT> ) || ( SizeInBits >> <NUM_LIT> ) != <NUM_LIT> ) return false ; auto * StoreEv = dyn_cast < SCEVAddRecExpr > ( SE -> getSCEV ( StorePtr ) ) ; if ( ! StoreEv || StoreEv -> getLoop ( ) != CurLoop || ! StoreEv -> isAffine ( ) ) return false ; int Stride = getSCEVStride


<s> if ( const SCEVConstant * BECst = dyn_cast < SCEVConstant > ( BECount ) ) AccessSize = ( BECst -> getValue ( ) -> getZExtValue ( ) + <NUM_LIT> ) * StoreSize ; MemoryLocation StoreLoc ( Ptr , AccessSize ) ; for ( auto * B : L -> blocks ( ) ) for ( auto & I : * B ) if ( Ignored . count ( & I ) == <NUM_LIT>

&& ( AA . getModRefInfo ( & I , StoreLoc ) & Access ) ) return true ;


<s> StringRef PP = ( PV . M ? <STR_LIT> : <STR_LIT> ) ; if ( ! PV . Inv ) dbgs ( ) << <STR_LIT> << PP << <STR_LIT> ; else dbgs ( ) << <STR_LIT> << PP << <STR_LIT> << PP << <STR_LIT> ; dbgs ( ) << <STR_LIT> << * PV . Res << <STR_LIT> << * PV . P << <STR_LIT> ; if ( PV . M ) dbgs ( ) << <STR_LIT> << * PV . M << <STR_LIT> ; dbgs ( ) << <STR_LIT> << * PV . Q << <STR_LIT> ; dbgs ( ) << <STR_LIT> << PV . IterCount << <STR_LIT> ; } ) ; BasicBlock :: iterator At ( EntryB -> getTerminator ( ) ) ; Value * PM = generate ( At , PV ) ; if ( PM == nullptr ) return false ; if ( PM -> getType ( ) != PV . Res -> getType ( ) ) PM = IRBuilder < >

( & * At ) . CreateIntCast ( PM , PV . Res -> getType ( ) , false ) ;


<s> if ( ExitB == nullptr ) return false ; BasicBlock * EntryB = CurLoop -> getLoopPreheader ( ) ; if ( EntryB == nullptr ) return false ; unsigned IterCount = <NUM_LIT> ; const SCEV * CT = SE . getBackedgeTakenCount ( CurLoop ) ; if ( isa < SCEVCouldNotCompute > ( CT ) ) return false ; if ( auto * CV = dyn_cast < SCEVConstant > ( CT ) ) IterCount = CV -> getValue ( ) -> getZExtValue ( ) + <NUM_LIT> ; Value * CIV = getCountIV ( LoopB ) ; ParsedValues PV ; PV . IterCount = IterCount ; auto CouldBePmpy = [ this , LoopB , EntryB , CIV , & PV ] ( bool PreScan ) -> std :: function < bool ( Instruction & I ) > { return [ this , LoopB , EntryB , CIV , & PV , PreScan ] ( Instruction & I ) -> bool { if ( auto * SelI = dyn_cast < SelectInst > ( & I ) ) return scanSelect ( SelI , LoopB , EntryB , CIV , PV , PreScan ) ; return false ; } ; } ; auto PreF = std :: find_if ( LoopB -> begin ( ) , LoopB -> end ( ) , CouldBePmpy ( true ) ) ; if ( PreF == LoopB -> end ( ) ) return false ; if ( ! PV


<s> } ; if ( ! std :: all_of ( ExitBlocks . begin ( ) , ExitBlocks . end ( ) , DominatedByBB ) ) return false ; bool MadeChange = false ; SmallVector < StoreInst * , <NUM_LIT> > Stores ; collectStores ( CurLoop , BB


<s> return DT -> dominates ( BB , EB ) ; } ; if ( ! std :: all_of ( ExitBlocks . begin ( ) , ExitBlocks . end ( ) , DominatedByBB ) ) return false ; bool MadeChange = false ; SmallVector < StoreInst * , <NUM_LIT> > Stores ; collectStores ( CurLoop , BB , Stores ) ; for ( auto & SI : Stores ) MadeChange |= processCopyingStore ( CurLoop , SI , BECount ) ; return


<s> StringRef Name = L -> getHeader ( ) -> getParent ( ) -> getName ( ) ; if ( Name == <STR_LIT> || Name == <STR_LIT> || Name == <STR_LIT> ) return false ; AA = & getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; DL = & L -> getHeader ( ) -> getModule ( ) -> getDataLayout ( ) ; DT = & getAnalysis < DominatorTreeWrapperPass > ( ) . getDomTree ( ) ; LF = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; TLI = & getAnalysis < TargetLibraryInfoWrapperPass > ( ) . getTLI ( * L -> getHeader ( ) -> getParent ( ) ) ; SE = & getAnalysis < ScalarEvolutionWrapperPass > ( ) . getSE ( ) ; HasMemcpy = TLI -> has


<s> static bool mayLoopAccessLocation ( Value * Ptr , ModRefInfo Access , Loop * L , const SCEV * BECount , unsigned StoreSize , AliasAnalysis & AA , SmallPtrSetImpl < Instruction * > & Ignored ) { uint64_t AccessSize = MemoryLocation :: UnknownSize ; if ( const SCEVConstant * BECst = dyn_cast < SCEVConstant > ( BECount ) ) AccessSize = ( BECst -> getValue ( ) -> getZExtValue ( ) + <NUM_LIT> ) * StoreSize ; MemoryLocation StoreLoc ( Ptr , AccessSize ) ; for ( auto * B : L -> blocks ( ) ) for ( auto & I : * B ) if ( Ignored . count ( & I ) == <NUM_LIT> && isModOrRefSet ( intersectModRef ( AA . getModRefInfo ( & I , StoreLoc )


<s> unsigned DestBW = DestTy -> getBitWidth ( ) ; if ( PHINode * P = dyn_cast < PHINode > ( In ) ) { unsigned N = P -> getNumIncomingValues ( ) ; for ( unsigned i = <NUM_LIT> ; i != N ; ++ i ) { BasicBlock * InB = P -> getIncomingBlock ( i ) ; if ( InB == LoopB ) continue ; Value * InV = P -> getIncomingValue ( i ) ; IntegerType * Ty = cast < IntegerType > ( InV -> getType ( ) ) ; if ( Ty != P -> getType ( ) ) { assert ( Ty -> getBitWidth ( ) < DestBW ) ; InV = IRBuilder < > ( InB -> getTerminator ( ) ) . CreateZExt ( InV , DestTy ) ; P -> setIncomingValue ( i , InV ) ; } } } else if ( ZExtInst * Z = dyn_cast < ZExtInst > ( In ) ) { Value * Op = Z -> getOperand ( <NUM_LIT> ) ; if ( Op -> getType ( ) == Z -> getType ( ) ) Z -> replaceAllUsesWith ( Op ) ; Z -> eraseFromParent ( ) ; return ; } if ( TruncInst * T = dyn_cast < TruncInst > ( In ) ) { IntegerType * TruncTy = cast < IntegerType > ( OrigTy ) ; Value * Mask = ConstantInt :: get ( DestTy , ( <NUM_LIT> << TruncTy -> getBitWidth ( ) ) - <NUM_LIT> ) ; Value * And = IRBuilder < > ( In ) . CreateAnd ( T -> getOperand ( <NUM_LIT> ) , Mask ) ; T -> replaceAllUsesWith ( And


<s> } for ( Instruction & In : * LoopB ) if ( ! In . isTerminator ( ) && ! isPromotableTo ( & In , DestTy ) ) return false ; std :: vector < Instruction * > LoopIns ; std :: transform ( LoopB -> begin ( ) , LoopB -> end ( ) , std :: back_inserter ( LoopIns ) , [ ] ( Instruction & In ) { return & In ; } ) ; for ( Instruction * In : LoopIns ) promoteTo ( In , DestTy , LoopB ) ; Instruction * EndI = ExitB -> getFirstNonPHI ( ) ; BasicBlock :: iterator End = EndI ? EndI -> getIterator ( ) : ExitB -> end ( ) ; for ( auto I = ExitB -> begin ( ) ; I != End ; ++ I ) { PHINode * P = dyn_cast < PHINode > ( I ) ; if ( ! P ) break ; Type * Ty0 = P -> getIncomingValue ( <NUM_LIT> ) -> getType ( ) ; Type * PTy = P -> getType ( ) ; if ( PTy != Ty0 ) { assert ( Ty0 == DestTy


<s> if ( P . getNumIncomingValues ( ) != <NUM_LIT> ) return false ; assert ( P . getIncomingBlock ( <NUM_LIT> ) == LoopB ) ; IntegerType * T = dyn_cast < IntegerType > ( P . getType ( ) ) ; if ( ! T || T -> getBitWidth ( ) > DestBW ) return false ; } for ( Instruction & In : * LoopB ) if ( ! In . isTerminator ( ) && ! isPromotableTo ( & In , DestTy ) ) return false ; std :: vector < Instruction * > LoopIns ; std :: transform ( LoopB -> begin ( ) , LoopB -> end ( ) , std :: back_inserter ( LoopIns ) , [ ] ( Instruction & In ) { return & In ; } ) ; for ( Instruction * In : LoopIns ) promoteTo ( In , DestTy , LoopB ) ; Instruction * EndI = ExitB -> getFirstNonPHI ( ) ; BasicBlock :: iterator End = EndI ? EndI -> getIterator ( ) : ExitB -> end ( ) ; for ( auto I = ExitB -> begin (


<s> for ( auto & In : * B ) { if ( isa < BranchInst > ( In ) || isa < DbgInfoIntrinsic > ( In ) ) continue ; if ( ! Worklist . count ( & In ) && In . mayHaveSideEffects ( ) ) return false ; for ( auto K : In . users ( ) ) { Instruction * UseI = dyn_cast < Instruction > ( K ) ; if ( ! UseI ) continue ; BasicBlock * UseB = UseI -> getParent ( ) ; if ( LF -> getLoopFor ( UseB ) != L )


<s> SmallSet < BasicBlock * , <NUM_LIT> > LoopBlocks ; for ( auto * B : L -> blocks ( ) ) LoopBlocks . insert ( B ) ; SetVector < Instruction * > Worklist ( Insts . begin ( ) , Insts . end ( ) ) ; for ( unsigned i = <NUM_LIT> ; i < Worklist . size ( ) ; ++ i ) { Instruction * In = Worklist [ i ] ; for ( auto I = In -> op_begin ( ) , E = In -> op_end ( ) ; I != E ; ++ I ) { Instruction * OpI = dyn_cast < Instruction > ( I ) ; if ( ! OpI ) continue ; BasicBlock * PB = OpI ->


<s> Pass * llvm :: create<ISA_LIT>LoopIdiomPass ( ) { return new <ISA_LIT>LoopIdiomRecognizeLegacyPass ( )


<s> Module * M = At -> getParent ( ) -> getParent ( ) -> getParent ( ) ; Function * PMF = <STR_LIT> ( M , <STR_LIT> ) ; Value * P = PV . P , * Q = PV . Q , * P0 = P ; unsigned IC = PV . IterCount ; if ( PV . M != nullptr ) P0 = P = B . CreateXor ( P , PV . M ) ; auto * BMI = ConstantInt :: get ( P -> getType ( ) , APInt :: getLowBitsSet ( <NUM_LIT> , IC ) ) ; if ( PV . IterCount != <NUM_LIT> ) P = B . CreateAnd ( P , BMI ) ; if ( PV . Inv ) { auto * QI = dyn_cast < ConstantInt > ( PV . Q ) ; assert ( QI && QI -> getBitWidth ( ) <= <NUM_LIT> ) ; unsigned M = ( <NUM_LIT> << PV . IterCount )


<s> AU . addRequiredID ( LCSSAID ) ; AU . addRequired < AAResultsWrapperPass > ( ) ; AU . addRequired < ScalarEvolutionWrapperPass > (


<s> AU . addRequired < AAResultsWrapperPass > ( ) ; AU . addRequired < ScalarEvolutionWrapperPass > ( ) ; AU . addRequired < DominatorTreeWrapperPass > ( ) ; AU . addRequired < TargetLibraryInfoWrapperPass >


<s> BasicBlock * PB = ( * PI == BB ) ? * std :: next ( PI ) : * PI ; for ( auto I = BB -> begin ( ) , E = BB -> end ( ) ; I != E && isa < PHINode > ( I ) ; ++ I ) { auto * PN = cast < PHINode > ( I ) ; Value * InitV = PN -> getIncomingValueForBlock ( PB ) ; if ( ! isa < ConstantInt > ( InitV ) || ! cast < ConstantInt > ( InitV ) -> isZero ( ) ) continue ; Value * IterV = PN -> getIncomingValueForBlock ( BB ) ; auto * BO = dyn_cast < BinaryOperator > ( IterV ) ; if ( ! BO ) continue ; if ( BO -> getOpcode ( ) != Instruction :: Add ) continue ; Value * IncV = nullptr ; if ( BO -> getOperand ( <NUM_LIT> ) == PN ) IncV = BO -> getOperand ( <NUM_LIT> ) ; else if ( BO -> getOperand ( <NUM_LIT> ) == PN ) IncV = BO -> getOperand ( <NUM_LIT> ) ; if ( IncV == nullptr ) continue ; if ( auto * T = dyn_cast < ConstantInt > ( IncV ) ) if ( T

-> getZExtValue ( ) == <NUM_LIT> ) return PN ;


<s> LocationSize AccessSize = LocationSize :: afterPointer ( ) ; if ( const SCEVConstant * BECst = dyn_cast < SCEVConstant > ( BECount ) ) AccessSize = LocationSize :: precise ( ( BECst -> getValue ( ) -> getZExtValue ( ) + <NUM_LIT> ) * StoreSize ) ; MemoryLocation StoreLoc ( Ptr , AccessSize ) ; for ( auto * B : L -> blocks ( ) ) for ( auto & I : * B ) if ( Ignored . count ( & I ) == <NUM_LIT> && isModOrRefSet ( intersectModRef ( AA . getModRefInfo ( & I , StoreLoc ) , Access ) ) ) return true ; return false


<s> if ( ! S . count ( V ) ) { Q . push_back ( V ) ; S . insert ( V )


<s> PreservedAnalyses <ISA_LIT>LoopIdiomRecognitionPass :: run ( Loop & L , LoopAnalysisManager & AM , LoopStandardAnalysisResults & AR , LPMUpdater & U ) { return <ISA_LIT>LoopIdiomRecognize ( & AR . AA , & AR . DT , & AR . LI , & AR . TLI , & AR . SE ) . run (

& L ) ? getLoopPassPreservedAnalyses ( ) : PreservedAnalyses :: all ( ) ;


<s> PreservedAnalyses <ISA_LIT>LoopIdiomRecognitionPass :: run ( Loop & L , LoopAnalysisManager & AM ,

LoopStandardAnalysisResults & AR , LPMUpdater & U ) {


<s> auto * AA = & getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; auto * DT = & getAnalysis < DominatorTreeWrapperPass > ( ) . getDomTree ( ) ; auto * LF = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo (


<s> auto * TLI = & getAnalysis < TargetLibraryInfoWrapperPass > ( ) . getTLI ( * L -> getHeader ( ) -> getParent ( ) ) ; auto * SE = & getAnalysis < ScalarEvolutionWrapperPass > ( ) . getSE ( ) ; return <ISA_LIT>LoopIdiomRecognize ( AA , DT , LF , TLI , SE ) . run


<s> bool PolynomialMultiplyRecognize :: isPromotableTo ( Value * Val , IntegerType * DestTy ) { IntegerType * T = dyn_cast < IntegerType > ( Val -> getType ( ) ) ; if ( ! T || T -> getBitWidth ( ) > DestTy -> getBitWidth ( ) ) return false ; if ( T -> getBitWidth ( ) == DestTy -> getBitWidth ( ) ) return true ; Instruction * In = dyn_cast < Instruction > ( Val ) ; if ( ! In ) return true ; switch ( In -> getOpcode ( ) ) { case Instruction :: PHI : case Instruction :: ZExt : case Instruction :: And : case Instruction :: Or : case Instruction :: Xor : case Instruction :: LShr : case Instruction ::


<s> if ( ! P ) break ; if ( P -> getNumIncomingValues ( ) != <NUM_LIT> ) return false ; assert ( P -> getIncomingBlock ( <NUM_LIT> ) == LoopB ) ; IntegerType * T = dyn_cast < IntegerType > ( P -> getType ( ) ) ; if ( ! T || T -> getBitWidth ( ) > DestBW ) return false ; } for ( Instruction & In : * LoopB ) if ( ! In . isTerminator ( ) && ! isPromotableTo ( & In , DestTy ) ) return false ; std :: vector < Instruction * > LoopIns ; std :: transform ( LoopB -> begin ( ) , LoopB -> end ( ) , std :: back_inserter ( LoopIns ) , [ ] ( Instruction & In ) { return & In ; } ) ; for ( Instruction * In : LoopIns ) promoteTo ( In , DestTy , LoopB ) ; Instruction * EndI = ExitB -> getFirstNonPHI ( ) ; BasicBlock :: iterator End = EndI ? EndI -> getIterator ( )


<s> if ( const SCEVConstant * BECst = dyn_cast < SCEVConstant > ( BECount ) ) AccessSize = ( BECst -> getValue ( ) -> getZExtValue ( ) + <NUM_LIT> ) * StoreSize ; MemoryLocation StoreLoc ( Ptr , AccessSize ) ; for ( auto * B : L -> blocks ( ) ) for ( auto & I : * B ) if ( Ignored . count ( & I ) == <NUM_LIT> && isModOrRefSet ( intersectModRef ( AA . getModRefInfo ( & I

, StoreLoc ) , Access ) ) ) return true ;


<s> if ( FoundScan ) break ; } assert ( FoundScan ) ; DEBUG ( { StringRef PP = ( PV . M ? <STR_LIT> : <STR_LIT> ) ; if ( ! PV . Inv ) dbgs ( ) << <STR_LIT> << PP << <STR_LIT> ; else dbgs ( ) << <STR_LIT> << PP << <STR_LIT> << PP << <STR_LIT> ; dbgs ( ) << <STR_LIT> << * PV . Res << <STR_LIT> << * PV . P << <STR_LIT> ; if ( PV . M ) dbgs ( ) << <STR_LIT> << * PV . M << <STR_LIT> ; dbgs ( ) << <STR_LIT> << * PV . Q << <STR_LIT> ; dbgs ( ) << <STR_LIT> << PV . IterCount << <STR_LIT> ; } ) ; BasicBlock :: iterator At ( EntryB -> getTerminator ( ) ) ; Value * PM = generate ( At , PV ) ; if ( PM == nullptr ) return false ; if ( PM -> getType ( ) != PV . Res -> getType ( ) ) PM = IRBuilder < > ( & * At ) . CreateIntCast ( PM , PV . Res -> getType ( ) , false ) ; PV . Res -> replaceAllUsesWith ( PM ) ; PV . Res -> eraseFromParent ( ) ; return true


<s> unsigned Count = <NUM_LIT> ; const unsigned Limit = <NUM_LIT> ; while ( ! Q . empty ( ) ) { if ( Count ++ >= Limit ) break ; Instruction * U = dyn_cast < Instruction > ( Q . pop_front_val ( ) ) ; if ( ! U || U -> getParent ( ) || ! C . Used . count ( U ) ) continue ; bool Changed = false ; for ( Rule & R :


<s> if ( Count ++ >= Limit ) break ; Instruction * U = dyn_cast < Instruction > ( Q . pop_front_val ( ) ) ; if ( ! U || U -> getParent ( ) || ! C . Used . count ( U ) ) continue ; bool Changed = false ; for ( Rule & R : Rules ) { Value * W = R ( U , C . Ctx ) ; if ( ! W ) continue ; Changed = true ; C . record ( W ) ; C . replace ( U , W ) ; Q . push_back ( C . Root ) ; break ; } if ( ! Changed ) { for ( Value * Op : U -> operands ( ) ) Q . push_back ( Op ) ; } } assert ( Count < Limit && <STR_LIT> ) ; return C .


<s> for ( auto * B : L -> blocks ( ) ) for ( auto & I : * B ) if ( Ignored . count ( & I ) == <NUM_LIT> && isModOrRefSet ( intersectModRef ( AA . getModRefInfo ( & I , StoreLoc ) , Access ) ) ) return true ; return false


<s> } if ( ! PV . Left ) { if ( ! promoteTypes ( LoopB , ExitB ) ) return false ; if ( ! convertShiftsToLeft ( LoopB , ExitB , IterCount ) ) return false ; cleanupLoopBody ( LoopB ) ; } bool FoundScan = false ; for ( Instruction & In : * LoopB ) { SelectInst * SelI = dyn_cast < SelectInst > ( & In ) ; if ( ! SelI ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << * SelI << '\n' ) ; FoundScan = scanSelect ( SelI , LoopB , EntryB , CIV , PV , false ) ; if ( FoundScan ) break ; } assert ( FoundScan ) ; DEBUG ( { StringRef PP = ( PV . M ? <STR_LIT> : <STR_LIT> ) ; if ( ! PV . Inv ) dbgs ( ) << <STR_LIT> << PP << <STR_LIT> ; else dbgs ( ) << <STR_LIT> << PP << <STR_LIT> << PP << <STR_LIT> ; dbgs ( ) << <STR_LIT> << * PV . Res << <STR_LIT> << * PV . P << <STR_LIT> ; if ( PV . M ) dbgs ( ) << <STR_LIT> << * PV . M << <STR_LIT> ; dbgs ( ) << <STR_LIT> << * PV . Q << <STR_LIT> ; dbgs ( ) << <STR_LIT> << PV . IterCount << <STR_LIT> ; } ) ; BasicBlock :: iterator At ( EntryB -> getTerminator ( ) ) ; Value * PM = generate ( At , PV ) ; if ( PM == nullptr ) return false


<s> if ( ! SelI ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << * SelI << '\n' ) ; FoundScan = scanSelect ( SelI , LoopB , EntryB , CIV , PV , false ) ; if ( FoundScan ) break ; } assert ( FoundScan ) ; DEBUG ( { StringRef PP = ( PV . M ? <STR_LIT> : <STR_LIT> ) ; if ( ! PV . Inv ) dbgs ( ) << <STR_LIT> << PP << <STR_LIT> ; else dbgs ( ) << <STR_LIT> << PP << <STR_LIT> << PP << <STR_LIT> ; dbgs ( ) << <STR_LIT> << * PV . Res << <STR_LIT> << * PV . P << <STR_LIT> ; if ( PV . M ) dbgs ( ) << <STR_LIT> << * PV . M << <STR_LIT> ; dbgs ( ) << <STR_LIT> << * PV . Q << <STR_LIT> ; dbgs ( ) << <STR_LIT> << PV . IterCount << <STR_LIT> ; } ) ; BasicBlock :: iterator At ( EntryB -> getTerminator ( ) ) ; Value * PM = generate ( At , PV ) ; if ( PM == nullptr ) return false ; if ( PM -> getType ( ) != PV . Res -> getType ( ) ) PM = IRBuilder < > (

& * At ) . CreateIntCast ( PM , PV . Res -> getType ( ) , false ) ;


<s> while ( ! Q . empty ( ) ) { if ( Count ++ >= Limit ) break ; Instruction * U = dyn_cast < Instruction > ( Q . pop_front_val ( ) ) ; if ( ! U || U -> getParent ( ) || ! C . Used . count ( U ) ) continue ; bool Changed = false ; for ( Rule & R : Rules ) { Value * W = R ( U , C . Ctx ) ; if ( ! W ) continue ; Changed = true ; C . record ( W ) ; C . replace ( U , W ) ; Q . push_back ( C . Root ) ; break ; } if ( ! Changed ) { for ( Value * Op : U -> operands ( )


<s> Value * Simplifier :: simplify ( Context & C ) { WorkListType Q ; Q . push_back ( C . Root ) ; unsigned Count = <NUM_LIT> ; const unsigned Limit = SimplifyLimit ; while ( ! Q . empty ( ) ) { if ( Count ++ >= Limit ) break ; Instruction * U = dyn_cast < Instruction > ( Q . pop_front_val ( ) ) ; if ( ! U || U -> getParent ( ) || ! C . Used . count ( U ) ) continue ; bool Changed = false ; for ( Rule & R : Rules ) { Value * W = R ( U , C . Ctx ) ; if ( ! W ) continue ; Changed = true ; C . record ( W ) ; C . replace ( U


<s> void addRule ( StringRef N , const Rule :: FuncType & F ) { Rules . push_back ( Rule ( N


<s> for ( unsigned J = <NUM_LIT> ; J < I ; ++ J ) First . insert ( Cycle [ J ] ) ; ValueSeq & Second = IsE ? Early : Late ; Second . insert ( Cycle [ I ] ) ; for ( ++ I ; I < N ; ++ I ) { Value * V = Cycle [ I ]


<s> case Instruction :: Xor : case Instruction :: LShr : case Instruction :: Shl : case Instruction :: Select : case Instruction ::


<s> Context ( Instruction * Exp ) : Ctx ( Exp -> getParent ( ) -> getParent ( ) -> getContext ( ) ) { initialize (


<s> SetVector < Instruction * > Worklist ( Insts . begin ( ) , Insts . end ( ) ) ; for ( unsigned i = <NUM_LIT> ; i < Worklist . size ( ) ; ++ i ) { Instruction * In = Worklist [ i ] ; for ( auto I = In -> op_begin ( ) , E = In -> op_end ( ) ; I != E ; ++ I ) { Instruction * OpI = dyn_cast < Instruction > ( I ) ; if ( ! OpI ) continue ; BasicBlock * PB = OpI -> getParent ( ) ; if ( ! LoopBlocks . count ( PB


<s> return new <ISA_LIT>LoopIdiomRecognize


<s> bool IsPhi = isa < PHINode > ( I ) ; if ( IsPhi && HadPhi ) return false ; HadPhi |= IsPhi ; if ( Cycle . count ( I ) ) return false ; Cycle . insert ( I ) ; if ( findCycle ( I , In , Cycle ) ) break ; Cycle . remove ( I ) ; } return ! Cycle . empty


<s> AU . addRequired < ScalarEvolutionWrapperPass > ( ) ; AU . addRequired < DominatorTreeWrapperPass > ( ) ; AU . addRequired < TargetLibraryInfoWrapperPass > (


<s> C [ <NUM_LIT> ] = <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) { unsigned T = <NUM_LIT> ; for ( unsigned j = <NUM_LIT> ; j < i ; ++ j ) T = T ^ ( C [ j ] & Q [ i - j ] ) ; C [ i ] = T ; } unsigned QV = <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) if ( C [ i ] ) QV |= ( <NUM_LIT> << i ) ; return


<s> int <ISA_LIT>LoopIdiomRecognize :: getSCEVStride ( const SCEVAddRecExpr * S ) { if ( const SCEVConstant * SC = dyn_cast < SCEVConstant > ( S -> getOperand ( <NUM_LIT> ) ) ) return SC -> getAPInt ( ) . getSExtValue ( ) ; return <NUM_LIT>


<s> int <ISA_LIT>LoopIdiomRecognize :: getSCEVStride ( const SCEVAddRecExpr * S ) { if ( const SCEVConstant * SC = dyn_cast < SCEVConstant > ( S -> getOperand ( <NUM_LIT> ) ) ) return SC ->


<s> if ( const auto SI = dyn_cast < const ConstantInt > ( I -> getOperand ( <NUM_LIT> ) ) ) return SI -> getZExtValue ( ) > <NUM_LIT> ; return false ; case Instruction :: Or : case Instruction :: Xor : return hasZeroSignBit ( I -> getOperand ( <NUM_LIT> ) ) && hasZeroSignBit ( I -> getOperand ( <NUM_LIT> ) ) ; case Instruction :: And : return hasZeroSignBit ( I -> getOperand ( <NUM_LIT> ) ) || hasZeroSignBit ( I -> getOperand ( <NUM_LIT> ) ) ; } return false


<s> if ( ! T ) return false ; KnownBits Known ( T -> getBitWidth ( ) ) ; computeKnownBits ( V , Known ,


<s> if ( ! ( SI -> isVolatile ( ) && <ISA_LIT>VolatileMemcpy ) && ! SI -> isSimple ( ) ) return false ; Value * StoredVal = SI -> getValueOperand ( ) ; Value * StorePtr = SI -> getPointerOperand ( ) ; uint64_t SizeInBits = DL -> getTypeSizeInBits ( StoredVal -> getType ( ) ) ; if ( ( SizeInBits & <NUM_LIT> ) || ( SizeInBits >> <NUM_LIT> ) != <NUM_LIT> ) return false ; auto * StoreEv = dyn_cast < SCEVAddRecExpr > ( SE -> getSCEV ( StorePtr ) ) ; if ( ! StoreEv || StoreEv -> getLoop ( ) != CurLoop || ! StoreEv -> isAffine ( ) ) return false ; int Stride = getSCEVStride ( StoreEv ) ; if ( Stride == <NUM_LIT> ) return false ; unsigned StoreSize = DL -> getTypeStoreSize ( SI -> getValueOperand ( ) -> getType ( ) ) ; if ( StoreSize != unsigned ( std :: abs ( Stride ) ) ) return false ; LoadInst * LI = dyn_cast < LoadInst > ( SI -> getValueOperand ( ) ) ; if ( ! LI || ! LI -> isSimple ( ) ) return false ; Value * LoadPtr = LI -> getPointerOperand ( ) ; auto * LoadEv = dyn_cast < SCEVAddRecExpr > ( SE -> getSCEV ( LoadPtr ) ) ; if ( ! LoadEv || LoadEv -> getLoop ( ) != CurLoop || ! LoadEv -> isAffine


<s> bool PolynomialMultiplyRecognize :: isOperandShifted ( Instruction * I , Value * Op ) { unsigned Opc = I -> getOpcode


<s> unsigned Opc = I -> getOpcode ( ) ; if ( Opc == Instruction :: Shl || Opc == Instruction :: LShr ) return Op != I -> getOperand


<s> if ( ! T || T -> getBitWidth ( ) > DestTy -> getBitWidth ( ) ) return false ; if ( T -> getBitWidth ( ) == DestTy -> getBitWidth ( ) ) return true ; Instruction * In = dyn_cast < Instruction > ( Val ) ; if ( ! In ) return true ; switch ( In -> getOpcode ( ) ) { case Instruction :: PHI : case Instruction :: ZExt : case Instruction


<s> Value * T = nullptr ; if ( match ( ShouldXoredV , m_Xor ( m_Value ( Y ) , m_Value ( Z ) ) ) ) { if ( ShouldSameV == Y ) T = Z ; else if ( ShouldSameV == Z ) T = Y ; else return false ; R = ShouldSameV ; } else if ( match ( ShouldSameV , m_Zero ( ) ) ) { if ( ! SelI -> hasOneUse ( ) ) return false ; T = ShouldXoredV ; Value * U = * SelI -> user_begin ( ) ; if ( ! match ( U , m_Xor ( m_Specific ( SelI ) , m_Value ( R ) ) ) && ! match ( U , m_Xor ( m_Value ( R ) , m_Specific ( SelI ) ) ) ) return false ; } else return false ; if ( ! match ( T , m_Shl ( m_Value ( Q ) , m_Specific ( CIV ) ) ) && ! match ( T , m_Shl ( m_ZExt ( m_Value ( Q ) ) , m_ZExt ( m_Specific ( CIV


<s> if ( match ( CondV , m_ICmp ( P , m_Value ( C ) , m_Zero ( ) ) ) || match ( CondV , m_ICmp ( P , m_Zero ( ) , m_Value ( C ) ) ) ) { if ( P != CmpInst :: ICMP_EQ && P != CmpInst :: ICMP_NE ) return false ; TrueIfZero = ( P == CmpInst :: ICMP_EQ ) ; } else if ( match ( CondV , m_ICmp ( P , m_Value ( C ) , m_One ( ) ) ) || match ( CondV , m_ICmp ( P , m_One ( ) , m_Value ( C ) ) ) ) { if ( P != CmpInst :: ICMP_EQ && P != CmpInst :: ICMP_NE ) return false ; TrueIfZero = ( P == CmpInst :: ICMP_NE ) ; } else return false ; Value * X = nullptr ; if ( ! match ( C , m_And ( m_Value ( X ) , m_One ( ) ) ) && ! match ( C , m_And ( m_One ( ) , m_Value ( X ) ) ) ) return false ; Value * R = nullptr , * Q = nullptr ; if ( TrueIfZero ) { if ( ! match ( TrueV , m_LShr ( m_Value ( R ) , m_One ( ) ) ) ) return false ; if ( ! match ( FalseV , m_Xor ( m_Specific ( TrueV ) , m_Value ( Q ) ) ) && ! match ( FalseV , m_Xor ( m_Value ( Q ) , m_Specific ( TrueV ) ) ) ) return false ; } else


<s> if ( TrueIfZero ) { if ( ! match ( TrueV , m_LShr ( m_Value ( R ) , m_One ( ) ) ) ) return false ; if ( ! match ( FalseV , m_Xor ( m_Specific ( TrueV ) , m_Value ( Q ) ) ) && ! match ( FalseV , m_Xor ( m_Value ( Q ) , m_Specific ( TrueV ) ) ) ) return false ; } else { if ( ! match ( FalseV , m_LShr ( m_Value ( R ) , m_One ( ) ) ) ) return false ; if ( ! match ( TrueV , m_Xor ( m_Specific ( FalseV ) , m_Value ( Q ) ) ) && ! match ( TrueV , m_Xor ( m_Value ( Q ) , m_Specific ( FalseV ) ) ) ) return false ; } PV . X = X ; PV . Q = Q ; PV . R


<s> MemoryLocation StoreLoc ( Ptr , AccessSize ) ; for ( auto * B : L -> blocks ( ) ) for ( auto & I : * B ) if ( Ignored . count ( & I ) ==

<NUM_LIT> && isModOrRefSet ( intersectModRef ( AA . getModRefInfo ( & I , StoreLoc ) , Access ) ) ) return true ;


<s> MemoryLocation StoreLoc ( Ptr , AccessSize ) ; for ( auto * B : L -> blocks ( ) ) for ( auto & I : * B ) if ( Ignored . count ( & I )

== <NUM_LIT> && isModOrRefSet ( intersectModRef ( AA . getModRefInfo ( & I , StoreLoc ) , Access ) ) ) return true ;


<s> Value * V = Q . front ( ) ; Q . pop_front ( ) ; S . erase ( V ) ; return


<s> Value * V = Q . front ( ) ; Q . pop_front (


<s> Z -> eraseFromParent ( ) ; return ; } if ( TruncInst * T = dyn_cast < TruncInst > ( In ) ) { IntegerType * TruncTy = cast < IntegerType > ( OrigTy ) ; Value * Mask = ConstantInt :: get ( DestTy , ( <NUM_LIT> << TruncTy -> getBitWidth ( ) ) - <NUM_LIT> ) ; Value * And = IRBuilder < > ( In ) . CreateAnd ( T -> getOperand ( <NUM_LIT> ) , Mask ) ; T -> replaceAllUsesWith ( And ) ; T -> eraseFromParent ( ) ; return ; } for ( unsigned i = <NUM_LIT> , n = In -> getNumOperands ( ) ; i != n ; ++ i ) { if ( ConstantInt * CI = dyn_cast < ConstantInt > ( In -> getOperand ( i ) ) ) if ( CI -> getType ( ) -> getBitWidth ( ) < DestBW ) In -> setOperand ( i , ConstantInt

:: get ( DestTy , CI -> getZExtValue ( ) ) ) ;


<s> } } else if ( ZExtInst * Z = dyn_cast < ZExtInst > ( In ) ) { Value * Op = Z -> getOperand ( <NUM_LIT> ) ; if ( Op -> getType ( ) == Z -> getType ( ) ) Z -> replaceAllUsesWith ( Op ) ; Z -> eraseFromParent ( ) ; return ; } if ( TruncInst * T = dyn_cast < TruncInst > ( In ) ) { IntegerType * TruncTy = cast < IntegerType > ( OrigTy ) ; Value * Mask = ConstantInt :: get ( DestTy , ( <NUM_LIT> << TruncTy -> getBitWidth ( ) ) - <NUM_LIT> ) ; Value * And = IRBuilder < > ( In ) . CreateAnd ( T -> getOperand ( <NUM_LIT> ) , Mask ) ; T -> replaceAllUsesWith ( And ) ; T -> eraseFromParent ( ) ; return ; } for ( unsigned i = <NUM_LIT> , n = In -> getNumOperands ( ) ; i != n ; ++ i ) { if ( ConstantInt * CI = dyn_cast < ConstantInt > ( In -> getOperand ( i ) ) ) if ( CI -> getType ( ) -> getBitWidth ( ) < DestBW ) In ->

setOperand ( i , ConstantInt :: get ( DestTy , CI -> getZExtValue ( ) ) ) ;


<s> if ( ! ExitB || ( ExitB -> getSinglePredecessor ( ) != LoopB ) ) return false ; IntegerType * DestTy = getPmpyType ( ) ; unsigned DestBW = DestTy -> getBitWidth ( ) ; for ( PHINode & P : ExitB -> phis ( ) ) { if ( P . getNumIncomingValues ( ) != <NUM_LIT> ) return false ; assert ( P . getIncomingBlock ( <NUM_LIT> ) == LoopB ) ; IntegerType * T = dyn_cast < IntegerType > ( P . getType ( ) ) ; if ( ! T || T -> getBitWidth ( ) > DestBW ) return false ; } for ( Instruction & In : * LoopB ) if ( ! In . isTerminator ( ) && ! isPromotableTo ( & In , DestTy ) ) return false ; std :: vector < Instruction * > LoopIns ; std :: transform ( LoopB -> begin ( ) , LoopB -> end ( ) , std :: back_inserter ( LoopIns ) , [ ] ( Instruction & In ) { return & In ; } ) ; for ( Instruction * In : LoopIns ) if ( ! In -> isTerminator ( ) ) promoteTo ( In , DestTy , LoopB ) ; Instruction * EndI = ExitB -> getFirstNonPHI ( ) ; BasicBlock :: iterator End = EndI ? EndI -> getIterator ( ) : ExitB -> end ( ) ; for ( auto I = ExitB -> begin ( ) ; I != End ; ++ I ) { PHINode * P = dyn_cast < PHINode > ( I ) ; if ( ! P ) break ; Type * Ty0 = P -> getIncomingValue ( <NUM_LIT> ) -> getType ( ) ; Type * PTy = P -> getType ( ) ; if ( PTy != Ty0 ) { assert ( Ty0 == DestTy ) ; P -> mutateType ( Ty0 ) ; Value * T = IRBuilder < > ( ExitB , End ) . CreateTrunc ( P , PTy ) ; P -> mutateType ( PTy ) ; P -> replaceAllUsesWith ( T


<s> bool PolynomialMultiplyRecognize :: recognize ( ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * CurLoop << '\n' ) ; BasicBlock * LoopB = CurLoop -> getHeader ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * LoopB ) ; if ( LoopB != CurLoop -> getLoopLatch ( ) ) return false ; BasicBlock * ExitB = CurLoop -> getExitBlock ( ) ; if ( ExitB == nullptr ) return false ; BasicBlock * EntryB = CurLoop -> getLoopPreheader ( ) ; if ( EntryB == nullptr ) return false ; unsigned IterCount = <NUM_LIT> ; const SCEV * CT = SE . getBackedgeTakenCount ( CurLoop ) ; if ( isa < SCEVCouldNotCompute > ( CT ) ) return false ; if ( auto * CV = dyn_cast < SCEVConstant > ( CT ) ) IterCount = CV -> getValue ( ) -> getZExtValue ( ) + <NUM_LIT> ; Value * CIV = getCountIV ( LoopB ) ; ParsedValues PV ; Simplifier PreSimp ; PV . IterCount = IterCount ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * CIV << <STR_LIT> << IterCount << '\n' ) ; setupPreSimplifier ( PreSimp ) ; bool FoundPreScan = false ; auto FeedsPHI = [ LoopB ] ( const Value * V ) -> bool { for ( const Value * U : V -> users ( ) ) { if ( const auto * P = dyn_cast < const PHINode > ( U ) ) if ( P -> getParent ( ) == LoopB ) return true ; } return false ; } ; for ( Instruction & In : * LoopB ) { SelectInst * SI = dyn_cast < SelectInst > ( & In ) ; if ( ! SI || ! FeedsPHI ( SI ) ) continue ; Simplifier :: Context C ( SI ) ; Value * T = PreSimp . simplify ( C ) ; SelectInst * SelI = ( T && isa < SelectInst > ( T ) ) ? cast < SelectInst > ( T ) : SI ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << PE ( C , SelI ) << '\n' ) ; if ( scanSelect ( SelI , LoopB , EntryB , CIV , PV , true ) ) { FoundPreScan =


<s> bool <ISA_LIT>LoopIdiomRecognize :: runOnLoopBlock ( Loop * CurLoop , BasicBlock * BB , const SCEV * BECount , SmallVectorImpl < BasicBlock * > & ExitBlocks ) { auto DominatedByBB = [ this , BB ] ( BasicBlock * EB ) -> bool { return DT -> dominates ( BB , EB ) ; } ; if ( ! all_of ( ExitBlocks , DominatedByBB ) ) return false ; bool MadeChange = false ; SmallVector < StoreInst * , <NUM_LIT> > Stores ; collectStores ( CurLoop , BB , Stores ) ; for ( auto & SI : Stores ) MadeChange |= processCopyingStore


<s> if ( ! RPhi ) return false ; if ( SelI != RPhi -> getIncomingValueForBlock ( LoopB ) ) return false ; PV . Res = SelI ; if ( CurLoop -> isLoopInvariant ( PV . X ) ) { PV . P = PV . X ; PV . Inv = false ; } else { PV . Inv = true ; if ( PV . X != PV . R ) { Value * Var = nullptr , * Inv = nullptr , * X1 = nullptr , * X2 = nullptr ; if ( ! match ( PV . X , m_Xor ( m_Value ( X1 ) , m_Value ( X2 ) ) ) ) return false ; auto * I1 = dyn_cast < Instruction > ( X1 ) ; auto * I2 = dyn_cast < Instruction > ( X2 ) ; if ( ! I1 || I1 -> getParent ( ) != LoopB ) { Var = X2 ; Inv = X1 ; } else if ( ! I2 || I2 -> getParent ( ) != LoopB ) { Var = X1 ; Inv = X2 ; } else return false ; if ( Var != PV . R ) return false ; PV . M = Inv ; } Value * EntryP = RPhi -> getIncomingValueForBlock ( PrehB ) ; PV . P = EntryP ; } return true ; } if ( matchRightShift ( SelI , PV ) ) { if ( PV . Inv && ! isa < ConstantInt > ( PV . Q ) ) return false ; if ( PreScan ) return true ; return false ; } return false


<s> if ( ! Xor || ! C0 ) return nullptr ; if ( Xor -> getOpcode ( ) != Instruction :: Xor ) return nullptr ; Instruction * And0 = dyn_cast < Instruction > ( Xor -> getOperand ( <NUM_LIT> ) ) ; Instruction * And1 = dyn_cast < Instruction > ( Xor -> getOperand ( <NUM_LIT> ) ) ; if ( ! And0 || And0 -> getOpcode ( ) != Instruction :: And ) std :: swap ( And0 , And1 ) ; ConstantInt * C1 = dyn_cast < ConstantInt > ( And0 -> getOperand ( <NUM_LIT>


<s> IRBuilder < > B ( Ctx ) ; return B . CreateBinOp ( cast < BinaryOperator > ( T ) -> getOpcode ( ) , B . CreateZExt ( T -> getOperand ( <NUM_LIT> ) , I -> getType ( ) ) , B . CreateZExt ( T -> getOperand ( <NUM_LIT> ) , I -> getType ( ) ) ) ; } ) ; S . addRule ( <STR_LIT> , [ ] ( Instruction * I , LLVMContext & Ctx ) -> Value * { if ( I -> getOpcode ( ) != Instruction :: Xor ) return nullptr ; Instruction * And0 = dyn_cast < Instruction > ( I -> getOperand ( <NUM_LIT> ) ) ; Instruction * And1 = dyn_cast < Instruction > ( I -> getOperand ( <NUM_LIT> ) ) ; if ( ! And0 || ! And1 ) return nullptr ; if ( And0 -> getOpcode ( ) != Instruction :: And || And1 -> getOpcode ( ) != Instruction :: And ) return nullptr ; if ( And0 -> getOperand ( <NUM_LIT> ) != And1 -> getOperand ( <NUM_LIT> ) ) return nullptr ; IRBuilder < > B ( Ctx ) ; return B . CreateAnd ( B . CreateXor ( And0 -> getOperand ( <NUM_LIT> ) , And1 -> getOperand ( <NUM_LIT> ) ) , And0 -> getOperand ( <NUM_LIT> ) ) ; } ) ; S . addRule ( <STR_LIT> , [ ] ( Instruction * I , LLVMContext & Ctx ) -> Value * { BinaryOperator * BO = dyn_cast < BinaryOperator > ( I ) ; if ( ! BO ) return nullptr ; Instruction :: BinaryOps Op = BO -> getOpcode ( ) ; if ( SelectInst * Sel = dyn_cast < SelectInst > ( BO -> getOperand ( <NUM_LIT> ) ) ) { IRBuilder < > B ( Ctx ) ; Value * X = Sel -> getTrueValue ( ) , * Y = Sel -> getFalseValue ( ) ; Value * Z = BO -> getOperand ( <NUM_LIT> ) ; return B . CreateSelect ( Sel -> getCondition ( ) , B . CreateBinOp ( Op , X , Z ) , B . CreateBinOp ( Op , Y , Z ) ) ; } if ( SelectInst * Sel = dyn_cast < SelectInst > ( BO -> getOperand ( <NUM_LIT> ) ) ) { IRBuilder < > B ( Ctx ) ; Value * X = BO -> getOperand ( <NUM_LIT>


<s> Q . push_back ( C . Root ) ; unsigned Count = <NUM_LIT> ; const unsigned Limit = SimplifyLimit ; while ( ! Q . empty ( ) ) { if ( Count ++ >= Limit ) break ; Instruction * U = dyn_cast < Instruction > ( Q . pop_front_val ( ) ) ; if ( ! U || U -> getParent ( ) || ! C . Used . count ( U ) ) continue ; bool Changed = false ; for ( Rule & R : Rules ) { Value * W = R . Fn ( U , C . Ctx ) ; if ( ! W ) continue ; Changed = true ; C . record ( W ) ; C . replace (


<s> int getFirstNamedArgFrameIndex ( ) { return


<s> int getLastNamedArgFrameIndex ( ) { return


<s> unsigned getStackAlignBaseVReg ( ) const { return


<s> void setFirstNamedArgFrameIndex ( int v ) { FirstNamedArgFrameIndex =


<s> void setLastNamedArgFrameIndex ( int v ) { LastNamedArgFrameIndex =


<s> void setLastNamedArgFrameIndex ( int v )


<s> void setRegSavedAreaStartFrameIndex ( int v ) { RegSavedAreaStartFrameIndex = v


<s> void setStackAlignBasePhysReg ( unsigned R ) { StackAlignBasePhysReg = R


<s> void addAllocaAdjustInst ( MachineInstr * MI ) { AllocaAdjustInsts . push_back


<s> const std :: vector < MachineInstr * > & getAllocaAdjustInsts


<s> unsigned getSRetReturnReg ( ) const { return SRetReturnReg


<s> int getVarArgsFrameIndex ( ) { return VarArgsFrameIndex


<s> bool isEndPacket ( const MachineInstr * MI ) const { return ( PacketInfo . count ( MI ) && ( PacketInfo . find ( MI ) -> second &


<s> bool isStartPacket ( const MachineInstr * MI ) const { return ( PacketInfo . count ( MI ) && ( PacketInfo . find ( MI


<s> void setHasEHReturn ( bool H = true ) {


<s> void setHasEHReturn ( bool H = true ) { HasEHReturn =


<s> I != E ; ++ I ) { SUnit & Succ = * I -> getSUnit ( ) ; if ( ! Succ . isScheduled ) { if ( OnlyAvailableSucc && OnlyAvailableSucc != &


<s> DAG = static_cast < VLIWMachineScheduler * > ( dag ) ; SchedModel = DAG -> getSchedModel ( ) ; Top . init ( DAG , SchedModel ) ; Bot . init ( DAG , SchedModel ) ; const InstrItineraryData * Itin = DAG -> getSchedModel ( ) -> getInstrItineraries ( ) ; const TargetSubtargetInfo & STI = DAG -> MF . getSubtarget ( ) ; const TargetInstrInfo * TII = STI . getInstrInfo ( ) ; delete Top . HazardRec ; delete Bot . HazardRec ; Top . HazardRec = TII -> CreateTargetMIHazardRecognizer ( Itin , DAG ) ; Bot . HazardRec = TII -> CreateTargetMIHazardRecognizer ( Itin , DAG ) ; delete Top . ResourceModel ; delete Bot . ResourceModel ; Top . ResourceModel = new VLIWResourceModel ( STI , DAG -> getSchedModel (


<s> const TargetSubtargetInfo & STI = DAG -> MF . getSubtarget ( ) ; const TargetInstrInfo * TII = STI . getInstrInfo ( ) ; delete Top . HazardRec ; delete Bot . HazardRec ; Top . HazardRec = TII -> CreateTargetMIHazardRecognizer ( Itin , DAG ) ; Bot . HazardRec = TII -> CreateTargetMIHazardRecognizer ( Itin , DAG ) ; delete Top . ResourceModel ; delete Bot . ResourceModel ; Top . ResourceModel = new VLIWResourceModel ( STI , DAG -> getSchedModel ( ) ) ; Bot . ResourceModel = new VLIWResourceModel ( STI ,


<s> break ; } for ( unsigned i = <NUM_LIT> , e = Packet . size ( ) ; i != e ; ++ i ) { if ( Packet [ i ] -> Succs . size ( ) == <NUM_LIT> ) continue ; for ( SUnit :: const_succ_iterator I = Packet [ i ] -> Succs . begin ( ) , E = Packet [ i ] -> Succs .


<s> if ( SUnit * SU = Top . pickOnlyChoice ( ) ) { IsTopNode = true ; return SU ; } SchedCandidate BotCand ; CandResult BotResult = pickNodeFromQueue ( Bot . Available , DAG -> getBotRPTracker ( ) , BotCand ) ; assert ( BotResult != NoCand && <STR_LIT> ) ; if ( BotResult == SingleExcess || BotResult == SingleCritical ) { IsTopNode = false ; return BotCand . SU ; } SchedCandidate TopCand ; CandResult TopResult = pickNodeFromQueue ( Top . Available , DAG -> getTopRPTracker (


<s> for ( SUnit :: succ_iterator I = SU -> Preds . begin ( ) , E = SU -> Preds . end ( ) ; I != E ; ++ I ) { unsigned PredReadyCycle = I -> getSUnit (


<s> return false ; } if ( ! isResourceAvailable ( SU ) ) { ResourcesModel -> clearResources ( ) ; Packet . clear ( ) ; TotalPackets ++ ; startNewCycle = true ; } switch ( SU -> getInstr ( ) -> getOpcode ( ) ) { default : ResourcesModel -> reserveResources ( SU -> getInstr ( ) ) ; break ; case TargetOpcode :: EXTRACT_SUBREG : case TargetOpcode :: INSERT_SUBREG : case TargetOpcode :: SUBREG_TO_REG : case TargetOpcode :: REG_SEQUENCE : case TargetOpcode :: IMPLICIT_DEF : case TargetOpcode :: KILL : case TargetOpcode :: CFI_INSTRUCTION : case TargetOpcode :: EH_LABEL : case TargetOpcode :: COPY : case TargetOpcode :: INLINEASM


<s> TotalPackets ++ ; startNewCycle = true ; } switch ( SU -> getInstr ( ) -> getOpcode ( ) ) { default : ResourcesModel -> reserveResources ( SU -> getInstr ( ) ) ; break ; case TargetOpcode :: EXTRACT_SUBREG : case TargetOpcode :: INSERT_SUBREG : case TargetOpcode :: SUBREG_TO_REG : case TargetOpcode :: REG_SEQUENCE : case TargetOpcode :: IMPLICIT_DEF : case TargetOpcode :: KILL : case TargetOpcode :: CFI_INSTRUCTION : case TargetOpcode :: EH_LABEL : case TargetOpcode :: COPY : case TargetOpcode :: INLINEASM : break ; } Packet . push_back ( SU ) ; DEBUG ( dbgs ( ) << <STR_LIT> << TotalPackets << <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = Packet . size ( ) ; i != e ; ++ i ) { DEBUG ( dbgs ( ) << <STR_LIT> << i << <STR_LIT> ) ; DEBUG ( dbgs ( ) << Packet [ i ] -> NodeNum << <STR_LIT> ) ; DEBUG ( Packet [ i ] -> getInstr ( ) -> dump ( ) ) ; } if ( Packet . size ( )


<s> postprocessDAG ( ) ; SmallVector < SUnit * , <NUM_LIT> > TopRoots , BotRoots ; findRootsAndBiasEdges ( TopRoots , BotRoots ) ; SchedImpl -> initialize ( this ) ; DEBUG ( unsigned maxH = <NUM_LIT> ; for ( unsigned su = <NUM_LIT> , e = SUnits . size ( ) ; su != e ; ++ su ) if ( SUnits [ su ] . getHeight ( ) > maxH ) maxH = SUnits [ su ] . getHeight ( ) ; dbgs ( ) << <STR_LIT> << maxH << <STR_LIT> ; ) ; DEBUG ( unsigned maxD = <NUM_LIT> ; for ( unsigned su = <NUM_LIT> , e = SUnits . size ( ) ; su != e ; ++ su ) if ( SUnits [ su ] . getDepth ( ) > maxD ) maxD = SUnits [ su ] . getDepth ( ) ; dbgs ( ) << <STR_LIT> << maxD << <STR_LIT> ; ) ; DEBUG ( for ( unsigned su = <NUM_LIT> , e = SUnits . size ( ) ; su != e ; ++ su ) SUnits [ su ] . dumpAll ( this ) ) ; initQueues ( TopRoots , BotRoots ) ; bool IsTopNode = false ; while ( true ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; SUnit * SU = SchedImpl -> pickNode ( IsTopNode ) ; if ( !


<s> DEBUG ( dbgs ( ) << <STR_LIT> << BB -> getNumber ( ) << <STR_LIT> << BB -> getName ( ) << <STR_LIT> << BB -> getParent ( ) -> getFunction ( ) -> getName ( ) << <STR_LIT> << MLI -> getLoopDepth ( BB ) << <STR_LIT> ) ; buildDAGWithRegPressure ( ) ; postprocessDAG ( ) ; SmallVector < SUnit * , <NUM_LIT> > TopRoots , BotRoots ; findRootsAndBiasEdges ( TopRoots , BotRoots ) ; SchedImpl -> initialize ( this ) ; DEBUG ( unsigned maxH = <NUM_LIT> ; for ( unsigned su = <NUM_LIT> , e = SUnits . size ( ) ; su != e ; ++ su ) if ( SUnits [ su ] . getHeight ( ) > maxH ) maxH = SUnits [ su ] . getHeight ( ) ; dbgs ( ) << <STR_LIT> << maxH << <STR_LIT> ; ) ; DEBUG ( unsigned maxD = <NUM_LIT> ; for ( unsigned su = <NUM_LIT> , e = SUnits . size ( ) ; su != e ; ++ su ) if ( SUnits [ su ] . getDepth ( ) > maxD ) maxD = SUnits [ su ] . getDepth ( ) ; dbgs ( ) << <STR_LIT> << maxD << <STR_LIT> ; ) ; DEBUG ( for ( unsigned su = <NUM_LIT> , e = SUnits . size ( ) ; su != e ; ++ su ) SUnits [ su ] . dumpAll ( this ) ) ; initQueues ( TopRoots , BotRoots ) ; bool IsTopNode = false ; while ( true ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; SUnit * SU = SchedImpl -> pickNode ( IsTopNode ) ; if ( ! SU ) break ; if ( ! checkSchedLimit ( ) ) break ; scheduleMI ( SU , IsTopNode


<s> } else { ResCount += ( SU -> getDepth ( ) * ScaleTwo ) ; if ( Bot . ResourceModel -> isResourceAvailable ( SU ) ) ResCount <<= FactorOne ; } unsigned NumNodesBlocking = <NUM_LIT> ; if ( Q . getID ( ) == TopQID ) { for ( SUnit :: const_succ_iterator I = SU -> Succs . begin ( ) , E = SU -> Succs . end ( ) ; I != E ; ++ I ) if ( getSingleUnscheduledPred ( I -> getSUnit ( ) ) == SU ) ++ NumNodesBlocking ; } else


<s> void ConvergingVLIWScheduler :: traceCandidate ( const char * Label , const ReadyQueue & Q , SUnit * SU , PressureChange P ) { dbgs ( ) << Label << <STR_LIT> << Q . getName ( ) << <STR_LIT> ; if ( P . isValid ( ) ) dbgs ( ) << DAG -> TRI -> getRegPressureSetName ( P . getPSet ( ) ) << <STR_LIT> << P . getUnitInc ( ) << <STR_LIT> ; else dbgs ( ) << <STR_LIT> ; SU -> dump (


<s> dbgs ( ) << Label << <STR_LIT> << Q . getName ( ) << <STR_LIT> ; if ( P . isValid ( ) ) dbgs ( ) << DAG -> TRI -> getRegPressureSetName ( P . getPSet ( ) ) << <STR_LIT> << P . getUnitInc ( ) << <STR_LIT> ; else dbgs ( )


<s> void <ISA_LIT>CallMutation :: apply ( ScheduleDAGInstrs * DAG ) { SUnit * LastSequentialCall = nullptr ; unsigned VRegHoldingRet = <NUM_LIT> ; unsigned RetRegister ; SUnit * LastUseOfRet = nullptr ; auto & TRI = * DAG -> MF . getSubtarget ( ) . getRegisterInfo ( ) ; auto & HII = * DAG -> MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) ; for ( unsigned su = <NUM_LIT> , e = DAG -> SUnits . size ( ) ; su != e ; ++ su ) { if ( DAG -> SUnits [ su ] . getInstr ( ) -> isCall ( ) ) LastSequentialCall = & DAG -> SUnits [ su ] ; else if ( DAG -> SUnits [ su ] . getInstr ( ) -> isCompare ( ) && LastSequentialCall ) DAG -> SUnits [ su ] . addPred ( SDep ( LastSequentialCall , SDep :: Barrier ) ) ; else if ( SchedPredsCloser && LastSequentialCall && su > <NUM_LIT> && su < e - <NUM_LIT> && shouldTFRICallBind ( HII , DAG -> SUnits [ su ] , DAG -> SUnits [ su + <NUM_LIT> ] ) ) DAG -> SUnits [ su ] . addPred ( SDep ( & DAG -> SUnits [ su - <NUM_LIT> ] , SDep :: Barrier ) ) ; else if ( SchedRetvalOptimization ) { const MachineInstr * MI = DAG -> SUnits [ su ] . getInstr ( ) ; if ( MI -> isCopy ( ) && ( MI -> readsRegister ( <STR_LIT> , & TRI ) || MI -> readsRegister ( <STR_LIT> , & TRI ) ) ) { VRegHoldingRet = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; RetRegister = MI -> getOperand ( <NUM_LIT> ) . getReg


<s> dbgstr << <STR_LIT> << std :: setw ( <NUM_LIT>


<s> case TargetOpcode :: INSERT_SUBREG : case TargetOpcode :: SUBREG_TO_REG : case TargetOpcode :: REG_SEQUENCE : case TargetOpcode :: IMPLICIT_DEF : case TargetOpcode :: COPY : case TargetOpcode :: INLINEASM : break ; } MachineFunction & MF = * SU -> getInstr ( ) -> getParent ( ) -> getParent ( ) ; auto & QII = * MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) ; for ( unsigned i = <NUM_LIT> , e = Packet . size ( ) ; i != e ; ++ i ) { if ( Packet [ i ] -> Succs . size ( ) == <NUM_LIT> ) continue ; if ( QII . mayBeCurLoad ( * Packet [ i ] -> getInstr ( ) ) ) continue ; for ( SUnit :: const_succ_iterator I = Packet [ i ] -> Succs . begin ( ) , E = Packet


<s> static inline bool isSingleUnscheduledSucc ( SUnit * SU , SUnit * SU2 ) { if ( SU -> NumSuccsLeft == <NUM_LIT> ) return false ; for ( auto & Succ : SU ->


<s> for ( auto & Succ : SU -> Succs ) { if ( ! Succ . getSUnit ( ) -> isScheduled && ( Succ . getSUnit ( ) != SU2 )


<s> if ( DAG -> top ( ) == DAG -> bottom ( ) ) { assert ( Top . Available . empty ( ) && Top . Pending . empty ( ) && Bot . Available . empty ( ) && Bot . Pending . empty ( ) && <STR_LIT> ) ; return nullptr ; } SUnit * SU ; if ( llvm :: ForceTopDown ) { SU = Top . pickOnlyChoice ( ) ; if ( ! SU ) { SchedCandidate TopCand ; CandResult TopResult = pickNodeFromQueue ( Top . Available , DAG -> getTopRPTracker ( ) , TopCand ) ; assert ( TopResult != NoCand && <STR_LIT> ) ; ( void ) TopResult ; SU = TopCand . SU ; } IsTopNode = true ; } else if ( llvm :: ForceBottomUp ) { SU = Bot . pickOnlyChoice ( ) ; if ( ! SU ) { SchedCandidate BotCand ; CandResult BotResult = pickNodeFromQueue ( Bot . Available , DAG -> getBotRPTracker ( ) , BotCand ) ; assert ( BotResult != NoCand && <STR_LIT> ) ; ( void ) BotResult ; SU = BotCand . SU ; } IsTopNode = false ; } else { SU = pickNodeBidrectional ( IsTopNode ) ; } if ( SU -> isTopReady ( ) ) Top . removeReady ( SU ) ; if ( SU -> isBottomReady ( ) ) Bot . removeReady ( SU ) ; DEBUG ( dbgs ( ) << <STR_LIT> << ( IsTopNode ? <STR_LIT> : <STR_LIT> ) << <STR_LIT> << ( IsTopNode ?

Top . CurrCycle : Bot . CurrCycle ) << '\n' ;


<s> return SU ; } if ( SUnit * SU = Top . pickOnlyChoice ( ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode = true ; return SU ; } SchedCandidate BotCand ; CandResult BotResult = pickNodeFromQueue ( Bot . Available , DAG -> getBotRPTracker ( ) , BotCand ) ; assert ( BotResult != NoCand && <STR_LIT> ) ; if ( BotResult == SingleExcess || BotResult == SingleCritical ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode = false ; return BotCand . SU ; } SchedCandidate TopCand ; CandResult TopResult = pickNodeFromQueue ( Top . Available , DAG -> getTopRPTracker ( ) , TopCand ) ; assert ( TopResult != NoCand && <STR_LIT> ) ; if ( TopResult == SingleExcess || TopResult == SingleCritical ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode = true ; return TopCand . SU ; } if ( BotResult == SingleMax ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode = false ; return BotCand . SU ; } if ( TopResult == SingleMax ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode = true ; return TopCand . SU ; } if ( TopCand . SCost > BotCand . SCost ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode = true ; return TopCand . SU ; } DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode =


<s> IsTopNode = true ; return SU ; } SchedCandidate BotCand ; CandResult BotResult = pickNodeFromQueue ( Bot . Available , DAG -> getBotRPTracker ( ) , BotCand ) ; assert ( BotResult != NoCand && <STR_LIT> ) ; if ( BotResult == SingleExcess || BotResult == SingleCritical ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode = false ; return BotCand . SU ; } SchedCandidate TopCand ; CandResult TopResult = pickNodeFromQueue ( Top . Available , DAG -> getTopRPTracker ( ) , TopCand ) ; assert ( TopResult != NoCand && <STR_LIT> ) ; if ( TopResult == SingleExcess || TopResult == SingleCritical ) { DEBUG ( dbgs ( ) << <STR_LIT>


<s> Candidate . RPDelta = RPDelta ; Candidate . SCost = CurrentCost ; FoundCandidate = BestCost ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ; } else if ( InstrLatency > CandLatency && ! TopUseShorterTie ) { Candidate . SU = * I ; Candidate . RPDelta = RPDelta ; Candidate . SCost = CurrentCost ; FoundCandidate = BestCost ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ; } } else if ( Q . getID ( ) == BotQID && CurrentCost == Candidate . SCost ) { if ( InstrLatency < CandLatency && BotUseShorterTie ) { Candidate . SU = * I ; Candidate . RPDelta = RPDelta ; Candidate . SCost = CurrentCost ; FoundCandidate = BestCost ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ; } else if ( InstrLatency > CandLatency && ! BotUseShorterTie ) { Candidate . SU = * I ; Candidate . RPDelta = RPDelta ; Candidate . SCost = CurrentCost ; FoundCandidate = BestCost ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ; } } } if ( CurrentCost == Candidate . SCost ) { if ( ( Q . getID ( ) == TopQID && ( * I ) -> Succs . size ( ) > Candidate . SU -> Succs . size ( ) ) || ( Q . getID ( ) == BotQID && ( * I ) -> Preds . size ( ) < Candidate . SU -> Preds . size ( ) ) ) { DEBUG ( traceCandidate ( <STR_LIT> , Q , * I , CurrentCost ) ) ; Candidate . SU = * I


<s> for ( const SDep & PI : SU -> Preds ) { unsigned PredReadyCycle = PI . getSUnit ( ) -> TopReadyCycle ; unsigned MinLatency = PI . getLatency ( ) ; Top . MaxMinLatency = std :: max ( MinLatency , Top . MaxMinLatency ) ; if ( SU -> TopReadyCycle < PredReadyCycle + MinLatency ) SU -> TopReadyCycle = PredReadyCycle + MinLatency ; } Top . releaseNode ( SU


<s> default : ResourcesModel -> reserveResources ( * SU -> getInstr ( ) ) ; break ; case TargetOpcode :: EXTRACT_SUBREG : case TargetOpcode :: INSERT_SUBREG : case TargetOpcode :: SUBREG_TO_REG : case TargetOpcode :: REG_SEQUENCE : case TargetOpcode :: IMPLICIT_DEF : case TargetOpcode :: KILL : case TargetOpcode :: CFI_INSTRUCTION : case TargetOpcode :: EH_LABEL : case TargetOpcode :: COPY : case TargetOpcode :: INLINEASM : break ; } Packet . push_back ( SU ) ; DEBUG ( dbgs ( ) << <STR_LIT> << TotalPackets << <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = Packet . size ( ) ; i != e ; ++ i ) { DEBUG ( dbgs ( ) << <STR_LIT> << i << <STR_LIT> ) ; DEBUG ( dbgs ( ) << Packet [ i ] -> NodeNum << <STR_LIT> ) ; DEBUG ( Packet [ i ] -> getInstr ( ) -> dump ( ) ) ; } if ( Packet . size ( ) >= SchedModel -> getIssueWidth ( ) ) { ResourcesModel -> clearResources ( ) ; savePacket ( ) ; Packet . clear ( ) ; TotalPackets ++ ; startNewCycle =


<s> void VLIWResourceModel :: savePacket ( ) { OldPacket = Packet


<s> initQueues ( TopRoots , BotRoots ) ; bool IsTopNode = false ; while ( true ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; SUnit * SU = SchedImpl -> pickNode ( IsTopNode ) ; if ( ! SU ) break ; if ( ! checkSchedLimit ( ) ) break ; scheduleMI ( SU , IsTopNode ) ; updateQueues ( SU , IsTopNode ) ; SchedImpl -> schedNode ( SU , IsTopNode ) ; } assert ( CurrentTop == CurrentBottom && <STR_LIT> ) ; placeDebugValues ( ) ; DEBUG ( { unsigned BBNum = begin ( ) -> getParent ( ) -> getNumber ( ) ; dbgs ( ) << <STR_LIT> << BBNum << <STR_LIT> ; dumpSchedule (


<s> if ( Inst1 . getInstr ( ) -> getOpcode ( ) != <STR_LIT> ) return false ; unsigned Type = HII . getType ( * Inst2 . getInstr (


<s> for ( SUnit :: const_succ_iterator I = SU -> Succs . begin ( ) , E = SU -> Succs . end ( ) ; I != E ; ++ I ) { SUnit & Succ = * I -> getSUnit ( ) ; if ( ! Succ


<s> void ConvergingVLIWScheduler :: initialize ( ScheduleDAGMI * dag ) { DAG = static_cast < VLIWMachineScheduler * > ( dag ) ; SchedModel = DAG -> getSchedModel ( ) ; TRI = DAG -> TRI ; Top . init ( DAG , SchedModel ) ; Bot . init ( DAG , SchedModel ) ; const InstrItineraryData * Itin = DAG -> getSchedModel ( ) -> getInstrItineraries (


<s> void ConvergingVLIWScheduler :: initialize ( ScheduleDAGMI * dag ) { DAG = static_cast < VLIWMachineScheduler * > ( dag ) ; SchedModel = DAG -> getSchedModel ( ) ; TRI = DAG -> TRI ; Top . init ( DAG , SchedModel ) ; Bot . init ( DAG , SchedModel ) ; const InstrItineraryData * Itin = DAG -> getSchedModel ( ) -> getInstrItineraries ( ) ; const TargetMachine & TM = DAG -> MF . getTarget ( ) ; delete Top . HazardRec ; delete Bot . HazardRec ; Top . HazardRec = TM . getInstrInfo ( ) -> CreateTargetMIHazardRecognizer ( Itin , DAG ) ; Bot . HazardRec = TM . getInstrInfo ( ) -> CreateTargetMIHazardRecognizer ( Itin , DAG ) ; Top . ResourceModel = new VLIWResourceModel ( TM , DAG -> getSchedModel ( ) ) ; Bot . ResourceModel = new VLIWResourceModel ( TM , DAG ->


<s> SUnit * ConvergingVLIWScheduler :: pickNode ( bool & IsTopNode ) { if ( DAG -> top ( ) == DAG -> bottom ( ) ) { assert ( Top . Available . empty ( ) && Top . Pending . empty ( ) && Bot . Available . empty ( ) && Bot . Pending . empty ( ) && <STR_LIT> ) ; return NULL ; } SUnit * SU ; if ( llvm :: ForceTopDown ) { SU = Top . pickOnlyChoice ( ) ; if ( ! SU ) { SchedCandidate TopCand ; CandResult TopResult = pickNodeFromQueue ( Top . Available , DAG -> getTopRPTracker ( ) , TopCand ) ; assert ( TopResult != NoCand && <STR_LIT> ) ; ( void ) TopResult ; SU = TopCand . SU ; } IsTopNode = true ; } else if ( llvm :: ForceBottomUp ) { SU = Bot . pickOnlyChoice ( ) ; if ( ! SU ) { SchedCandidate BotCand ; CandResult BotResult = pickNodeFromQueue ( Bot . Available , DAG -> getBotRPTracker ( ) , BotCand ) ; assert ( BotResult != NoCand && <STR_LIT> ) ; ( void ) BotResult ; SU = BotCand . SU ; } IsTopNode = false ; } else { SU = pickNodeBidrectional ( IsTopNode ) ; } if ( SU -> isTopReady ( ) ) Top . removeReady ( SU


<s> unsigned SuccReadyCycle = I -> getSUnit ( ) -> BotReadyCycle ; unsigned MinLatency = I -> getMinLatency ( ) ; Bot . MaxMinLatency = std :: max ( MinLatency , Bot . MaxMinLatency ) ; if ( SU -> BotReadyCycle < SuccReadyCycle + MinLatency ) SU -> BotReadyCycle = SuccReadyCycle + MinLatency ; } Bot . releaseNode ( SU ,


<s> for ( SUnit :: succ_iterator I = SU -> Succs . begin ( ) , E = SU -> Succs . end ( ) ; I != E ; ++ I ) { unsigned SuccReadyCycle = I -> getSUnit ( ) -> BotReadyCycle ; unsigned MinLatency = I -> getMinLatency ( ) ; Bot . MaxMinLatency = std :: max ( MinLatency


<s> void ConvergingVLIWScheduler :: releaseTopNode ( SUnit * SU ) { if ( SU -> isScheduled ) return ; for ( SUnit :: succ_iterator I = SU -> Preds . begin ( ) , E = SU -> Preds . end ( ) ; I != E ; ++ I ) { unsigned PredReadyCycle = I -> getSUnit ( ) -> TopReadyCycle ; unsigned MinLatency = I ->


<s> dbgs ( ) << <STR_LIT> << maxH << <STR_LIT> ; ) ; DEBUG ( unsigned maxD = <NUM_LIT> ; for ( unsigned su = <NUM_LIT> , e = SUnits . size ( ) ; su != e ; ++ su ) if ( SUnits [ su ] . getDepth ( ) > maxD ) maxD = SUnits [ su ] . getDepth ( ) ; dbgs ( ) << <STR_LIT> << maxD << <STR_LIT> ; ) ; DEBUG ( for ( unsigned su = <NUM_LIT> , e = SUnits . size ( ) ; su != e ; ++ su ) SUnits [ su ] . dumpAll ( this ) ) ; initQueues ( TopRoots , BotRoots ) ; bool IsTopNode = false


<s> ResCount += ( SU -> getHeight ( ) * ScaleTwo ) ; if ( Top . ResourceModel -> isResourceAvailable ( SU ) ) ResCount <<= FactorOne ; } else { ResCount += ( SU -> getDepth ( ) * ScaleTwo ) ; if ( Bot . ResourceModel -> isResourceAvailable ( SU ) ) ResCount <<= FactorOne ; } unsigned NumNodesBlocking = <NUM_LIT> ; if ( Q . getID ( ) == TopQID ) { for ( SUnit :: const_succ_iterator I = SU -> Succs . begin ( ) , E = SU -> Succs . end ( ) ; I != E ; ++ I ) if ( getSingleUnscheduledPred ( I -> getSUnit ( ) )


<s> void ConvergingVLIWScheduler :: traceCandidate ( const char * Label , const ReadyQueue & Q , SUnit * SU , PressureElement P ) { dbgs ( ) << Label << <STR_LIT> << Q . getName ( ) << <STR_LIT> ; if ( P . isValid ( ) ) dbgs ( ) << TRI -> getRegPressureSetName ( P . PSetID ) << <STR_LIT> << P . UnitIncrease << <STR_LIT> ; else dbgs ( ) <<


<s> LLVM_DEBUG ( if ( verbose ) { std :: stringstream dbgstr ; dbgstr << <STR_LIT> << std :: setw ( <NUM_LIT> ) << ResCount << <STR_LIT> ; dbgs ( ) << dbgstr . str (


<s> switch ( SU -> getInstr ( ) -> getOpcode ( ) ) { default : if ( ! ResourcesModel -> canReserveResources ( * SU -> getInstr ( ) ) ) return false ; break ; case TargetOpcode :: EXTRACT_SUBREG : case TargetOpcode :: INSERT_SUBREG : case TargetOpcode :: SUBREG_TO_REG : case TargetOpcode :: REG_SEQUENCE : case TargetOpcode :: IMPLICIT_DEF : case TargetOpcode :: COPY : case TargetOpcode :: INLINEASM : case TargetOpcode


<s> for ( SUnit :: succ_iterator I = SU -> Succs . begin ( ) , E = SU -> Succs . end ( ) ; I != E ; ++ I ) { unsigned SuccReadyCycle = I -> getSUnit ( ) -> BotReadyCycle ; unsigned MinLatency = I -> getLatency (


<s> void ConvergingVLIWScheduler :: releaseBottomNode ( SUnit * SU ) { assert ( SU -> getInstr ( ) && <STR_LIT> ) ; for ( SUnit :: succ_iterator I = SU -> Succs . begin ( ) , E = SU -> Succs . end ( ) ; I != E ; ++ I ) { unsigned SuccReadyCycle = I -> getSUnit ( ) -> BotReadyCycle ; unsigned MinLatency = I -> getLatency ( ) ; Bot . MaxMinLatency = std :: max ( MinLatency , Bot . MaxMinLatency ) ; if ( SU -> BotReadyCycle < SuccReadyCycle + MinLatency ) SU ->

BotReadyCycle = SuccReadyCycle + MinLatency ;


<s> for ( const SDep & PI : SU -> Preds ) { unsigned PredReadyCycle = PI . getSUnit ( ) -> TopReadyCycle ; unsigned MinLatency = PI . getLatency ( ) ; Top . MaxMinLatency = std :: max ( MinLatency , Top . MaxMinLatency ) ; if ( SU -> TopReadyCycle < PredReadyCycle + MinLatency ) SU

-> TopReadyCycle = PredReadyCycle + MinLatency ;


<s> void ConvergingVLIWScheduler :: schedNode ( SUnit * SU , bool IsTopNode ) { if ( IsTopNode ) { Top . bumpNode ( SU


<s> if ( P . isValid ( ) ) dbgs ( ) << DAG -> TRI -> getRegPressureSetName ( P . getPSet ( ) ) << <STR_LIT> << P . getUnitInc ( ) << <STR_LIT> ; else dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ; DAG -> dumpNode ( * SU


<s> dbgs ( ) << Label << <STR_LIT> << Q . getName ( ) << <STR_LIT> ; if ( P . isValid ( ) ) dbgs ( ) << DAG -> TRI -> getRegPressureSetName ( P . getPSet ( ) ) << <STR_LIT> << P . getUnitInc ( ) << <STR_LIT> ; else dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ; DAG -> dumpNode ( *


<s> delete Top . ResourceModel ; delete Bot . ResourceModel ; Top . ResourceModel = new VLIWResourceModel ( STI , DAG -> getSchedModel ( ) ) ; Bot . ResourceModel = new VLIWResourceModel ( STI , DAG -> getSchedModel ( ) ) ; assert ( ( ! ForceTopDown || ! ForceBottomUp


<s> if ( DAG -> top ( ) == DAG -> bottom ( ) ) { assert ( Top . Available . empty ( ) && Top . Pending . empty ( ) && Bot . Available . empty ( ) && Bot . Pending . empty ( ) && <STR_LIT> ) ; return nullptr ; } SUnit * SU ; if ( ForceTopDown ) { SU = Top . pickOnlyChoice ( ) ; if ( ! SU ) { SchedCandidate TopCand ; CandResult TopResult = pickNodeFromQueue ( Top . Available , DAG -> getTopRPTracker ( ) , TopCand ) ; assert ( TopResult != NoCand && <STR_LIT> ) ; ( void ) TopResult ; SU = TopCand . SU ; } IsTopNode = true ; } else if ( ForceBottomUp ) { SU = Bot . pickOnlyChoice ( ) ; if ( ! SU ) { SchedCandidate BotCand ; CandResult BotResult = pickNodeFromQueue ( Bot . Available , DAG -> getBotRPTracker ( ) , BotCand ) ; assert ( BotResult != NoCand && <STR_LIT> ) ; ( void


<s> void init ( VLIWMachineScheduler * dag ,


<s> bool isInPacket ( SUnit * SU ) const { return is_contained ( Packet ,


<s> assert ( ResourcesModel && <STR_LIT> ) ; Packet . resize ( SchedModel -> getIssueWidth ( ) ) ; Packet . clear ( ) ; OldPacket . resize ( SchedModel -> getIssueWidth (


<s> SchedModel ( SM ) , TotalPackets ( <NUM_LIT> ) { ResourcesModel = TM . getSubtargetImpl ( ) -> getInstrInfo ( ) -> CreateTargetScheduleState ( * TM . getSubtargetImpl ( ) ) ; assert ( ResourcesModel && <STR_LIT> ) ; Packet . resize ( SchedModel -> getIssueWidth ( ) ) ; Packet . clear ( )


<s> assert ( ResourcesModel && <STR_LIT> ) ; Packet . resize ( SchedModel -> getIssueWidth ( ) ) ; Packet . clear ( ) ; ResourcesModel -> clearResources


<s> case TargetOpcode :: COPY : case TargetOpcode :: INLINEASM : break ; } MachineBasicBlock * MBB = SU -> getInstr ( ) -> getParent ( ) ; auto & QST = MBB -> getParent ( ) -> getSubtarget < <ISA_LIT>Subtarget > ( ) ; const auto & QII = * QST . getInstrInfo ( ) ; if ( IsTop ) { for ( unsigned i = <NUM_LIT> , e = Packet . size ( ) ; i != e ; ++ i ) if ( hasDependence ( Packet [ i ] , SU , QII ) ) return false ; } else { for ( unsigned i = <NUM_LIT> , e = Packet . size ( ) ; i != e ; ++ i ) if ( hasDependence

( SU , Packet [ i ] , QII ) ) return false ;


<s> case TargetOpcode :: COPY : case TargetOpcode :: INLINEASM : break ; } Packet . push_back ( SU ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << TotalPackets << <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = Packet . size ( ) ; i != e ; ++ i ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << i << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << Packet [ i ] ->


<s> break ; case TargetOpcode :: EXTRACT_SUBREG : case TargetOpcode :: INSERT_SUBREG : case TargetOpcode :: SUBREG_TO_REG : case TargetOpcode :: REG_SEQUENCE : case TargetOpcode :: IMPLICIT_DEF : case TargetOpcode :: KILL : case TargetOpcode :: CFI_INSTRUCTION : case TargetOpcode :: EH_LABEL : case TargetOpcode :: COPY : case TargetOpcode :: INLINEASM : break ; } Packet . push_back ( SU ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << TotalPackets << <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = Packet . size ( ) ; i != e ; ++ i ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << i <<


<s> ConvergingVLIWScheduler ( ) : DAG ( <NUM_LIT> ) , TRI ( <NUM_LIT> ) , Top ( TopQID , <STR_LIT> ) , Bot


<s> DAG ( <NUM_LIT> ) , TRI ( <NUM_LIT> ) , Top ( TopQID , <STR_LIT> ) , Bot


<s> VLIWResourceModel * <ISA_LIT>ConvergingVLIWScheduler :: createVLIWResourceModel ( const TargetSubtargetInfo &

STI , const TargetSchedModel * SchedModel ) const {


<s> DAG ( <NUM_LIT> ) , Available ( ID , Name + <STR_LIT> ) , Pending ( ID << ConvergingVLIWScheduler :: LogMaxQID , Name + <STR_LIT> ) , CheckPending ( false ) , HazardRec ( <NUM_LIT>

) , ResourceModel ( <NUM_LIT> ) , CurrCycle ( <NUM_LIT> ) , IssueCount ( <NUM_LIT> ) , MinReadyCycle ( UINT_MAX ) , MaxMinLatency ( <NUM_LIT> ) {


<s> DAG ( <NUM_LIT> ) , Available ( ID , Name + <STR_LIT> ) , Pending ( ID << ConvergingVLIWScheduler :: LogMaxQID , Name + <STR_LIT> ) , CheckPending ( false ) , HazardRec ( <NUM_LIT> ) , ResourceModel ( <NUM_LIT> ) , CurrCycle ( <NUM_LIT>

) , IssueCount ( <NUM_LIT> ) , MinReadyCycle ( UINT_MAX ) , MaxMinLatency ( <NUM_LIT> ) {


<s> if ( ! SU || SU -> isScheduled ) return ResCount ; auto & QST = DAG -> MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; auto & QII = * QST . getInstrInfo ( ) ; if ( SU -> isInstr ( ) && QII . mayBeCurLoad ( * SU -> getInstr ( ) ) ) { if ( Q . getID ( ) == TopQID && Top . ResourceModel -> isResourceAvailable ( SU , true ) ) { ResCount += PriorityTwo ; LLVM_DEBUG ( if ( verbose ) dbgs ( ) <<


<s> VLIWResourceModel ( const TargetMachine & TM ) : InstrItins ( TM . getInstrItineraryData ( ) ) , TotalPackets ( <NUM_LIT> ) { ResourcesModel = TM . getInstrInfo ( ) -> CreateTargetScheduleState ( & TM , NULL ) ; assert ( ResourcesModel && <STR_LIT> ) ; Packet . resize ( InstrItins -> SchedModel -> IssueWidth ) ; Packet . clear ( ) ; ResourcesModel -> clearResources (


<s> ~ SchedBoundary ( ) { delete ResourceModel


<s> delete ResourceModel ; delete HazardRec


<s> SchedModel ( SM ) , TotalPackets ( <NUM_LIT> ) { ResourcesModel = TM . getInstrInfo ( ) -> CreateTargetScheduleState ( & TM


<s> DAG ( <NUM_LIT> ) , SchedModel ( <NUM_LIT> ) , Top ( TopQID , <STR_LIT> ) , Bot ( BotQID


<s> DAG ( <NUM_LIT> ) , SchedModel ( <NUM_LIT> ) , Top ( TopQID ,

<STR_LIT> ) , Bot ( BotQID , <STR_LIT> ) {


<s> VLIWSchedBoundary ( unsigned ID , const Twine & Name ) : DAG ( <NUM_LIT> ) , SchedModel ( <NUM_LIT> ) , Available ( ID , Name + <STR_LIT> ) , Pending ( ID << ConvergingVLIWScheduler :: LogMaxQID , Name + <STR_LIT> ) , CheckPending ( false ) , HazardRec ( <NUM_LIT> ) , ResourceModel ( <NUM_LIT> ) , CurrCycle ( <NUM_LIT> ) , IssueCount ( <NUM_LIT>

) , MinReadyCycle ( UINT_MAX ) , MaxMinLatency ( <NUM_LIT> ) {


<s> case TargetOpcode :: EXTRACT_SUBREG : case TargetOpcode :: INSERT_SUBREG : case TargetOpcode :: SUBREG_TO_REG : case TargetOpcode :: REG_SEQUENCE : case TargetOpcode :: IMPLICIT_DEF : case TargetOpcode :: COPY : case TargetOpcode :: INLINEASM : break ; } MachineFunction & MF = * SU -> getInstr ( ) -> getParent ( ) -> getParent ( ) ; auto & QII = * MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) ; for ( unsigned i = <NUM_LIT> , e = Packet . size ( ) ; i != e ; ++ i ) { if ( Packet [ i ] -> Succs . size ( ) == <NUM_LIT> ) continue ; if ( QII . mayBeCurLoad ( Packet [ i ] -> getInstr ( ) ) ) continue ; for ( SUnit :: const_succ_iterator I = Packet [ i ] -> Succs . begin ( ) , E = Packet [ i ] -> Succs . end ( ) ; I != E ; ++ I ) { if ( I -> isCtrl ( ) ) continue ; if ( I -> getSUnit ( ) == SU ) return false ; } } return true


<s> DEBUG ( dbgs ( ) << <STR_LIT> << CandLatency << <STR_LIT> << InstrLatency << <STR_LIT> << * MI << * CandI << <STR_LIT> ) ; if ( Q . getID ( ) == TopQID && CurrentCost == Candidate . SCost ) { if ( InstrLatency < CandLatency && TopUseShorterTie ) { Candidate . SU = * I ; Candidate . RPDelta = RPDelta ; Candidate . SCost = CurrentCost ; FoundCandidate = BestCost ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ; } else if ( InstrLatency > CandLatency && ! TopUseShorterTie ) { Candidate . SU = * I ; Candidate . RPDelta = RPDelta ; Candidate . SCost = CurrentCost ; FoundCandidate = BestCost ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ; } } else if ( Q . getID ( ) == BotQID && CurrentCost == Candidate . SCost ) { if ( InstrLatency < CandLatency && BotUseShorterTie ) { Candidate . SU = * I ; Candidate . RPDelta = RPDelta ; Candidate . SCost = CurrentCost ; FoundCandidate = BestCost ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ; } else if ( InstrLatency > CandLatency && ! BotUseShorterTie ) { Candidate . SU = *


<s> } auto & QST = DAG -> MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; auto & QII = * QST . getInstrInfo ( ) ; if ( SU -> isInstr ( ) && QII . mayBeCurLoad ( SU -> getInstr ( ) ) ) { if ( Q . getID ( ) == TopQID && Top . ResourceModel -> isResourceAvailable ( SU ) ) { ResCount += PriorityTwo ; DEBUG ( if ( verbose ) dbgs ( ) << <STR_LIT> ) ; } else if ( Q . getID ( ) == BotQID && Bot . ResourceModel -> isResourceAvailable ( SU ) ) { ResCount += PriorityTwo ; DEBUG ( if ( verbose ) dbgs ( ) << <STR_LIT> ) ; } } if ( Q . getID ( ) == TopQID ) { for ( const SDep & PI : SU -> Preds ) { if ( ! PI . getSUnit ( ) -> getInstr ( ) -> isPseudo ( ) && PI . isAssignedRegDep ( ) && PI . getLatency ( ) == <NUM_LIT> && Top . ResourceModel -> isInPacket ( PI . getSUnit ( ) ) ) { ResCount += PriorityThree ; DEBUG ( if ( verbose ) dbgs ( ) << <STR_LIT> ) ; } } } else { for ( const SDep & SI : SU -> Succs ) { if ( ! SI . getSUnit ( ) -> getInstr ( ) -> isPseudo ( ) && SI . isAssignedRegDep ( ) && SI . getLatency ( ) == <NUM_LIT> && Bot . ResourceModel -> isInPacket ( SI . getSUnit ( ) ) ) { ResCount += PriorityThree ; DEBUG ( if ( verbose ) dbgs ( ) << <STR_LIT> ) ; } } } if ( QII . isV60VectorInstruction ( Instr ) ) { if ( Q . getID ( ) == TopQID ) { for ( auto J : Top . ResourceModel -> OldPacket ) if ( QII . producesStall ( J -> getInstr ( ) , Instr ) ) ResCount -= PriorityOne ; } else { for ( auto J : Bot . ResourceModel -> OldPacket ) if ( QII . producesStall ( Instr , J -> getInstr ( ) ) ) ResCount -= PriorityOne ; } } if ( CheckEarlyAvail ) { if ( Q . getID ( ) == TopQID ) { for ( const auto & PI : SU -> Preds ) { if ( PI . getLatency ( ) > <NUM_LIT> && Top . ResourceModel -> isInPacket ( PI . getSUnit ( ) ) ) { ResCount -= PriorityOne ; DEBUG ( if ( verbose ) dbgs ( ) << <STR_LIT> ) ; } } } else { for ( const auto & SI : SU ->


<s> bool <ISA_LIT>CallMutation :: shouldTFRICallBind ( const <ISA_LIT>InstrInfo & HII , const SUnit & Inst1 ,


<s> SchedModel = smodel ; CurrCycle = <NUM_LIT> ; IssueCount = <NUM_LIT> ; CriticalPathLength = DAG -> getBBSize ( ) / SchedModel -> getIssueWidth ( ) ; if ( DAG -> getBBSize ( ) < <NUM_LIT> ) CriticalPathLength >>= <NUM_LIT> ; else { unsigned MaxPath


<s> Top . DAG = DAG ; Bot . DAG = DAG ; const TargetMachine & TM = DAG -> MF . getTarget ( ) ; const InstrItineraryData * Itin = TM . getInstrItineraryData ( ) ; Top . HazardRec = TM . getInstrInfo ( ) -> CreateTargetMIHazardRecognizer ( Itin , DAG ) ; Bot . HazardRec = TM . getInstrInfo ( ) -> CreateTargetMIHazardRecognizer (


<s> unsigned reportPackets ( ) { return Top . ResourceModel -> getTotalPackets ( ) + Bot . ResourceModel ->


<s> } if ( ! isResourceAvailable ( SU ) ) { ResourcesModel -> clearResources ( ) ; Packet . clear ( ) ; TotalPackets ++ ; startNewCycle = true ; } switch ( SU -> getInstr ( ) -> getOpcode ( ) ) { default : ResourcesModel -> reserveResources ( SU -> getInstr ( ) ) ; break ; case TargetOpcode :: EXTRACT_SUBREG : case TargetOpcode :: INSERT_SUBREG : case TargetOpcode :: SUBREG_TO_REG : case TargetOpcode :: REG_SEQUENCE : case TargetOpcode :: IMPLICIT_DEF : case TargetOpcode :: KILL : case TargetOpcode :: PROLOG_LABEL : case TargetOpcode :: EH_LABEL : case TargetOpcode :: COPY : case TargetOpcode :: INLINEASM : break ; } Packet . push_back ( SU ) ; DEBUG ( dbgs ( ) << <STR_LIT>


<s> VLIWResourceModel ( const TargetSubtargetInfo & STI , const TargetSchedModel * SM ) : SchedModel ( SM ) { ResourcesModel = STI . getInstrInfo ( ) -> CreateTargetScheduleState ( STI ) ; assert ( ResourcesModel && <STR_LIT> ) ; Packet . resize ( SchedModel -> getIssueWidth ( ) ) ; Packet . clear


<s> Top . init ( DAG , SchedModel ) ; Bot . init ( DAG , SchedModel ) ; const InstrItineraryData * Itin = DAG -> getSchedModel ( ) -> getInstrItineraries ( ) ; const TargetMachine & TM = DAG -> MF . getTarget ( ) ; Top . HazardRec = TM . getInstrInfo ( ) -> CreateTargetMIHazardRecognizer ( Itin , DAG ) ; Bot . HazardRec = TM . getInstrInfo ( ) -> CreateTargetMIHazardRecognizer (


<s> case TargetOpcode :: SUBREG_TO_REG : case TargetOpcode :: REG_SEQUENCE : case TargetOpcode :: IMPLICIT_DEF : case TargetOpcode :: COPY : case TargetOpcode :: INLINEASM : break ; } for ( unsigned i = <NUM_LIT> , e = Packet . size ( ) ; i != e ; ++ i ) { if ( Packet [ i ] -> Succs . size ( ) == <NUM_LIT> ) continue ; for ( SUnit :: const_succ_iterator I = Packet [ i ] -> Succs . begin ( ) , E = Packet [ i ] -> Succs . end ( ) ; I != E ; ++ I ) { if ( I -> isCtrl ( ) )


<s> DAG ( <NUM_LIT> ) , SchedModel ( <NUM_LIT> ) , Available ( ID , Name + <STR_LIT> ) , Pending ( ID << ConvergingVLIWScheduler :: LogMaxQID , Name + <STR_LIT> ) , CheckPending ( false ) , HazardRec ( <NUM_LIT> ) , ResourceModel ( <NUM_LIT> ) , CurrCycle ( <NUM_LIT> ) , IssueCount ( <NUM_LIT> )

, MinReadyCycle ( UINT_MAX ) , MaxMinLatency ( <NUM_LIT> ) {


<s> DAG ( <NUM_LIT> ) , SchedModel ( <NUM_LIT> ) , Available ( ID , Name + <STR_LIT> ) , Pending ( ID << ConvergingVLIWScheduler :: LogMaxQID , Name + <STR_LIT> ) , CheckPending ( false ) , HazardRec

( <NUM_LIT> ) , ResourceModel ( <NUM_LIT> ) , CurrCycle ( <NUM_LIT> ) , IssueCount ( <NUM_LIT> ) , MinReadyCycle ( UINT_MAX ) , MaxMinLatency ( <NUM_LIT> ) {


<s> Bot . HazardRec = TM . getSubtargetImpl ( ) -> getInstrInfo ( ) -> CreateTargetMIHazardRecognizer ( Itin , DAG ) ; delete Top . ResourceModel ; delete Bot . ResourceModel ; Top . ResourceModel = new VLIWResourceModel ( TM , DAG -> getSchedModel ( ) ) ; Bot . ResourceModel = new VLIWResourceModel ( TM , DAG -> getSchedModel ( ) ) ; assert ( ( ! llvm :: ForceTopDown || ! llvm ::


<s> DAG = static_cast < VLIWMachineScheduler * > ( dag ) ; SchedModel = DAG -> getSchedModel ( ) ; Top . init ( DAG , SchedModel ) ; Bot . init ( DAG , SchedModel ) ; const InstrItineraryData * Itin = DAG -> getSchedModel ( ) -> getInstrItineraries ( ) ; const TargetMachine & TM = DAG -> MF . getTarget ( ) ; delete Top . HazardRec ; delete Bot . HazardRec ; Top . HazardRec = TM . getSubtargetImpl ( ) -> getInstrInfo ( ) -> CreateTargetMIHazardRecognizer ( Itin , DAG ) ; Bot . HazardRec = TM . getSubtargetImpl ( ) -> getInstrInfo ( ) -> CreateTargetMIHazardRecognizer ( Itin , DAG ) ; delete Top . ResourceModel ; delete Bot .


<s> dbgs ( ) << <STR_LIT> << maxH << <STR_LIT> ; ) ; DEBUG ( unsigned maxD = <NUM_LIT> ; for ( unsigned su = <NUM_LIT> , e = SUnits . size ( ) ; su != e ; ++ su ) if ( SUnits [ su ] . getDepth ( ) > maxD ) maxD = SUnits [ su ] . getDepth ( ) ; dbgs ( ) << <STR_LIT> << maxD << <STR_LIT> ; ) ; DEBUG ( for ( unsigned su = <NUM_LIT> , e = SUnits . size ( ) ; su != e ; ++ su ) SUnits [ su ] . dumpAll ( this ) ) ; initQueues ( TopRoots , BotRoots ) ; bool IsTopNode =


<s> VLIWResourceModel ( const TargetMachine & TM , const TargetSchedModel * SM ) : SchedModel ( SM ) , TotalPackets ( <NUM_LIT> ) { ResourcesModel = TM . getSubtargetImpl ( ) -> getInstrInfo ( ) -> CreateTargetScheduleState ( & TM , nullptr ) ; assert ( ResourcesModel && <STR_LIT>


<s> VLIWMachineScheduler ( MachineSchedContext * C , MachineSchedStrategy


<s> static unsigned getWeakLeft ( const SUnit * SU ,


<s> auto & QST = MBB -> getParent ( ) -> getSubtarget < <ISA_LIT>Subtarget > ( ) ; const auto & QII = * QST . getInstrInfo ( ) ; if ( IsTop ) { for ( unsigned i = <NUM_LIT> , e = Packet . size ( ) ; i != e ; ++ i ) if ( hasDependence ( Packet [ i ] , SU , QII ) ) return false ; } else { for ( unsigned i = <NUM_LIT> , e = Packet . size ( ) ; i != e ; ++ i ) if ( hasDependence ( SU , Packet [ i ] , QII ) ) return false ; } return


<s> DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode = false ; return SU ; } if ( SUnit * SU = Top . pickOnlyChoice ( ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode = true ; return SU ; } SchedCandidate BotCand ; CandResult BotResult = pickNodeFromQueue ( Bot , DAG -> getBotRPTracker ( ) , BotCand ) ; assert ( BotResult != NoCand && <STR_LIT> ) ; if ( BotResult == SingleExcess || BotResult == SingleCritical ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode = false ; return BotCand . SU ; } SchedCandidate TopCand ; CandResult TopResult = pickNodeFromQueue ( Top , DAG -> getTopRPTracker ( ) , TopCand ) ; assert ( TopResult != NoCand && <STR_LIT> ) ; if ( TopResult == SingleExcess || TopResult == SingleCritical ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode = true


<s> if ( SUnit * SU = Top . pickOnlyChoice ( ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode = true ; return SU ; } SchedCandidate BotCand ; CandResult BotResult = pickNodeFromQueue ( Bot , DAG -> getBotRPTracker ( ) , BotCand ) ; assert ( BotResult != NoCand && <STR_LIT> ) ; if ( BotResult == SingleExcess || BotResult == SingleCritical ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode = false ; return BotCand . SU ; } SchedCandidate TopCand ; CandResult TopResult = pickNodeFromQueue ( Top , DAG -> getTopRPTracker ( ) , TopCand ) ; assert ( TopResult != NoCand && <STR_LIT> ) ; if ( TopResult == SingleExcess || TopResult == SingleCritical ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; IsTopNode = true