<s> for ( auto & PredMI : Insts ) { if ( PredMI . Predicates . count ( VCTP ) || isVCTP ( PredMI . MI ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * PredMI . MI << <STR_LIT> ; for ( auto * MI : PredMI . Predicates ) dbgs ( ) << <STR_LIT> << * MI ) ; return false ; } } if ( ! ValidateLiveOuts ( ) ) return false ; Register NumElements = VCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( RDA . hasLocalDefBefore ( VCTP , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MachineBasicBlock * InsertBB = StartInsertPt -> getParent ( ) ; if ( ! RDA . isReachingDefLiveOut ( StartInsertPt , NumElements ) ) { if ( auto * ElemDef = RDA . getLocalLiveOutMIDef ( InsertBB , NumElements ) ) { if ( RDA . isSafeToMoveForwards ( ElemDef , StartInsertPt ) ) { ElemDef -> removeFromParent ( ) ; InsertBB -> insert ( MachineBasicBlock :: iterator ( StartInsertPt ) , ElemDef ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ElemDef ) ; } else if ( RDA . isSafeToMoveBackwards ( StartInsertPt , ElemDef ) ) { StartInsertPt -> removeFromParent ( ) ; InsertBB -> insertAfter ( MachineBasicBlock :: iterator ( ElemDef ) , StartInsertPt ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ElemDef ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return false ; } } } auto CannotProvideElements = [ this ] ( MachineBasicBlock * MBB , Register NumElements ) { if ( RDA . hasLocalDefBefore ( & MBB -> back ( ) , NumElements ) ) return true ; if ( MBB -> pred_size ( ) > <NUM_LIT> ) return true ; return false ; } ; MachineBasicBlock * MBB = MLI . findLoopPreheader ( & ML , true ) ; if ( ! MBB ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } while ( MBB != InsertBB ) { if ( CannotProvideElements ( MBB


<s> InsertPt = Revert ? nullptr : IsSafeToDefineLR ( RDA ) ; if ( ! InsertPt ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Revert = true ; return ; } else LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * InsertPt ) ; if ( ! IsTailPredicationLegal ( ) ) { LLVM_DEBUG ( if ( ! VCTP ) dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ) ; return ; } assert ( ML -> getBlocks ( ) . size ( ) == <NUM_LIT> && <STR_LIT> ) ; CannotTailPredicate = ! ValidateTailPredicate ( InsertPt , RDA , MLI ) ; LLVM_DEBUG ( if ( CannotTailPredicate ) dbgs ( ) <<


<s> PredicatedMI * Divergent = Block . getDivergent ( ) ; if ( isVCTP ( Divergent -> MI ) ) { uint64_t Size = get<ISA_LIT>VPTBlockMask ( Block . size ( ) - <NUM_LIT> ) ; Block . getVPST ( ) -> getOperand ( <NUM_LIT> ) . setImm ( Size ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } else if ( Block . IsOnlyPredicatedOn ( LoLoop . VCTP ) ) { for ( auto I = ++ MachineBasicBlock :: iterator ( Block . getVPST ( ) ) , E = ++ MachineBasicBlock :: iterator ( Divergent -> MI ) ; I != E ; ++ I ) RemovePredicate ( & * I ) ; unsigned Size = <NUM_LIT> ; auto E = MachineBasicBlock :: reverse_iterator ( Divergent -> MI ) ; auto I = MachineBasicBlock :: reverse_iterator ( Insts . back ( ) . MI ) ; MachineInstr * InsertAt = nullptr ; while ( I != E ) { InsertAt = &


<s> for ( auto * MBB : ML -> getBlocks ( ) ) dbgs ( ) << <STR_LIT> << MBB -> getName ( ) << <STR_LIT> ; ) ; std :: function < MachineInstr * ( MachineBasicBlock * ) > SearchForStart = [ & SearchForStart ] ( MachineBasicBlock * MBB ) -> MachineInstr * { for ( auto & MI : * MBB ) { if ( isLoopStart ( MI ) ) return & MI ; } if ( MBB -> pred_size ( ) == <NUM_LIT> ) return SearchForStart ( * MBB -> pred_begin ( ) ) ; return nullptr ; } ; LowOverheadLoop LoLoop ( ML ) ; if ( auto * Preheader = ML -> getLoopPreheader ( ) ) LoLoop . Start = SearchForStart ( Preheader ) ; else if ( auto * Preheader = MLI -> findLoopPreheader ( ML , true ) ) LoLoop . Start = SearchForStart ( Preheader ) ; else return false ; for ( auto * MBB : reverse ( ML -> getBlocks ( ) ) ) { for ( auto & MI : * MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . Dec = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . End = & MI ; else if ( isLoopStart ( MI ) ) LoLoop . Start = & MI ; else if ( MI . getDesc ( ) . isCall ( ) ) { LoLoop . Revert = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } else { LoLoop . AnalyseMVEInst ( & MI ) ; } if ( ! LoLoop . Dec || LoLoop . End || LoLoop . Revert ) continue ; for ( auto & MO : MI . operands ( ) ) { if ( MI . getOpcode ( ) != <STR_LIT> && MO . isReg ( ) && MO . getReg ( ) == <STR_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; LoLoop . Revert =


<s> CurrentBlock = & VPTBlocks . back ( ) ; return true ; } else if ( isVCTP ( MI ) ) VCTP = MI ; else if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) return false ; bool IsUse = false ; bool IsDef = false ; const MCInstrDesc & MCID = MI -> getDesc ( ) ; for ( int i = MI -> getNumOperands ( ) - <NUM_LIT> ; i >= <NUM_LIT> ; -- i ) { const MachineOperand & MO = MI -> getOperand ( i ) ; if ( ! MO . isReg ( ) || MO . getReg ( ) != <STR_LIT> ) continue ; if ( MO . isDef ( ) ) { CurrentPredicate . insert ( MI ) ; IsDef = true ; } else if ( <STR_LIT> ( MCID . OpInfo [ i ] . OperandType ) ) { CurrentBlock -> addInst ( MI , CurrentPredicate


<s> if ( CannotTailPredicate ) return false ; if ( isVCTP ( MI ) && VCTP ) return false ; if ( MI -> getOpcode ( ) == <STR_LIT> ) { VPTBlocks . emplace_back ( MI , CurrentPredicate ) ; CurrentBlock = & VPTBlocks . back ( ) ; return true ; } else if ( isVCTP ( MI ) ) VCTP = MI ; else if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) return false ; bool IsUse = false ; bool IsDef = false ; const MCInstrDesc & MCID = MI -> getDesc ( ) ; for ( int i = MI -> getNumOperands ( ) - <NUM_LIT> ; i >= <NUM_LIT> ; -- i ) { const MachineOperand & MO = MI -> getOperand ( i ) ; if ( ! MO . isReg ( ) || MO . getReg ( ) != <STR_LIT> ) continue ; if ( MO . isDef ( ) ) { CurrentPredicate . insert ( MI ) ; IsDef = true ; } else if ( <STR_LIT> ( MCID . OpInfo [ i ] . OperandType ) ) { CurrentBlock -> addInst ( MI , CurrentPredicate ) ; IsUse = true ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; return false ; } } if ( IsDef && ! IsUse && VCTP && ! isVCTP ( MI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; return


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; if ( auto * Preheader = ML -> getLoopPreheader ( ) ) dbgs ( ) << <STR_LIT> << Preheader -> getName ( ) << <STR_LIT> ; else if ( auto * Preheader = MLI -> findLoopPreheader ( ML ) ) dbgs ( ) << <STR_LIT> << Preheader -> getName ( ) << <STR_LIT> ; else if ( auto * Preheader = MLI -> findLoopPreheader ( ML , true ) ) dbgs ( ) << <STR_LIT> << Preheader -> getName ( ) << <STR_LIT> ; for ( auto * MBB : ML -> getBlocks ( ) ) dbgs ( ) << <STR_LIT> << MBB -> getName ( ) << <STR_LIT> ; ) ; std :: function < MachineInstr * ( MachineBasicBlock * ) > SearchForStart = [ & SearchForStart ] ( MachineBasicBlock * MBB ) -> MachineInstr * { for ( auto & MI : * MBB ) { if ( isLoopStart ( MI ) ) return & MI ; } if ( MBB -> pred_size ( ) == <NUM_LIT> ) return SearchForStart ( * MBB -> pred_begin ( ) ) ; return nullptr ; } ; LowOverheadLoop LoLoop ( * ML , * MLI , * RDA , * TRI , * TII ) ; if ( LoLoop . Preheader ) LoLoop . Start = SearchForStart ( LoLoop . Preheader ) ; else return false ; for ( auto * MBB : reverse ( ML -> getBlocks ( ) ) ) { for ( auto & MI : * MBB ) { if ( MI . isDebugValue ( ) ) continue ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . Dec = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . End = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . End = LoLoop . Dec = & MI ; else if ( isLoopStart ( MI ) ) LoLoop . Start = & MI ; else if ( MI . getDesc ( ) . isCall ( ) ) { LoLoop . Revert = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } else { LoLoop . AnalyseMVEInst ( & MI ) ; } } } LLVM_DEBUG ( LoLoop . dump ( ) ) ; if ( ! LoLoop . FoundAllComponents ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false


<s> if ( Start -> getOpcode ( ) == <STR_LIT> ) { TPNumElements = Start -> getOperand ( <NUM_LIT> ) ; StartInsertPt = Start ; StartInsertBB = Start -> getParent ( ) ; } else { TPNumElements = VCTP -> getOperand ( <NUM_LIT> ) ; MCRegister NumElements = TPNumElements . getReg ( ) . asMCReg ( ) ; if ( RDA . hasLocalDefBefore ( VCTP , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( StartInsertPt != StartInsertBB -> end ( ) && ! RDA . isReachingDefLiveOut ( & * StartInsertPt , NumElements ) ) { if ( auto * ElemDef = RDA . getLocalLiveOutMIDef ( StartInsertBB , NumElements ) ) { if ( RDA . isSafeToMoveForwards ( ElemDef , & * StartInsertPt ) ) { ElemDef -> removeFromParent ( ) ; StartInsertBB -> insert ( StartInsertPt , ElemDef ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ElemDef ) ; } else if ( RDA . isSafeToMoveBackwards ( & * StartInsertPt , ElemDef ) ) { StartInsertPt -> removeFromParent ( ) ; StartInsertBB -> insertAfter ( MachineBasicBlock :: iterator ( ElemDef ) , & * StartInsertPt ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ElemDef ) ; } else { MachineOperand Operand = ElemDef -> getOperand ( <NUM_LIT> ) ; if ( isMovRegOpcode ( ElemDef -> getOpcode ( ) ) && RDA . getUniqueReachingMIDef ( ElemDef , Operand . getReg ( ) . asMCReg ( ) ) == RDA . getUniqueReachingMIDef ( & * StartInsertPt , Operand . getReg ( ) . asMCReg ( ) ) ) { TPNumElements = Operand ; NumElements = TPNumElements . getReg ( ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return false ; } } } } auto CannotProvideElements = [ this ] ( MachineBasicBlock * MBB , MCRegister NumElements ) { if ( MBB -> empty ( ) ) return false ; if ( RDA . hasLocalDefBefore ( & MBB -> back ( ) , NumElements ) ) return true ; if ( MBB -> pred_size ( ) > <NUM_LIT> ) return true ; return false ; } ; MachineBasicBlock * MBB = Preheader ; while ( MBB && MBB != StartInsertBB ) { if ( CannotProvideElements ( MBB , NumElements )


<s> MIB . add ( Count ) ; if ( ! IsDo ) MIB . add ( Start -> getOperand ( <NUM_LIT> ) ) ; LoLoop . ToRemove . insert ( Start ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; return &


<s> if ( ! IsTailPredicationLegal ( ) ) return IsDo ? <STR_LIT> : <STR_LIT> ; return VCTPOpcodeToLSTP ( VCTPs . back ( ) -> getOpcode ( ) , IsDo


<s> if ( ! IsTailPredicationLegal ( ) ) return IsDo ? <STR_LIT> : <STR_LIT> ; return VCTPOpcodeToLSTP ( VCTPs . back ( ) -> getOpcode


<s> return MI -> findRegisterUseOperandIdx ( <STR_LIT> )


<s> if ( isEntryPredicatedOnVCTP ( Block , false ) || hasImplicitlyValidVPT ( Block , RDA ) ) continue ; SmallVectorImpl < MachineInstr * > & Insts = Block . getInsts ( ) ; for ( auto * MI : Insts ) { if ( isVCTP ( MI ) && getVPTInstrPredicate ( * MI ) != <STR_LIT> ) return false ; if ( MI -> getOpcode ( ) == <STR_LIT> || isVectorPredicate ( MI ) ) continue ; if ( ! isPredicatedOnVCTP ( MI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; return


<s> } if ( MBB -> pred_size ( ) == <NUM_LIT> ) return SearchForStart ( * MBB -> pred_begin ( ) ) ; return nullptr ; } ; LowOverheadLoop LoLoop ( * ML , * MLI , * RDA , * TRI , * TII ) ; if ( LoLoop . Preheader ) LoLoop . Start = SearchForStart ( LoLoop . Preheader ) ; else return false ; for ( auto * MBB : reverse ( ML -> getBlocks ( ) ) ) { for ( auto & MI : * MBB ) { if ( MI . isDebugValue ( ) ) continue ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . Dec = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . End = & MI ; else if ( isLoopStart ( MI ) ) LoLoop . Start = & MI ; else if ( MI . getDesc ( )


<s> } bool SetFlags = RDA -> isSafeToDefRegAt ( MI , MCRegister :: from ( <STR_LIT> ) , Ignore ) ; MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . addDef ( <STR_LIT> ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg


<s> return isDomainMVE ( & MI ) || isVectorPredicate ( & MI


<s> return false ; } if ( Start -> getOpcode ( ) == <STR_LIT> && ( BBUtils -> getOffsetOf ( Start ) > BBUtils -> getOffsetOf ( Start -> getOperand ( <NUM_LIT> ) . getMBB ( ) ) || ! BBUtils -> isBBInRange ( Start , Start -> getOperand ( <NUM_LIT> ) . getMBB ( ) , <NUM_LIT> ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } return true ; } ; auto FindStartInsertionPoint = [ ] ( MachineInstr * Start , MachineInstr * Dec , MachineBasicBlock :: iterator & InsertPt , MachineBasicBlock * & InsertBB , ReachingDefAnalysis & RDA , InstSet & ToRemove ) { if ( Start -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) { InsertPt = MachineBasicBlock :: iterator ( Start ) ; InsertBB = Start -> getParent ( ) ; return true ; } Register CountReg = Start -> getOperand ( <NUM_LIT> ) . getReg ( ) ; auto IsMoveLR = [ & CountReg ] ( MachineInstr * MI ) { return MI -> getOpcode ( ) == <STR_LIT> && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == CountReg && MI -> getOperand ( <NUM_LIT> ) . getImm ( ) == <STR_LIT> ; } ; if ( auto * LRDef = RDA . getUniqueReachingMIDef ( Start , MCRegister :: from ( <STR_LIT> ) ) ) { if ( IsMoveLR ( LRDef ) && RDA . hasSameReachingDef ( Start , LRDef , CountReg . asMCReg ( ) ) ) { SmallPtrSet < MachineInstr * , <NUM_LIT> > Ignore = { Dec } ; if ( ! TryRemove ( LRDef , RDA , ToRemove , Ignore ) ) return false ; InsertPt = MachineBasicBlock :: iterator ( Start ) ; InsertBB = Start -> getParent ( ) ; return true ; } } MachineBasicBlock * MBB = Start -> getParent ( ) ; if ( auto * LRDef = RDA . getLocalLiveOutMIDef ( MBB , MCRegister :: from ( <STR_LIT> ) ) ) { if ( IsMoveLR ( LRDef ) && RDA . hasSameReachingDef ( Start , LRDef , CountReg ) ) { SmallPtrSet < MachineInstr * , <NUM_LIT> > Ignore = { Start , Dec } ; if ( ! TryRemove ( LRDef , RDA , ToRemove , Ignore ) ) return false ; InsertPt = MachineBasicBlock :: iterator ( LRDef ) ; InsertBB = LRDef -> getParent ( ) ; return true ; } } if ( ! RDA . isSafeToDefRegAt ( Start , MCRegister :: from ( <STR_LIT> ) ) ) return false ; InsertPt = MachineBasicBlock :: iterator ( Start ) ; InsertBB = Start -> getParent (


<s> return true ; } MachineInstr * Prev = VCTPs . back ( ) ; if ( ! Prev -> getOperand ( <NUM_LIT> ) . isIdenticalTo ( MI -> getOperand ( <NUM_LIT> ) ) || ! RDA . hasSameReachingDef ( Prev , MI , MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } VCTPs . push_back ( MI ) ; return


<s> for ( auto & MI : * MBB ) { if ( MI . isDebugValue ( ) ) continue ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . Dec = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . End = & MI ; else if ( isLoopStart ( MI ) ) LoLoop . Start = & MI ; else if ( MI . getDesc ( ) . isCall ( ) ) { LoLoop . Revert = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } else { LoLoop . AnalyseMVEInst ( & MI ) ; } } } LLVM_DEBUG ( LoLoop . dump ( ) ) ; if ( ! LoLoop . FoundAllComponents ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } SmallPtrSet < MachineInstr * , <NUM_LIT> > Uses ; RDA -> getReachingLocalUses ( LoLoop . Dec , <STR_LIT> , Uses ) ; if ( Uses . size ( ) > <NUM_LIT> || ! Uses . count ( LoLoop . End ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LoLoop . Revert = true ; } LoLoop . Validate ( BBUtils . get ( ) ) ; Expand ( LoLoop ) ; return


<s> std :: map < MachineInstr * , SmallPtrSet < MachineInstr * , <NUM_LIT> >> ITBlocks ; for ( auto * MBB : BasicBlocks ) { for ( auto & IT : * MBB ) { if ( IT . getOpcode ( ) != <STR_LIT> ) continue ; RDA . getReachingLocalUses ( & IT , <STR_LIT> , ITBlocks [ & IT ] ) ; } } SmallPtrSet < MachineInstr * , <NUM_LIT> > ModifiedITs ; SmallPtrSet < MachineInstr * , <NUM_LIT> > RemoveITs ; for ( auto * Dead : Killed ) { if ( MachineOperand * MO = Dead -> findRegisterUseOperand ( <STR_LIT> ) ) { MachineInstr * IT = RDA . getMIOperand ( Dead , * MO ) ; RemoveITs . insert ( IT ) ; auto & CurrentBlock = ITBlocks [ IT ] ; CurrentBlock . erase ( Dead ) ; if ( CurrentBlock . empty ( ) ) ModifiedITs . erase ( IT ) ; else ModifiedITs . insert ( IT ) ; } } if ( ! ModifiedITs . empty ( ) ) return false ; Killed . insert ( RemoveITs . begin ( ) , RemoveITs . end ( ) ) ; return true ; } ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Uses ; if ( ! RDA . isSafeToRemove ( MI , Uses , Ignore ) ) return false ; if ( WontCorruptITs ( Uses , RDA ) ) { ToRemove . insert ( Uses . begin ( ) , Uses . end ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI << <STR_LIT> ; for ( auto * Use : Uses ) dbgs ( ) << <STR_LIT> << * Use ) ; SmallPtrSet < MachineInstr * , <NUM_LIT>


<s> return true ; } ; auto FindStartInsertionPoint = [ ] ( MachineInstr * Start , MachineInstr * Dec , MachineBasicBlock :: iterator & InsertPt , MachineBasicBlock * & InsertBB , ReachingDefAnalysis & RDA , InstSet & ToRemove ) { if ( Start -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) { InsertPt = MachineBasicBlock :: iterator ( Start ) ; InsertBB = Start -> getParent ( ) ; return true ; } unsigned CountReg = Start -> getOperand ( <NUM_LIT> ) . getReg ( ) ; auto IsMoveLR = [ & CountReg ] ( MachineInstr * MI ) { return MI -> getOpcode ( ) == <STR_LIT> && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == CountReg && MI -> getOperand ( <NUM_LIT> ) . getImm ( ) == <STR_LIT> ; } ; if ( auto * LRDef = RDA . getUniqueReachingMIDef ( Start , <STR_LIT> ) ) { if ( IsMoveLR ( LRDef ) && RDA . hasSameReachingDef ( Start , LRDef , CountReg ) ) { SmallPtrSet < MachineInstr * , <NUM_LIT> > Ignore = { Dec } ; if ( ! TryRemove ( LRDef , RDA , ToRemove , Ignore ) ) return false ; InsertPt = MachineBasicBlock :: iterator ( Start ) ; InsertBB = Start -> getParent ( ) ; return true ; } } MachineBasicBlock * MBB = Start -> getParent ( ) ; if ( auto * LRDef = RDA . getLocalLiveOutMIDef ( MBB , <STR_LIT> ) ) { if ( IsMoveLR ( LRDef ) && RDA . hasSameReachingDef ( Start , LRDef , CountReg ) ) { SmallPtrSet < MachineInstr * , <NUM_LIT> > Ignore = { Start , Dec } ; if ( ! TryRemove ( LRDef , RDA , ToRemove , Ignore ) ) return false ; InsertPt = MachineBasicBlock :: iterator ( LRDef ) ; InsertBB = LRDef -> getParent ( ) ; return true ; } } if ( ! RDA . isSafeToDefRegAt ( Start , <STR_LIT> ) ) return false ; InsertPt = MachineBasicBlock :: iterator ( Start ) ; InsertBB = Start -> getParent ( ) ; return true ; } ; auto TryAdjustInsertionPoint = [ ] ( MachineBasicBlock :: iterator & InsertPt , MachineInstr * Start , ReachingDefAnalysis & RDA ) { MachineBasicBlock * MBB = InsertPt -> getParent ( ) ; MachineBasicBlock :: iterator FirstNonTerminator = MBB -> getFirstTerminator ( ) ; unsigned CountReg = Start -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( FirstNonTerminator == MBB -> end ( ) ) { if ( RDA . isReachingDefLiveOut ( Start , CountReg ) && RDA . isReachingDefLiveOut ( Start , <STR_LIT> ) )


<s> for ( auto * MI : reverse ( FalseLanesUnknown ) ) { for ( auto & MO : MI -> operands ( ) ) { if ( ! isRegInClass ( MO , QPRs ) || ! MO . isDef ( ) ) continue ; if ( ! HasPredicatedUsers ( MI , MO , Predicated ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << TRI . getRegAsmName ( MO . getReg ( ) ) << <STR_LIT> << * MI ) ; NonPredicated . insert ( MI ) ; break ; } } if ( ! NonPredicated . contains ( MI ) ) Predicated . insert ( MI ) ; } SmallPtrSet < MachineInstr * , <NUM_LIT> > LiveOutMIs ; SmallVector < MachineBasicBlock * , <NUM_LIT> > ExitBlocks ; ML . getExitBlocks ( ExitBlocks ) ; assert ( ML . getNumBlocks ( ) == <NUM_LIT> && <STR_LIT> ) ; assert ( ExitBlocks . size ( ) == <NUM_LIT> && <STR_LIT> ) ; MachineBasicBlock * ExitBB = ExitBlocks . front ( ) ; for ( const MachineBasicBlock :: RegisterMaskPair & RegMask : ExitBB -> liveins ( ) ) { if ( RegMask . PhysReg == <STR_LIT> ) return false ; if ( QPRs -> contains ( RegMask . PhysReg ) ) if ( auto * MI = RDA . getLocalLiveOutMIDef ( Header , RegMask . PhysReg ) ) LiveOutMIs . insert ( MI ) ; } for ( auto * MI : LiveOutMIs ) { if ( NonPredicated . count ( MI ) && FalseLanesUnknown . contains ( MI ) ) { LLVM_DEBUG ( dbgs ( ) <<


<s> } } } MachineInstr * VCTP = VCTPs . back ( ) ; TPNumElements = VCTP -> getOperand ( <NUM_LIT> ) ; Register NumElements = TPNumElements . getReg ( ) ; if ( RDA . hasLocalDefBefore ( VCTP , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto CannotInsertWDLSTPBetween = [ ] ( MachineBasicBlock :: iterator I , MachineBasicBlock :: iterator E ) { for ( ; I != E ; ++ I ) { if ( shouldInspect ( * I ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << * I ) ; return true ; } } return false ; } ; if ( CannotInsertWDLSTPBetween ( StartInsertPt , StartInsertBB -> end ( ) ) ) return false ; auto CannotProvideElements = [ this ] ( MachineBasicBlock * MBB , Register NumElements ) { if ( RDA . hasLocalDefBefore ( & MBB -> back ( ) , NumElements ) ) return true ; if ( MBB -> pred_size ( ) > <NUM_LIT> ) return true ; return false ; } ; MachineBasicBlock * MBB = Preheader ; while ( MBB && MBB != StartInsertBB ) { if ( CannotProvideElements ( MBB , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MBB = * MBB -> pred_begin ( ) ; } auto IsValidSub = [ ] ( MachineInstr * MI , int ExpectedVecWidth ) { return - getAddSubImmediate ( * MI ) == ExpectedVecWidth ; } ; MBB = VCTP -> getParent ( ) ; if ( auto * Def = RDA . getUniqueReachingMIDef ( & MBB -> back ( ) , VCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { SmallPtrSet < MachineInstr * , <NUM_LIT> > ElementChain ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Ignore ; unsigned ExpectedVectorWidth = getTailPredVectorWidth ( VCTP -> getOpcode ( ) ) ; Ignore . insert ( VCTPs . begin ( ) , VCTPs . end ( ) ) ; if ( TryRemove ( Def , RDA , ElementChain , Ignore ) ) { bool FoundSub = false ; for ( auto * MI : ElementChain ) { if ( isMovRegOpcode ( MI -> getOpcode ( ) ) ) continue ; if ( isSubImmOpcode ( MI -> getOpcode ( ) ) ) { if ( FoundSub || ! IsValidSub ( MI


<s> if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { return false ; } if ( isVCTP ( MI ) && ! AddVCTP ( MI ) ) return false ; const MCInstrDesc & MCID = MI -> getDesc ( ) ; bool IsUse = false ; unsigned LastOpIdx = MI -> getNumOperands ( ) - <NUM_LIT> ; for ( auto & Op : enumerate ( reverse ( MCID . operands ( ) ) ) ) { const MachineOperand & MO = MI -> getOperand ( LastOpIdx - Op . index ( ) ) ; if ( ! MO . isReg ( ) || ! MO . isUse ( ) || MO . getReg ( ) != <STR_LIT> ) continue ; if ( <STR_LIT> ( Op . value ( ) . OperandType ) ) { VPTState :: addInst ( MI ) ; IsUse = true ; } else if ( MI -> getOpcode ( ) != <STR_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; return false ; } } bool RequiresExplicitPredication = ( MCID . TSFlags & <STR_LIT> ) == <NUM_LIT> ; if ( isDomainMVE ( MI ) && RequiresExplicitPredication ) { if ( ! IsUse && producesDoubleWidthResult ( * MI ) ) { DoubleWidthResultInstrs . insert ( MI ) ; return


<s> MachineBasicBlock * MBB = Preheader ; while ( MBB && MBB != StartInsertBB ) { if ( CannotProvideElements ( MBB , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MBB = * MBB -> pred_begin ( ) ; } auto IsValidSub = [ ] ( MachineInstr * MI , int ExpectedVecWidth ) { return - getAddSubImmediate ( * MI ) == ExpectedVecWidth ; } ; MBB = VCTP -> getParent ( ) ; if ( auto * Def = RDA . getUniqueReachingMIDef ( & MBB -> back ( ) , VCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { SmallPtrSet < MachineInstr * , <NUM_LIT> > ElementChain ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Ignore ; unsigned ExpectedVectorWidth = getTailPredVectorWidth ( VCTP -> getOpcode ( ) ) ; Ignore . insert ( VCTPs . begin ( ) , VCTPs . end ( ) ) ; if ( TryRemove ( Def , RDA , ElementChain , Ignore ) ) { bool FoundSub = false ; for ( auto * MI : ElementChain ) { if ( isMovRegOpcode ( MI -> getOpcode ( ) ) ) continue ; if ( isSubImmOpcode ( MI -> getOpcode ( ) ) ) { if ( FoundSub || ! IsValidSub ( MI , ExpectedVectorWidth ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> << * MI ) ; return false ; } FoundSub = true ; }


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; MachineBasicBlock * MBB = MI -> getParent ( ) ; MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . addReg ( <STR_LIT> ) ; MIB . addImm ( <NUM_LIT> ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT> ) ; MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT>


<s> MachineBasicBlock * MBB = MI -> getParent ( ) ; MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <NUM_LIT> ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT> ) ; MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg (


<s> MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <NUM_LIT> ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT> ) ; MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT> ) ; MI -> eraseFromParent ( )


<s> return & * MIB ; } ; auto RemoveDeadBranch = [ ] ( MachineInstr * I ) { MachineBasicBlock * BB = I -> getParent ( ) ; MachineInstr * Terminator = & BB -> instr_back ( ) ; if ( Terminator -> isUnconditionalBranch ( ) && I != Terminator ) { MachineBasicBlock * Succ = Terminator -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( BB -> isLayoutSuccessor ( Succ ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Terminator ) ; Terminator -> eraseFromParent ( ) ; } } } ; if ( Revert ) { if ( Start -> getOpcode ( ) == <STR_LIT> ) RevertWhile ( Start ) ; else Start -> eraseFromParent ( ) ; bool FlagsAlreadySet = RevertLoopDec ( Dec , true ) ; RevertLoopEnd ( End , FlagsAlreadySet ) ; } else { Start = ExpandLoopStart ( ML , Start , InsertPt ) ; RemoveDeadBranch ( Start ) ; End = ExpandLoopEnd ( ML , Dec


<s> auto IsMoveLR = [ ] ( MachineInstr * MI , unsigned Reg ) { return MI -> getOpcode ( ) == <STR_LIT> && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == Reg && MI -> getOperand ( <NUM_LIT> ) . getImm ( ) == <STR_LIT> ; } ; MachineBasicBlock * MBB = Start -> getParent ( ) ; unsigned CountReg = Start -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * PredLRDef = SearchForDef ( Start , MBB -> rend ( ) , <STR_LIT> ) ; if ( PredLRDef && ! IsMoveLR ( PredLRDef , CountReg ) ) PredLRDef = nullptr ; MachineInstr * SuccLRDef = SearchForDef ( Start , MBB -> end ( ) , <STR_LIT> ) ; if ( SuccLRDef && ! IsMoveLR ( SuccLRDef , CountReg ) ) SuccLRDef = nullptr ; if ( SuccLRDef ) { MachineBasicBlock :: iterator End ( SuccLRDef ) ; if ( ! SearchForDef ( Start , End , CountReg ) ) { return SuccLRDef ; } else SuccLRDef = nullptr ; } if ( PredLRDef


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; MachineBasicBlock * MBB = MI -> getParent ( ) ; bool SetFlags = false ; if ( AllowFlags ) { if ( auto * Def = SearchForDef ( MI , MBB -> end ( ) , <STR_LIT> ) ) { if ( ! SearchForUse ( MI , MBB -> end ( ) , <STR_LIT> ) && Def -> getOpcode ( ) == <STR_LIT> ) SetFlags = true ; } } MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . addDef ( <STR_LIT> ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <NUM_LIT> ) ; if ( SetFlags ) { MIB . addReg ( <STR_LIT> ) ; MIB -> getOperand ( <NUM_LIT> ) . setIsDef ( true


<s> MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . addDef ( <STR_LIT> ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <NUM_LIT> ) ; if ( SetFlags )


<s> const MCInstrDesc & MCID = MI -> getDesc ( ) ; uint64_t Flags = MCID . TSFlags ; if ( ( Flags & <STR_LIT> )


<s> ) ; return false ; } } Register NumElements = VCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( RDA -> getReachingDef ( VCTP , NumElements ) >= <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MachineBasicBlock * InsertBB = InsertPt -> getParent ( ) ; if ( ! RDA -> isReachingDefLiveOut ( InsertPt , NumElements ) ) { if ( auto * ElemDef = RDA -> getLocalLiveOutMIDef ( InsertBB , NumElements ) ) { if ( IsSafeToMove < MachineBasicBlock :: reverse_iterator > ( ElemDef , InsertPt , RDA ) ) { ElemDef -> removeFromParent ( ) ; InsertBB -> insert ( MachineBasicBlock :: iterator ( InsertPt ) , ElemDef ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ElemDef ) ; } else if ( IsSafeToMove < MachineBasicBlock :: iterator > ( InsertPt , ElemDef , RDA ) ) { InsertPt -> removeFromParent ( ) ; InsertBB -> insertAfter ( MachineBasicBlock :: iterator ( ElemDef ) , InsertPt ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ElemDef ) ; } else return false ; } } auto CannotProvideElements = [ & RDA ] ( MachineBasicBlock * MBB , Register NumElements ) { if ( RDA -> getReachingDef ( & MBB -> back ( ) , NumElements ) >= <NUM_LIT> ) return true ; if ( MBB -> pred_size ( ) > <NUM_LIT> ) return true ; return false ; } ; MachineBasicBlock * MBB = MLI -> findLoopPreheader ( ML , true ) ; if ( ! MBB ) return false ; while ( MBB != InsertBB ) { if ( CannotProvideElements ( MBB , NumElements )


<s> SmallPtrSet < MachineInstr * , <NUM_LIT> > Remove ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Ignore = { LoLoop . Start , LoLoop . Dec , LoLoop . End , LoLoop . InsertPt } ; SmallVector < MachineInstr * , <NUM_LIT> > Chain = { Def } ; while ( ! Chain . empty ( ) ) { MachineInstr * MI = Chain . back ( ) ; Chain . pop_back ( ) ; MachineInstr * IT = nullptr ; if ( TII -> getPredicate ( * MI ) != <STR_LIT> ) { auto PrevMI = std :: prev ( MI -> getIterator ( ) ) ; auto NextMI = std :: next ( MI -> getIterator ( ) ) ; if ( PrevMI -> getOpcode ( ) == <STR_LIT> && TII -> getPredicate ( * NextMI ) == <STR_LIT> ) IT = & * PrevMI ; else return ; } if ( RDA -> isSafeToRemove ( MI , Remove , Ignore ) ) { for ( auto & MO : MI -> operands ( ) ) { if ( ! MO . isReg ( ) || ! MO . isUse (

) || MO . getReg ( ) == <NUM_LIT> ) continue ;


<s> for ( auto * MBB : ML -> getBlocks ( ) ) dbgs ( ) << <STR_LIT> << MBB -> getName ( ) << <STR_LIT> ; ) ; std :: function < MachineInstr * ( MachineBasicBlock * ) > SearchForStart = [ & SearchForStart ] ( MachineBasicBlock * MBB ) -> MachineInstr * { for ( auto & MI : * MBB ) { if ( isLoopStart ( MI ) ) return & MI ; } if ( MBB -> pred_size ( ) == <NUM_LIT> ) return SearchForStart ( * MBB -> pred_begin ( ) ) ; return nullptr ; } ; LowOverheadLoop LoLoop ( * ML , * MLI , * RDA ) ; if ( auto * Preheader = ML -> getLoopPreheader ( ) ) LoLoop . Start = SearchForStart ( Preheader ) ; else if ( auto * Preheader = MLI -> findLoopPreheader ( ML , true ) ) LoLoop . Start = SearchForStart ( Preheader ) ; else return false ; for ( auto * MBB : reverse ( ML -> getBlocks ( ) ) ) { for ( auto & MI : * MBB ) { if ( MI . isDebugValue ( ) ) continue ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . Dec = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . End = & MI ; else if ( isLoopStart ( MI ) ) LoLoop . Start = & MI ; else if ( MI . getDesc ( ) . isCall ( ) ) { LoLoop . Revert = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } else { LoLoop . AnalyseMVEInst ( & MI ) ; } } } LLVM_DEBUG ( LoLoop . dump ( ) ) ; if ( ! LoLoop . FoundAllComponents ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } SmallPtrSet < MachineInstr * , <NUM_LIT> > Uses ; RDA -> getReachingLocalUses ( LoLoop . Dec , <STR_LIT> , Uses ) ; if ( Uses . size ( ) > <NUM_LIT> || ! Uses . count ( LoLoop . End ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LoLoop . Revert = true ; } LoLoop . CheckLegality ( BBUtils . get (


<s> } if ( MBB -> pred_size ( ) == <NUM_LIT> ) return SearchForStart ( * MBB -> pred_begin ( ) ) ; return nullptr ; } ; LowOverheadLoop LoLoop ( * ML , * MLI , * RDA ) ; if ( auto * Preheader = ML -> getLoopPreheader ( ) ) LoLoop . Start = SearchForStart ( Preheader ) ; else if ( auto * Preheader = MLI -> findLoopPreheader ( ML , true ) ) LoLoop . Start = SearchForStart ( Preheader ) ; else return false ; for ( auto * MBB : reverse ( ML -> getBlocks ( ) ) ) { for ( auto & MI : * MBB ) { if ( MI . isDebugValue ( ) ) continue ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . Dec = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . End = & MI ; else if ( isLoopStart ( MI ) ) LoLoop . Start = & MI ; else if ( MI . getDesc ( ) . isCall ( ) ) { LoLoop . Revert = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } else { LoLoop . AnalyseMVEInst ( & MI ) ; } } } LLVM_DEBUG ( LoLoop . dump ( ) ) ; if ( ! LoLoop . FoundAllComponents ( ) ) { LLVM_DEBUG ( dbgs ( ) <<


<s> } while ( MBB != InsertBB ) { if ( CannotProvideElements ( MBB , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MBB = * MBB -> pred_begin ( ) ; } auto IsValidSub = [ ] ( MachineInstr * MI , unsigned ExpectedVecWidth ) { unsigned ImmOpIdx = <NUM_LIT> ; switch ( MI -> getOpcode ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case <STR_LIT> : case <STR_LIT> : ImmOpIdx = <NUM_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : ImmOpIdx = <NUM_LIT> ; break ; } return MI -> getOperand ( ImmOpIdx ) . getImm ( ) == ExpectedVecWidth ; } ; MBB = VCTP -> getParent ( ) ; if ( MachineInstr * Def = RDA . getReachingMIDef ( & MBB -> back ( ) , NumElements ) ) { SmallPtrSet < MachineInstr * , <NUM_LIT> > ElementChain ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Ignore = { VCTP } ; unsigned ExpectedVectorWidth = getTailPredVectorWidth ( VCTP -> getOpcode ( ) ) ; if ( RDA . isSafeToRemove ( Def , ElementChain , Ignore ) ) { bool FoundSub = false ; for ( auto * MI : ElementChain ) { if ( isMovRegOpcode ( MI -> getOpcode ( ) ) ) continue ; if ( isSubImmOpcode ( MI -> getOpcode ( ) ) ) { if ( FoundSub || ! IsValidSub ( MI ,


<s> if ( ! ST . hasLOB ( ) ) return false ; MF = & mf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MF -> getName ( ) << <STR_LIT> ) ; MLI = & getAnalysis < MachineLoopInfo > ( ) ; RDA = & getAnalysis < ReachingDefAnalysis > ( ) ; MF -> getProperties ( ) . set ( MachineFunctionProperties :: Property :: TracksLiveness ) ; MRI = & MF -> getRegInfo ( ) ; TII = static_cast < const <ISA_LIT>BaseInstrInfo * > ( ST . getInstrInfo ( ) ) ; BBUtils = std :: unique_ptr < <ISA_LIT>BasicBlockUtils > ( new <ISA_LIT>BasicBlockUtils ( * MF ) ) ; BBUtils -> computeAllBlockSizes ( ) ; BBUtils -> adjustBBOffsetsAfter ( & MF -> front ( ) ) ; bool Changed = false ; for ( auto ML : * MLI ) { if ( ! ML -> getParentLoop ( ) ) Changed


<s> if ( ! End -> getOperand ( <NUM_LIT> ) . isMBB ( ) ) report_fatal_error ( <STR_LIT> ) ; if ( End -> getOperand ( <NUM_LIT> ) . getMBB ( ) != ML -> getHeader ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Revert = true ; return ; } if ( BBUtils -> getOffsetOf ( End ) < BBUtils -> getOffsetOf ( ML -> getHeader ( ) ) || ! BBUtils -> isBBInRange ( End , ML -> getHeader ( ) , <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Revert = true ; return ; } if ( Start -> getOpcode ( ) == <STR_LIT> && ( BBUtils -> getOffsetOf ( Start ) > BBUtils -> getOffsetOf ( Start -> getOperand ( <NUM_LIT> ) . getMBB ( ) ) || ! BBUtils -> isBBInRange ( Start , Start -> getOperand ( <NUM_LIT> ) . getMBB ( ) , <NUM_LIT> ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Revert =


<s> if ( BBUtils -> getOffsetOf ( End ) < BBUtils -> getOffsetOf ( ML -> getHeader ( ) ) || ! BBUtils -> isBBInRange ( End , ML -> getHeader ( ) , <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Revert = true ; return ; } if ( Start -> getOpcode ( ) == <STR_LIT> && ( BBUtils -> getOffsetOf ( Start ) > BBUtils -> getOffsetOf ( Start -> getOperand ( <NUM_LIT> ) . getMBB ( ) ) || ! BBUtils -> isBBInRange

( Start , Start -> getOperand ( <NUM_LIT> ) . getMBB ( ) , <NUM_LIT> ) ) ) {


<s> if ( Terminator -> isUnconditionalBranch ( ) && I != Terminator ) { MachineBasicBlock * Succ = Terminator -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( BB -> isLayoutSuccessor ( Succ ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Terminator ) ; Terminator -> eraseFromParent ( ) ; } } } ; if ( LoLoop . Revert ) { if ( LoLoop . Start -> getOpcode ( ) == <STR_LIT> ) RevertWhile ( LoLoop . Start ) ; else LoLoop . Start -> eraseFromParent ( ) ; bool FlagsAlreadySet = RevertLoopDec ( LoLoop . Dec , true ) ; RevertLoopEnd ( LoLoop . End , FlagsAlreadySet ) ; } else { LoLoop . Start = ExpandLoopStart ( LoLoop ) ; RemoveDeadBranch ( LoLoop . Start ) ; LoLoop . End = ExpandLoopEnd


<s> SmallVector < MachineInstr * , <NUM_LIT> > Killed ; SmallVector < MachineInstr * , <NUM_LIT> > Dead ; if ( auto * Def = RDA -> getReachingMIDef ( LoLoop . Start , LoLoop . Start -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { SmallPtrSet < MachineInstr * , <NUM_LIT> > Visited ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Remove ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Ignore = { LoLoop . Start , LoLoop . Dec , LoLoop . End , LoLoop . VCTP , LoLoop . InsertPt } ; SmallVector < MachineInstr * , <NUM_LIT> > Chain = { Def } ; while ( ! Chain . empty ( ) ) { MachineInstr * MI = Chain . back ( ) ; Chain . pop_back ( ) ; if ( IsSafeToRemove ( MI , RDA , Visited , Remove , Ignore ) ) { for ( auto & MO : MI -> operands ( ) ) { if ( ! MO . isReg ( ) || ! MO . isUse ( ) || MO . getReg ( ) == <NUM_LIT> ) continue ; if ( auto * Op = RDA -> getReachingMIDef ( MI , MO . getReg ( ) ) ) Chain . push_back ( Op ) ; } Ignore . insert ( MI ) ; } } LoLoop . ToRemove . insert ( Remove . begin ( ) , Remove . end ( ) ) ; } } MachineInstr * InsertPt = LoLoop . InsertPt ; MachineInstr * Start = LoLoop .


<s> return MI -> getOpcode ( ) == <STR_LIT> && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == CountReg && MI -> getOperand ( <NUM_LIT> ) . getImm ( ) == <STR_LIT> ; } ; MachineBasicBlock * MBB = Start -> getParent ( ) ; if ( auto * LRDef = RDA -> getReachingMIDef ( Start , <STR_LIT> ) ) if ( IsMoveLR ( LRDef ) && RDA -> hasSameReachingDef ( Start , LRDef , CountReg ) ) return LRDef ; if ( auto * LRDef = RDA -> getLocalLiveOutMIDef ( MBB , <STR_LIT> ) ) if ( IsMoveLR ( LRDef ) && RDA -> hasSameReachingDef ( Start , LRDef , CountReg ) ) return LRDef ; if ( ! RDA -> isRegUsedAfter ( Start , <STR_LIT> ) ) return Start ; return nullptr


<s> LowOverheadLoop ( MachineLoop * ML , MachineLoopInfo * MLI , ReachingDefAnalysis * RDA ) : ML ( ML ) , MLI ( MLI ) , RDA ( RDA ) { MF = ML -> getHeader ( ) ->


<s> LowOverheadLoop ( MachineLoop * ML , MachineLoopInfo * MLI , ReachingDefAnalysis * RDA ) : ML


<s> LLVM_DEBUG ( if ( Start ) dbgs ( ) << <STR_LIT> << * Start ; if ( Dec ) dbgs ( ) << <STR_LIT> << * Dec ; if ( End ) dbgs ( ) << <STR_LIT> << * End ; ) ; if ( ! Start && ! Dec && ! End ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return Changed ; } else if ( ! ( Start && Dec && End ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( ! End -> getOperand ( <NUM_LIT> ) . isMBB ( ) || End -> getOperand ( <NUM_LIT> ) . getMBB ( ) != ML -> getHeader ( ) ) report_fatal_error ( <STR_LIT> ) ; if ( BBUtils -> getOffsetOf ( End ) < BBUtils -> getOffsetOf ( ML -> getHeader ( ) ) || ! BBUtils -> isBBInRange ( End , ML -> getHeader ( ) , <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Revert = true ; } if ( Start -> getOpcode ( ) == <STR_LIT> && ( BBUtils -> getOffsetOf ( Start ) > BBUtils -> getOffsetOf ( Start -> getOperand ( <NUM_LIT> ) . getMBB ( ) ) || ! BBUtils -> isBBInRange ( Start , Start -> getOperand ( <NUM_LIT> ) . getMBB ( ) , <NUM_LIT> ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Revert = true


<s> else if ( MI . getOpcode ( ) == <STR_LIT> ) End = & MI ; else if ( IsLoopStart ( MI ) ) Start = & MI ; else if ( MI . getDesc ( ) . isCall ( ) ) Revert = true ; if ( ! Dec ) continue ; if ( MI . mayLoad ( ) || MI . mayStore ( ) ) Revert = MI . getOperand ( <NUM_LIT> ) . isReg ( ) && MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ; } if ( Dec && End && Revert ) break ; } LLVM_DEBUG ( if ( Start ) dbgs ( ) << <STR_LIT> << * Start ; if ( Dec ) dbgs ( ) << <STR_LIT> << * Dec ; if ( End ) dbgs ( ) << <STR_LIT> << * End ; ) ; if ( ! Start && ! Dec && ! End ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return Changed ; } else if ( ! ( Start && Dec && End ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( ! End -> getOperand ( <NUM_LIT> ) . isMBB ( ) || End -> getOperand ( <NUM_LIT> ) . getMBB ( ) != ML -> getHeader ( ) ) report_fatal_error ( <STR_LIT> ) ; if ( BBUtils -> getOffsetOf ( End ) < BBUtils -> getOffsetOf ( ML -> getHeader ( ) ) || ! BBUtils -> isBBInRange ( End , ML -> getHeader ( ) , <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Revert = true ; } if ( Start -> getOpcode ( ) == <STR_LIT> && ( BBUtils -> getOffsetOf ( Start ) > BBUtils -> getOffsetOf ( Start -> getOperand ( <NUM_LIT> ) . getMBB ( ) ) || ! BBUtils -> isBBInRange ( Start , Start -> getOperand ( <NUM_LIT> ) . getMBB


<s> else return false ; for ( auto * MBB : reverse ( ML -> getBlocks ( ) ) ) { for ( auto & MI : * MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . Dec = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . End = & MI ; else if ( isLoopStart ( MI ) ) LoLoop . Start = & MI ; else if ( MI . getDesc ( ) . isCall ( ) ) { LoLoop . Revert = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } else { LoLoop . AnalyseMVEInst ( & MI ) ; } } } LLVM_DEBUG ( LoLoop . dump ( ) ) ; if ( ! LoLoop . FoundAllComponents ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } SmallPtrSet < MachineInstr * , <NUM_LIT> > Visited ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Ignore = { LoLoop . End } ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Remove ; if ( ! IsSafeToRemove ( LoLoop . Dec , RDA , Visited ,


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; MachineBasicBlock * Pred = nullptr ; for ( auto * MBB : ML -> getHeader ( ) -> predecessors ( ) ) { if ( ! ML -> contains ( MBB ) ) { if ( Pred ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Start = nullptr ; break ; } Pred = MBB ; Start = SearchForStart ( MBB ) ; } } } for ( auto * MBB : reverse ( ML -> getBlocks ( ) ) ) { for ( auto & MI : * MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) Dec = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) End = & MI ; else if ( MI . getDesc ( ) . isCall ( ) ) Revert = true ; if ( ! Dec ) continue ; if ( MI . mayLoad ( ) || MI . mayStore ( ) ) Revert = MI . getOperand ( <NUM_LIT> ) . isReg ( ) && MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ; } if ( Dec && End && Revert ) break ; } if ( ! Start && ! Dec && ! End ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return Changed ; } if ( ! ( Start && Dec && End )


<s> } } } for ( auto * MBB : reverse ( ML -> getBlocks ( ) ) ) { for ( auto & MI : * MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) Dec = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) End = & MI ; else if ( MI . getDesc ( ) . isCall ( ) ) Revert = true ; if ( ! Dec ) continue ; if ( MI . mayLoad ( ) || MI . mayStore ( ) ) Revert = MI . getOperand ( <NUM_LIT> ) . isReg ( ) && MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ; } if ( Dec && End && Revert ) break ; } if ( ! Start && ! Dec && ! End ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return Changed ; } if ( ! ( Start && Dec && End ) ) { report_fatal_error ( <STR_LIT> ) ; } if ( ! End -> getOperand ( <NUM_LIT> ) . isMBB ( ) || End -> getOperand ( <NUM_LIT> ) . getMBB ( ) != ML -> getHeader ( ) ) report_fatal_error ( <STR_LIT> ) ; if ( ! BBUtils -> isBBInRange ( End , ML ->


<s> case <STR_LIT> : { return I -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && I -> memoperands ( ) . size ( ) == <NUM_LIT> && GetFrameIndex ( I -> memoperands ( ) . front ( ) ) >= <NUM_LIT> ; } default : return false ; } } ; if ( MI -> getOpcode ( ) != <STR_LIT> || ! IsStackOp ( MI ) ) return false ; if ( MI -> memoperands ( ) . size ( ) == <NUM_LIT> ) return false ; int FI = GetFrameIndex ( MI -> memoperands ( ) . front ( ) ) ; MachineFrameInfo FrameInfo = MI -> getParent ( ) -> getParent ( ) -> getFrameInfo ( ) ; if ( FI == - <NUM_LIT> || ! FrameInfo . isSpillSlotObjectIndex ( FI ) ) return false ; SmallVector < MachineBasicBlock * > Frontier ; ML -> getExitBlocks (


<s> ML -> getExitBlocks ( Frontier ) ; SmallPtrSet < MachineBasicBlock * , <NUM_LIT> > Visited { MI -> getParent ( ) } ; unsigned Idx = <NUM_LIT> ; while ( Idx < Frontier . size ( ) ) { MachineBasicBlock * BB = Frontier [ Idx ] ; bool LookAtSuccessors = true ; for ( auto & I : * BB ) { if ( ! IsStackOp ( & I ) || I . memoperands ( ) . size ( ) == <NUM_LIT> ) continue ; if ( GetFrameIndex ( I . memoperands ( ) . front ( ) ) != FI ) continue ; if ( I . getOpcode ( ) == <STR_LIT> ) { LookAtSuccessors = false ; break ; } if ( I . getOpcode ( ) == <STR_LIT> ) return false ; } if ( LookAtSuccessors ) { for ( auto Succ : BB -> successors ( ) ) { if ( ! Visited . contains ( Succ ) && ! is_contained (

Frontier , Succ ) ) Frontier . push_back ( Succ ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MF . getName ( ) << <STR_LIT> ) ; auto & MLI = getAnalysis < MachineLoopInfo > ( ) ; MRI = & MF . getRegInfo ( ) ; TII = static_cast < const <ISA_LIT>BaseInstrInfo * > ( MF . getSubtarget ( ) . getInstrInfo ( ) ) ; BBUtils = std :: unique_ptr < <ISA_LIT>BasicBlockUtils > ( new <ISA_LIT>BasicBlockUtils ( MF ) ) ; BBUtils -> computeAllBlockSizes ( )


<s> PredicatedInsts . emplace ( MI , std :: make_unique <

PredicatedMI > ( MI , CurrentPredicates ) ) ;


<s> bool LowOverheadLoop :: AddVCTP ( MachineInstr * MI ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; if ( VCTPs . empty ( ) ) { VCTPs . push_back ( MI ) ; return true ; } MachineInstr * Prev = VCTPs . back ( ) ; if ( ! Prev -> getOperand ( <NUM_LIT> ) . isIdenticalTo ( MI -> getOperand ( <NUM_LIT> ) ) || ! RDA . hasSameReachingDef ( Prev , MI , MI -> getOperand ( <NUM_LIT> ) . getReg ( ) . asMCReg ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false


<s> } MachineInstr * Prev = VCTPs . back ( ) ; if ( ! Prev -> getOperand ( <NUM_LIT> ) . isIdenticalTo ( MI -> getOperand ( <NUM_LIT> ) ) || ! RDA . hasSameReachingDef ( Prev , MI , MI -> getOperand ( <NUM_LIT> ) . getReg ( ) . asMCReg ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } VCTPs . push_back ( MI ) ; return true


<s> CannotTailPredicate = ! ValidateMVEInst


<s> CannotTailPredicate = ! ValidateMVEInst ( MI


<s> if ( producesDoubleWidthResult ( MI ) ) return true ; switch ( MI . getOpcode ( ) ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true


<s> MIB . add ( TheVCMP -> getOperand ( <NUM_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; LoLoop . BlockMasksToRecompute . insert ( MIB . getInstr ( ) ) ; LoLoop . ToRemove . insert ( TheVCMP ) ; TheVCMP = nullptr ; } ; if ( VPTState :: isEntryPredicatedOnVCTP ( Block , true ) ) { MachineInstr * VPST = Insts . front ( ) ; if ( VPTState :: hasUniformPredicate ( Block ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * VPST ) ; for ( unsigned i = <NUM_LIT> ; i < Insts . size ( ) ; ++ i ) RemovePredicate ( Insts [ i ] ) ; } else { MachineInstr * Divergent = VPTState :: getDivergent ( Block ) ; MachineBasicBlock * MBB = Divergent -> getParent ( ) ; auto DivergentNext = ++ MachineBasicBlock :: iterator ( Divergent ) ; while ( DivergentNext != MBB -> end ( ) && DivergentNext -> isDebugInstr ( ) ) ++ DivergentNext ; bool DivergentNextIsPredicated = DivergentNext != MBB -> end ( ) && getVPTInstrPredicate ( * DivergentNext ) != <STR_LIT> ; for ( auto I = ++ MachineBasicBlock :: iterator ( VPST ) , E = DivergentNext ; I != E ; ++ I ) RemovePredicate ( & * I ) ; MachineInstr * VCMP = VCMPOpcodeToVPT ( Divergent -> getOpcode ( ) ) != <NUM_LIT> ? Divergent : nullptr ; if ( DivergentNextIsPredicated ) { if ( ! VCMP ) { MachineInstrBuilder MIB = BuildMI ( * Divergent -> getParent ( ) , Divergent , Divergent -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . addImm ( <NUM_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; LoLoop . BlockMasksToRecompute . insert ( MIB . getInstr ( ) ) ; } else { ReplaceVCMPWithVPT ( VCMP , VCMP ) ; } } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * VPST ) ; LoLoop . ToRemove . insert ( VPST ) ; } else if ( Block . containsVCTP ( ) ) { MachineInstr * VPST = Insts . front ( ) ; if ( Block . size ( ) == <NUM_LIT> ) { assert ( VPST -> getOpcode ( ) == <STR_LIT> && <STR_LIT> ) ; LoLoop . ToRemove . insert ( VPST ) ; } else LoLoop . BlockMasksToRecompute . insert ( VPST ) ; } else if ( Insts . front ( ) -> getOpcode ( ) == <STR_LIT> ) { MachineInstr * VPST = Insts


<s> FunctionPass * llvm :: create<ISA_LIT>LowOverheadLoopsPass ( ) { return new <ISA_LIT>LowOverheadLoops (


<s> FunctionPass * llvm :: create<ISA_LIT>LowOverheadLoopsPass ( )


<s> assert ( ( CurrentPredicates . size ( ) || MI -> getParent ( ) -> isLiveIn ( <STR_LIT> ) ) && <STR_LIT> ) ; Blocks . emplace_back ( MI ) ; PredicatedInsts . emplace ( MI , std :: make_unique < PredicatedMI >


<s> if ( ! VCTPs . empty ( ) ) { dbgs ( ) << <STR_LIT> ; for ( auto * MI : VCTPs ) dbgs ( ) << <STR_LIT> << * MI ; } if ( ! FoundAllComponents ( ) )


<s> if ( ! VCTPs . empty ( ) ) { dbgs ( ) << <STR_LIT> ; for ( auto * MI : VCTPs ) dbgs ( ) << <STR_LIT> << * MI ; } if ( ! FoundAllComponents ( ) ) dbgs ( ) << <STR_LIT> ; else if ( ! ( Start && Dec && End ) ) dbgs (


<s> MachineBasicBlock * Succ = Terminator -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( BB -> isLayoutSuccessor ( Succ ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Terminator ) ; Terminator -> eraseFromParent ( ) ; } } } ; auto ExpandVMOVCopies = [ this ] ( SmallPtrSet < MachineInstr * , <NUM_LIT> > & VMOVCopies ) { for ( auto * MI : VMOVCopies ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; assert ( MI -> getOpcode ( ) == <STR_LIT> && <STR_LIT> ) ; MachineBasicBlock * MBB = MI -> getParent ( ) ; Register Dst = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Src = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; auto MIB1 = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> + ( Dst - <STR_LIT> ) * <NUM_LIT> ) . addReg ( <STR_LIT> + ( Src - <STR_LIT> ) * <NUM_LIT> ) . add ( predOps ( <STR_LIT> ) ) ; ( void ) MIB1 ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB1 ) ; auto MIB2 = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> + ( Dst - <STR_LIT> ) * <NUM_LIT> + <NUM_LIT> ) . addReg ( <STR_LIT> + ( Src - <STR_LIT> ) * <NUM_LIT> + <NUM_LIT> ) . add ( predOps ( <STR_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB2 ) ; ( void ) MIB2 ; MI -> eraseFromParent ( ) ; } } ; if ( LoLoop . Revert ) { if ( isWhileLoopStart ( * LoLoop . Start ) ) RevertWhile ( LoLoop . Start ) ; else RevertDo ( LoLoop . Start ) ; if ( LoLoop . Dec == LoLoop . End ) RevertLoopEndDec ( LoLoop . End ) ; else RevertLoopEnd ( LoLoop . End , RevertLoopDec ( LoLoop . Dec ) ) ; } else { ExpandVMOVCopies ( LoLoop . VMOVCopies ) ; LoLoop . Start = ExpandLoopStart ( LoLoop


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; IterationCountDCE ( LoLoop ) ; MachineBasicBlock :: iterator InsertPt = LoLoop . StartInsertPt ; MachineInstr * Start = LoLoop . Start ; MachineBasicBlock * MBB = LoLoop . StartInsertBB ; unsigned Opc = LoLoop . getStartOpcode ( ) ; MachineOperand & Count = LoLoop . getLoopStartOperand ( ) ; MachineInstr * NewStart ; if ( Opc == <STR_LIT> && Count . isReg ( ) && Count . getReg ( ) == <STR_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; NewStart =


<s> SmallVectorImpl < MachineInstr * > & Insts = Block . getInsts ( ) ; for ( unsigned i = <NUM_LIT> ; i < Insts . size ( ) ; ++ i ) { MachineInstr * Next = Insts [ i ] ; if ( isVectorPredicate ( Next


<s> SmallVectorImpl < MachineInstr * > & getInsts ( ) { return


<s> return Start -> getOperand ( <NUM_LIT>


<s> const SmallVectorImpl < MachineBasicBlock * > & getOrder ( ) const { return


<s> const SmallVectorImpl < MachineBasicBlock * > & getOrder ( ) const { return Order


<s> if ( ! IsTailPredicationLegal ( ) ) return IsDo ? <STR_LIT> : <STR_LIT> ; return VCTPOpcodeToLSTP ( VCTPs . back ( ) -> getOpcode ( )


<s> unsigned getStartOpcode ( ) const { bool IsDo = isDoLoopStart ( * Start ) ; if ( ! IsTailPredicationLegal ( ) ) return IsDo


<s> SmallVectorImpl < VPTState > & getVPTBlocks (


<s> return Op && PredicatedInsts . count ( Op ) && isPredicatedOnVCTP ( Op ) ; } ; auto IsOperandInvariant = [ & ] ( MachineInstr * MI , unsigned Idx ) { MachineOperand & MO = MI -> getOperand ( Idx ) ; if ( ! MO . isReg ( ) || ! MO . getReg ( ) ) return true ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Defs ; RDA . getGlobalReachingDefs ( MI , MO . getReg ( ) , Defs ) ; if ( Defs . empty ( ) ) return true ; for ( auto * Def : Defs ) if ( Def -> getParent ( ) == VPT -> getParent ( ) ) return false ; return


<s> static bool hasVPRUse ( MachineInstr & MI ) { return MI . findRegisterUseOperandIdx ( <STR_LIT> )


<s> void insert ( MachineInstr * MI ) { Insts . push_back ( MI ) ; assert ( Insts . size ( ) <= <NUM_LIT> && <STR_LIT>


<s> void insert ( MachineInstr * MI ) { Insts . push_back ( MI


<s> static bool isDomainMVE ( MachineInstr * MI ) { uint64_t Domain = MI -> getDesc ( ) . TSFlags & <STR_LIT> ; return Domain == <STR_LIT>


<s> static bool isDomainMVE ( MachineInstr * MI ) { uint64_t Domain = MI -> getDesc ( ) . TSFlags & <STR_LIT> ; return Domain


<s> SmallVectorImpl < MachineInstr * > & Insts = Block . getInsts ( ) ; return isPredicatedOnVCTP ( Insts . front ( )


<s> static bool isHorizontalReduction ( const MachineInstr & MI ) { const MCInstrDesc & MCID = MI . getDesc ( ) ; uint64_t Flags = MCID . TSFlags ; return ( Flags & <STR_LIT>


<s> static bool isHorizontalReduction ( const MachineInstr & MI ) { const MCInstrDesc & MCID = MI . getDesc ( ) ; uint64_t Flags =


<s> for ( auto * PredMI : Predicates ) if ( isVCTP ( PredMI ) ) return true ; return false


<s> return MO . isReg ( ) && MO . getReg ( )

&& Class -> contains ( MO . getReg ( ) ) ;


<s> return ! Revert && FoundAllComponents ( ) && ! VCTPs . empty ( ) && ! CannotTailPredicate && ML . getNumBlocks (


<s> bool IsTailPredicationLegal ( ) const { return ! Revert && FoundAllComponents ( ) && ! VCTPs . empty ( ) && ! CannotTailPredicate && ML . getNumBlocks


<s> static bool isVectorPredicate ( MachineInstr


<s> static bool isVectorPredicate ( MachineInstr * MI ) { return MI -> findRegisterDefOperandIdx ( <STR_LIT> ) !=


<s> return PIdx != - <NUM_LIT> && MI -> getOperand ( PIdx + <NUM_LIT> )


<s> SmallPtrSet < MachineInstr * , <NUM_LIT> > Killed = { LoLoop . Start , LoLoop . Dec , LoLoop . End } ; if ( ! TryRemove ( Def , * RDA , LoLoop . ToRemove

, Killed ) ) LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ;


<s> LowOverheadLoop ( MachineLoop & ML , MachineLoopInfo & MLI , ReachingDefAnalysis & RDA , const TargetRegisterInfo & TRI , const <ISA_LIT>BaseInstrInfo & TII ) : ML ( ML ) , MLI ( MLI )

, RDA ( RDA ) , TRI ( TRI ) , TII ( TII ) , TPNumElements ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) {


<s> else if ( auto * Preheader = MLI -> findLoopPreheader ( ML , true , true ) ) dbgs ( ) << <STR_LIT> << printMBBReference ( * Preheader ) << <STR_LIT> ; for ( auto * MBB : ML -> getBlocks ( ) ) dbgs ( ) << <STR_LIT> << printMBBReference ( * MBB ) << <STR_LIT> ; } ) ; std :: function < MachineInstr * ( MachineBasicBlock * ) > SearchForStart = [ & SearchForStart ] ( MachineBasicBlock * MBB ) -> MachineInstr * { for ( auto & MI : * MBB ) { if ( isLoopStart ( MI ) ) return & MI ; } if ( MBB -> pred_size ( ) == <NUM_LIT> ) return SearchForStart ( * MBB -> pred_begin ( ) ) ; return nullptr ; } ; LowOverheadLoop LoLoop ( * ML , * MLI , * RDA , * TRI , * TII ) ; if ( LoLoop . Preheader ) LoLoop . Start = SearchForStart ( LoLoop . Preheader ) ; else return Changed ; for ( auto * MBB : reverse ( ML -> getBlocks ( ) ) ) { for ( auto & MI : * MBB


<s> auto IsZeroInit = [ ] ( MachineInstr * Def ) { return ! isVectorPredicated ( Def ) && Def -> getOpcode ( ) == <STR_LIT> && Def -> getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> ; } ; bool AllowScalars = isHorizontalReduction ( MI ) ; for ( auto & MO : MI . operands ( ) ) { if ( ! MO . isReg ( ) || ! MO . getReg ( ) ) continue ; if ( ! isRegInClass ( MO , QPRs ) && AllowScalars ) continue ; int PIdx = llvm ::


<s> static void reset ( ) { Blocks . clear ( ) ; PredicatedInsts . clear ( ) ; CurrentPredicates . clear ( )


<s> static bool retainsPreviousHalfElement ( const MachineInstr & MI ) { const MCInstrDesc & MCID = MI . getDesc (


<s> void <ISA_LIT>LowOverheadLoops :: RevertDo ( MachineInstr * MI ) const { LLVM_DEBUG ( dbgs ( ) << <STR_LIT>


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; MachineBasicBlock * DestBB = MI -> getOperand ( <NUM_LIT> ) .


<s> MIB -> getOperand ( <NUM_LIT> ) . setIsDef ( true ) ; MachineBasicBlock * DestBB = MI -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; unsigned BrOpc = BBUtils -> isBBInRange ( MI , DestBB , <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( BrOpc ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT>


<s> void <ISA_LIT>LowOverheadLoops :: RevertWhile ( MachineInstr * MI ) const { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; MachineBasicBlock * DestBB = getWhileLoopStartTargetBB ( * MI ) ; unsigned BrOpc = BBUtils -> isBBInRange ( MI , DestBB , <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; RevertWhileLoopStartLR ( MI , TII


<s> return isDomainMVE ( & MI ) || isVectorPredicate ( &


<s> static bool TryRemove ( MachineInstr * MI , ReachingDefAnalysis & RDA , InstSet & ToRemove , InstSet & Ignore ) { auto WontCorruptITs = [ ] ( InstSet & Killed , ReachingDefAnalysis & RDA ) { SmallPtrSet < MachineBasicBlock * , <NUM_LIT> > BasicBlocks ; for ( auto * Dead : Killed ) BasicBlocks . insert ( Dead -> getParent ( ) ) ; std :: map < MachineInstr * , SmallPtrSet < MachineInstr * , <NUM_LIT> >> ITBlocks ; for ( auto * MBB : BasicBlocks ) { for ( auto & IT : * MBB ) { if ( IT . getOpcode ( ) != <STR_LIT> ) continue ; RDA . getReachingLocalUses ( & IT , MCRegister :: from ( <STR_LIT> ) , ITBlocks [ & IT ] ) ; } } SmallPtrSet < MachineInstr * , <NUM_LIT> > ModifiedITs ; SmallPtrSet < MachineInstr * , <NUM_LIT> > RemoveITs ; for ( auto * Dead : Killed ) { if ( MachineOperand * MO = Dead -> findRegisterUseOperand ( <STR_LIT> ) ) { MachineInstr * IT = RDA . getMIOperand ( Dead , * MO ) ; RemoveITs . insert ( IT ) ; auto & CurrentBlock = ITBlocks [ IT ] ; CurrentBlock . erase ( Dead ) ; if ( CurrentBlock . empty ( ) ) ModifiedITs . erase ( IT ) ; else ModifiedITs . insert ( IT ) ; } } if ( ! ModifiedITs . empty ( ) ) return false ; Killed . insert ( RemoveITs . begin ( ) , RemoveITs . end ( ) ) ; return true ; } ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Uses ; if ( ! RDA . isSafeToRemove ( MI , Uses , Ignore ) ) return false ; if ( WontCorruptITs ( Uses , RDA ) ) { ToRemove . insert ( Uses . begin ( ) , Uses . end ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI << <STR_LIT> ; for ( auto * Use : Uses ) dbgs ( ) << <STR_LIT> << * Use ) ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Killed ; RDA . collectKilledOperands ( MI , Killed ) ; if ( WontCorruptITs ( Killed , RDA ) ) { ToRemove . insert ( Killed . begin ( ) , Killed . end ( ) ) ; LLVM_DEBUG ( for ( auto * Dead : Killed ) dbgs ( ) << <STR_LIT> << * Dead ) ; } return true ; } return false


<s> SmallPtrSet < MachineBasicBlock * , <NUM_LIT> > BasicBlocks ; for ( auto * Dead : Killed ) BasicBlocks . insert ( Dead -> getParent ( ) ) ; std :: map < MachineInstr * , SmallPtrSet < MachineInstr * , <NUM_LIT> >> ITBlocks ; for ( auto * MBB : BasicBlocks ) { for ( auto & IT : * MBB ) { if ( IT . getOpcode ( ) != <STR_LIT> ) continue ; RDA . getReachingLocalUses ( & IT , MCRegister :: from ( <STR_LIT> ) , ITBlocks [ & IT ] ) ; } } SmallPtrSet < MachineInstr * , <NUM_LIT> > ModifiedITs ; SmallPtrSet < MachineInstr * , <NUM_LIT> > RemoveITs ; for ( auto * Dead : Killed ) { if ( MachineOperand * MO = Dead -> findRegisterUseOperand ( <STR_LIT> ) ) { MachineInstr * IT = RDA . getMIOperand ( Dead , * MO ) ; RemoveITs . insert ( IT ) ; auto & CurrentBlock = ITBlocks [ IT ] ; CurrentBlock . erase ( Dead ) ; if ( CurrentBlock . empty ( ) ) ModifiedITs . erase ( IT ) ; else ModifiedITs . insert ( IT ) ; } } if ( ! ModifiedITs . empty ( ) ) return false ; Killed . insert ( RemoveITs . begin ( ) , RemoveITs . end ( ) ) ; return true ; } ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Uses ; if ( ! RDA . isSafeToRemove ( MI , Uses , Ignore ) ) return false ; if ( WontCorruptITs ( Uses , RDA ) ) { ToRemove . insert ( Uses . begin ( ) , Uses . end ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI << <STR_LIT> ; for ( auto * Use : Uses ) dbgs ( ) << <STR_LIT> << * Use ) ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Killed ; RDA . collectKilledOperands ( MI , Killed


<s> if ( TgtBB != ML . getHeader ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( BBUtils -> getOffsetOf ( End ) < BBUtils -> getOffsetOf ( ML . getHeader ( ) ) || ! BBUtils -> isBBInRange ( End , ML . getHeader ( ) , <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( isWhileLoopStart ( * Start ) ) { MachineBasicBlock * TargetBB = getWhileLoopStartTargetBB ( * Start ) ; if ( BBUtils -> getOffsetOf ( Start ) > BBUtils -> getOffsetOf ( TargetBB ) || ! BBUtils -> isBBInRange ( Start , TargetBB , <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } } return true ; } ; StartInsertPt = MachineBasicBlock :: iterator ( Start ) ; StartInsertBB = Start -> getParent (


<s> if ( Revert ) return ; auto ValidateRanges = [ ] ( MachineInstr * Start , MachineInstr * End , <ISA_LIT>BasicBlockUtils * BBUtils , MachineLoop & ML ) { MachineBasicBlock * TgtBB = End -> getOpcode ( ) == <STR_LIT> ? End -> getOperand ( <NUM_LIT> ) . getMBB ( ) : End -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( TgtBB != ML . getHeader ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( BBUtils -> getOffsetOf ( End ) < BBUtils -> getOffsetOf ( ML . getHeader ( ) ) || ! BBUtils -> isBBInRange ( End , ML . getHeader ( ) , <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( isWhileLoopStart ( * Start ) ) { MachineBasicBlock * TargetBB = getWhileLoopStartTargetBB ( * Start ) ; if ( BBUtils -> getOffsetOf ( Start ) > BBUtils -> getOffsetOf ( TargetBB ) || ! BBUtils -> isBBInRange ( Start , TargetBB , <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT>


<s> SmallPtrSet < MachineInstr * , <NUM_LIT> > FalseLanesZero ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Predicated ; MachineBasicBlock * Header = ML . getHeader ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; for ( auto & MI : * Header ) { if ( ! shouldInspect ( MI ) ) continue ; if ( isVCTP ( & MI ) || isVPTOpcode ( MI . getOpcode ( ) ) ) continue ; bool isPredicated = isVectorPredicated ( & MI ) ; bool retainsOrReduces = retainsPreviousHalfElement ( MI ) || isHorizontalReduction ( MI ) ; if ( isPredicated ) Predicated . insert ( & MI ) ; if ( producesFalseLanesZero ( MI , QPRs , RDA , FalseLanesZero ) ) FalseLanesZero . insert ( & MI ) ; else if ( MI . getNumDefs ( ) == <NUM_LIT> ) continue ; else if ( ! isPredicated && retainsOrReduces ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; return false ; } else if ( ! isPredicated && MI . getOpcode ( ) != <STR_LIT> ) FalseLanesUnknown . insert ( & MI ) ; } LLVM_DEBUG ( { dbgs ( ) << <STR_LIT> ; for ( auto * I : Predicated ) dbgs ( ) << <STR_LIT> << *


<s> const TargetRegisterClass * QPRs = TRI . getRegClass ( <STR_LIT> ) ; SetVector < MachineInstr * > FalseLanesUnknown ; SmallPtrSet < MachineInstr * , <NUM_LIT> > FalseLanesZero ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Predicated ; MachineBasicBlock * Header = ML . getHeader ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; for ( auto & MI : * Header ) { if ( ! shouldInspect ( MI ) ) continue ; if ( isVCTP ( & MI ) || isVPTOpcode ( MI . getOpcode ( ) ) ) continue ; bool isPredicated = isVectorPredicated ( & MI ) ; bool retainsOrReduces = retainsPreviousHalfElement ( MI ) || isHorizontalReduction ( MI ) ; if ( isPredicated ) Predicated . insert ( & MI ) ; if ( producesFalseLanesZero ( MI , QPRs , RDA , FalseLanesZero ) ) FalseLanesZero . insert ( & MI ) ; else if ( MI . getNumDefs ( ) == <NUM_LIT> ) continue ; else if ( ! isPredicated && retainsOrReduces ) { LLVM_DEBUG ( dbgs ( ) <<


<s> if ( MI -> getOpcode ( ) != <STR_LIT> || ! IsStackOp ( MI ) ) return false ; if ( MI -> memoperands ( ) . size ( ) == <NUM_LIT> ) return false ; int FI = GetFrameIndex ( MI -> memoperands ( ) . front ( ) ) ; auto & FrameInfo = MI -> getParent ( ) -> getParent ( ) -> getFrameInfo ( ) ; if ( FI == - <NUM_LIT> || ! FrameInfo . isSpillSlotObjectIndex ( FI ) ) return false ; SmallVector < MachineBasicBlock * > Frontier ; ML -> getExitBlocks ( Frontier ) ; SmallPtrSet < MachineBasicBlock * , <NUM_LIT> > Visited { MI -> getParent ( ) } ; unsigned Idx = <NUM_LIT> ; while ( Idx < Frontier . size ( ) ) { MachineBasicBlock * BB = Frontier [ Idx ] ; bool LookAtSuccessors = true ; for ( auto & I : * BB ) { if ( ! IsStackOp ( & I ) || I . memoperands ( ) . size ( ) == <NUM_LIT> ) continue ; if ( GetFrameIndex ( I . memoperands ( ) . front ( ) ) != FI ) continue ; if ( I . getOpcode ( ) == <STR_LIT> ) { LookAtSuccessors = false ; break ; } if ( I . getOpcode ( )


<s> } else { TPNumElements = VCTP -> getOperand ( <NUM_LIT> ) ; MCRegister NumElements = TPNumElements . getReg ( ) . asMCReg ( ) ; if ( RDA . hasLocalDefBefore ( VCTP , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( StartInsertPt != StartInsertBB -> end ( ) && ! RDA . isReachingDefLiveOut ( & * StartInsertPt , NumElements ) ) { if ( auto * ElemDef = RDA . getLocalLiveOutMIDef ( StartInsertBB , NumElements ) ) { if ( RDA . isSafeToMoveForwards ( ElemDef , & * StartInsertPt ) ) { ElemDef -> removeFromParent ( ) ; StartInsertBB -> insert ( StartInsertPt , ElemDef ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ElemDef ) ; } else if ( RDA . isSafeToMoveBackwards ( & * StartInsertPt , ElemDef ) ) { StartInsertPt -> removeFromParent ( ) ; StartInsertBB -> insertAfter ( MachineBasicBlock :: iterator ( ElemDef ) , & * StartInsertPt ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ElemDef ) ; } else { MachineOperand Operand = ElemDef -> getOperand ( <NUM_LIT> ) ; if ( isMovRegOpcode ( ElemDef -> getOpcode ( ) ) && RDA . getUniqueReachingMIDef ( ElemDef , Operand . getReg ( ) . asMCReg ( ) ) == RDA . getUniqueReachingMIDef ( & * StartInsertPt , Operand . getReg ( ) . asMCReg ( ) ) ) { TPNumElements = Operand ; NumElements = TPNumElements . getReg ( ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return false ; } } } } auto CannotProvideElements = [ this ] ( MachineBasicBlock * MBB , MCRegister NumElements ) { if ( MBB -> empty ( ) ) return false ; if ( RDA . hasLocalDefBefore ( & MBB -> back ( ) , NumElements ) ) return true ; if ( MBB -> pred_size ( ) > <NUM_LIT> ) return true ; return false ; } ; MachineBasicBlock * MBB = Preheader ; while ( MBB && MBB != StartInsertBB ) { if ( CannotProvideElements ( MBB , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MBB = * MBB -> pred_begin ( ) ; } } if ( std :: any_of ( StartInsertPt , StartInsertBB -> end ( ) , shouldInspect ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } unsigned VCTPVecSize = getVecSize ( * VCTP ) ; for ( MachineInstr * MI : DoubleWidthResultInstrs ) { unsigned InstrVecSize = getVecSize ( * MI ) ; if ( InstrVecSize > VCTPVecSize ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <<


<s> SmallPtrSet < const GlobalVariable * , <NUM_LIT> > & getGlobalsPromotedToConstantPool ( ) { return PromotedGlobals


<s> unsigned getArgRegsSaveSize ( ) const { return ArgRegsSaveSize


<s> It = CoalescedWeights . insert ( std :: make_pair ( MBB , <NUM_LIT> ) ) . first ; } return


<s> unsigned getDPRCalleeSavedAreaOffset ( ) const { return


<s> unsigned getDPRCalleeSavedAreaSize ( ) const { return


<s> unsigned getDPRCalleeSavedAreaSize ( ) const { return DPRCSSize


<s> unsigned getDPRCalleeSavedGapSize ( ) const { return


<s> unsigned getFramePtrSpillOffset ( ) const { return


<s> unsigned getGPRCalleeSavedArea1Size ( ) const { return


<s> unsigned getGPRCalleeSavedArea1Size ( )


<s> unsigned getGPRCalleeSavedArea2Size (


<s> unsigned getNumPICLabels ( ) const { return PICLabelUId


<s> unsigned getOriginalCPIdx ( unsigned CloneIdx ) const { DenseMap < unsigned , unsigned > :: const_iterator I = CPEClones . find ( CloneIdx ) ; if ( I != CPEClones . end ( ) ) return I -> second ; else return -


<s> DenseMap < unsigned , unsigned > :: const_iterator I = CPEClones . find ( CloneIdx ) ; if ( I != CPEClones . end ( )


<s> unsigned getReturnRegsCount ( ) const { return ReturnRegsCount


<s> unsigned getStoredByValParamsPadding ( ) const { return


<s> bool hasITBlocks ( ) const { return HasITBlocks


<s> bool hasStackFrame ( ) const { return


<s> void initPICLabelUId ( unsigned UId ) { PICLabelUId =


<s> void initPICLabelUId ( unsigned UId ) { PICLabelUId = UId


<s> bool isThumb1OnlyFunction ( ) const { return isThumb && !


<s> bool isThumbFunction ( ) const { return isThumb


<s> if ( ! CPEClones . insert ( std :: make_pair ( CPCloneIdx , CPIdx )

) . second ) llvm_unreachable ( <STR_LIT> ) ;


<s> void setArgRegsSaveSize ( unsigned s ) { ArgRegsSaveSize =


<s> void setDPRCalleeSavedAreaOffset ( unsigned o ) { DPRCSOffset =


<s> void setDPRCalleeSavedGapSize ( unsigned


<s> void setFramePtrSpillOffset ( unsigned o ) { FramePtrSpillOffset =


<s> void setGPRCalleeSavedArea1Size ( unsigned s


<s> void setGPRCalleeSavedArea2Size ( unsigned s


<s> void setLRIsSpilledForFarJump ( bool s ) { LRSpilledForFarJump =


<s> void setShouldRestoreSPFromFP ( bool s


<s> void setShouldRestoreSPFromFP ( bool s ) { RestoreSPFromFP =


<s> void setStoredByValParamsPadding ( unsigned p ) { StByValParamsPadding =


<s> void setVarArgsFrameIndex ( int Index )


<s> bool shouldRestoreSPFromFP ( ) const { return


<s> int Size = DPRCSFrames . size ( ) ; if ( fi >= Size ) { Size *= <NUM_LIT> ; if ( fi >= Size ) Size


<s> if ( fi >= Size ) { Size *= <NUM_LIT> ; if ( fi >= Size ) Size = fi + <NUM_LIT> ; GPRCS2Frames . resize ( Size ) ; } GPRCS2Frames [ fi


<s> <ISA_LIT>FunctionInfo ( MachineFunction & MF ) : isThumb ( MF . getTarget ( ) . getSubtarget < <ISA_LIT>Subtarget > ( ) . isThumb ( ) ) , Align ( isThumb ? <NUM_LIT> : <NUM_LIT> ) , VarArgsRegSaveSize ( <NUM_LIT> ) , HasStackFrame ( false ) , LRSpilledForFarJump ( false ) , R3IsLiveIn ( false ) , FramePtrSpillOffset ( <NUM_LIT> ) , GPRCS1Offset ( <NUM_LIT> ) , GPRCS2Offset ( <NUM_LIT> ) , DPRCSOffset ( <NUM_LIT> ) , GPRCS1Size ( <NUM_LIT> ) , GPRCS2Size ( <NUM_LIT> ) , DPRCSSize ( <NUM_LIT> ) , GPRCS1Frames ( <NUM_LIT> ) , GPRCS2Frames ( <NUM_LIT> ) , DPRCSFrames ( <NUM_LIT> ) , SpilledCSRegs ( MF . getTarget

( ) . getRegisterInfo ( ) -> getNumRegs ( ) ) , JumpTableUId ( <NUM_LIT> ) , ConstPoolEntryUId ( <NUM_LIT> ) {


<s> unsigned getAlign ( ) const { return Align


<s> const BitVector & getSpilledCSRegisters (


<s> bool isDPRCalleeSavedAreaFrame ( int fi ) const { if ( fi < <NUM_LIT> || fi >= ( int ) DPRCSFrames


<s> bool isGPRCalleeSavedArea2Frame ( int fi ) const { if ( fi < <NUM_LIT> || fi >= ( int ) GPRCS2Frames . size ( ) ) return false ; return GPRCS2Frames [ fi


<s> if ( fi < <NUM_LIT> || fi >= ( int ) GPRCS2Frames . size ( ) ) return false ; return GPRCS2Frames [


<s> void setCSRegisterIsSpilled ( unsigned Reg ) { SpilledCSRegs . set


<s> void setVarArgsRegSaveSize ( unsigned


<s> unsigned getArgumentStackToRestore ( )


<s> bool getPreservesR0 ( ) const { return PreservesR0


<s> bool isCmseNSCallFunction ( ) const { return IsCmseNSCall


<s> bool isCmseNSEntryFunction ( ) const { return IsCmseNSEntry


<s> void setArgumentStackToRestore ( unsigned v ) { ArgumentStackToRestore =


<s> bool shouldSignReturnAddress ( bool SpillsLR ) const { if ( ! SignReturnAddress


<s> if ( SignReturnAddressAll ) return true ; return SpillsLR


<s> if ( ! CPEClones . insert ( std :: make_pair ( CPCloneIdx , CPIdx

) ) . second ) assert ( <NUM_LIT> && <STR_LIT> ) ;


<s> <ISA_LIT>FunctionInfo ( ) : isThumb ( false ) , hasThumb2 ( false ) , ArgRegsSaveSize ( <NUM_LIT> ) , ReturnRegsCount ( <NUM_LIT> ) , HasStackFrame ( false ) , RestoreSPFromFP ( false ) , LRSpilledForFarJump ( false ) , FramePtrSpillOffset ( <NUM_LIT> ) , GPRCS1Offset ( <NUM_LIT> ) , GPRCS2Offset ( <NUM_LIT> ) , DPRCSOffset ( <NUM_LIT> ) , GPRCS1Size ( <NUM_LIT> ) , GPRCS2Size ( <NUM_LIT> ) , DPRCSAlignGapSize ( <NUM_LIT> ) , DPRCSSize ( <NUM_LIT> ) , NumAlignedDPRCS2Regs ( <NUM_LIT> ) , JumpTableUId ( <NUM_LIT> ) , PICLabelUId ( <NUM_LIT> ) , VarArgsFrameIndex

( <NUM_LIT> ) , HasITBlocks ( false ) , GlobalBaseReg ( <NUM_LIT> ) {


<s> <ISA_LIT>FunctionInfo ( ) : isThumb ( false ) , hasThumb2 ( false ) , ArgRegsSaveSize ( <NUM_LIT> ) , ReturnRegsCount ( <NUM_LIT> ) , HasStackFrame ( false ) , RestoreSPFromFP ( false ) , LRSpilledForFarJump ( false ) , FramePtrSpillOffset ( <NUM_LIT> ) , GPRCS1Offset ( <NUM_LIT> ) , GPRCS2Offset ( <NUM_LIT> ) , DPRCSOffset ( <NUM_LIT> ) , GPRCS1Size ( <NUM_LIT> ) , GPRCS2Size ( <NUM_LIT>

) , DPRCSAlignGapSize ( <NUM_LIT> ) , DPRCSSize ( <NUM_LIT> ) , NumAlignedDPRCS2Regs ( <NUM_LIT> ) , JumpTableUId ( <NUM_LIT> ) , PICLabelUId ( <NUM_LIT> ) , VarArgsFrameIndex ( <NUM_LIT> ) , HasITBlocks ( false ) , GlobalBaseReg ( <NUM_LIT> ) {


<s> <ISA_LIT>FunctionInfo ( ) : isThumb ( false ) , hasThumb2 ( false ) , ArgRegsSaveSize ( <NUM_LIT> ) , ReturnRegsCount ( <NUM_LIT> ) , HasStackFrame ( false ) , RestoreSPFromFP ( false ) , LRSpilledForFarJump ( false ) , FramePtrSpillOffset ( <NUM_LIT> ) , GPRCS1Offset ( <NUM_LIT> ) , GPRCS2Offset ( <NUM_LIT> ) , DPRCSOffset ( <NUM_LIT> ) , GPRCS1Size ( <NUM_LIT> ) , GPRCS2Size ( <NUM_LIT> ) , DPRCSAlignGapSize ( <NUM_LIT> ) , DPRCSSize ( <NUM_LIT> )

, NumAlignedDPRCS2Regs ( <NUM_LIT> ) , PICLabelUId ( <NUM_LIT> ) , VarArgsFrameIndex ( <NUM_LIT> ) , HasITBlocks ( false ) , IsSplitCSR ( false ) {


<s> <ISA_LIT>FunctionInfo ( ) : isThumb ( false ) , hasThumb2 ( false ) , ArgRegsSaveSize ( <NUM_LIT> ) , ReturnRegsCount ( <NUM_LIT> ) , HasStackFrame ( false ) , RestoreSPFromFP ( false ) , LRSpilledForFarJump ( false ) , FramePtrSpillOffset ( <NUM_LIT> ) , GPRCS1Offset ( <NUM_LIT> ) , GPRCS2Offset ( <NUM_LIT> ) , DPRCSOffset ( <NUM_LIT> ) , GPRCS1Size ( <NUM_LIT> ) , GPRCS2Size ( <NUM_LIT> ) , DPRCSAlignGapSize ( <NUM_LIT> ) , DPRCSSize ( <NUM_LIT> ) ,

NumAlignedDPRCS2Regs ( <NUM_LIT> ) , PICLabelUId ( <NUM_LIT> ) , VarArgsFrameIndex ( <NUM_LIT> ) , HasITBlocks ( false ) , IsSplitCSR ( false ) {


<s> isThumb ( false ) , hasThumb2 ( false ) , ArgRegsSaveSize ( <NUM_LIT> ) , ReturnRegsCount ( <NUM_LIT> ) , HasStackFrame ( false ) , RestoreSPFromFP ( false ) , LRSpilledForFarJump ( false ) , FramePtrSpillOffset ( <NUM_LIT> ) , GPRCS1Offset ( <NUM_LIT> ) , GPRCS2Offset ( <NUM_LIT> ) , DPRCSOffset ( <NUM_LIT> ) , GPRCS1Size ( <NUM_LIT> ) , GPRCS2Size ( <NUM_LIT> ) , DPRCSAlignGapSize ( <NUM_LIT> ) , DPRCSSize ( <NUM_LIT> ) , NumAlignedDPRCS2Regs ( <NUM_LIT> ) , PICLabelUId ( <NUM_LIT> ) , VarArgsFrameIndex ( <NUM_LIT> )

, HasITBlocks ( false ) , GlobalBaseReg ( <NUM_LIT> ) {


<s> isThumb ( MF . getTarget ( ) . getSubtarget < <ISA_LIT>Subtarget > ( ) . isThumb ( ) ) , hasThumb2 ( MF . getTarget ( ) . getSubtarget < <ISA_LIT>Subtarget > ( ) . hasThumb2 ( ) ) , VarArgsRegSaveSize ( <NUM_LIT> ) , HasStackFrame ( false ) , RestoreSPFromFP ( false ) , LRSpilledForFarJump ( false ) , FramePtrSpillOffset ( <NUM_LIT> ) , GPRCS1Offset ( <NUM_LIT> ) , GPRCS2Offset ( <NUM_LIT> ) , DPRCSOffset ( <NUM_LIT> ) , GPRCS1Size ( <NUM_LIT> ) , GPRCS2Size ( <NUM_LIT> ) , DPRCSSize ( <NUM_LIT> ) ,

GPRCS1Frames ( <NUM_LIT> ) , GPRCS2Frames ( <NUM_LIT> ) , DPRCSFrames ( <NUM_LIT> ) , JumpTableUId ( <NUM_LIT> ) , PICLabelUId ( <NUM_LIT> ) , VarArgsFrameIndex ( <NUM_LIT> ) , HasITBlocks ( false ) , GlobalBaseReg ( <NUM_LIT> ) {


<s> isThumb ( MF . getTarget ( ) . getSubtarget < <ISA_LIT>Subtarget > ( ) . isThumb ( ) ) , hasThumb2 ( MF . getTarget ( ) . getSubtarget < <ISA_LIT>Subtarget > ( ) . hasThumb2 ( ) ) , StByValParamsPadding ( <NUM_LIT> ) , ArgRegsSaveSize ( <NUM_LIT> ) , HasStackFrame ( false ) , RestoreSPFromFP ( false ) , LRSpilledForFarJump ( false ) , FramePtrSpillOffset ( <NUM_LIT> ) , GPRCS1Offset ( <NUM_LIT> ) , GPRCS2Offset ( <NUM_LIT> ) , DPRCSOffset ( <NUM_LIT> ) , GPRCS1Size ( <NUM_LIT> ) , GPRCS2Size ( <NUM_LIT> ) , DPRCSSize ( <NUM_LIT> ) , GPRCS1Frames ( <NUM_LIT> ) , GPRCS2Frames ( <NUM_LIT> ) , DPRCSFrames ( <NUM_LIT> ) , JumpTableUId ( <NUM_LIT> ) , PICLabelUId ( <NUM_LIT> ) ,

VarArgsFrameIndex ( <NUM_LIT> ) , HasITBlocks ( false ) , GlobalBaseReg ( <NUM_LIT> ) {


<s> explicit <ISA_LIT>FunctionInfo ( MachineFunction & MF ) : isThumb ( MF . getTarget ( ) . getSubtarget < <ISA_LIT>Subtarget > ( ) . isThumb ( ) ) , hasThumb2 ( MF . getTarget ( ) . getSubtarget < <ISA_LIT>Subtarget > ( ) . hasThumb2 ( ) ) , StByValParamsPadding ( <NUM_LIT> ) , ArgRegsSaveSize ( <NUM_LIT> ) , HasStackFrame ( false ) , RestoreSPFromFP ( false ) , LRSpilledForFarJump ( false ) , FramePtrSpillOffset ( <NUM_LIT> ) , GPRCS1Offset ( <NUM_LIT> ) , GPRCS2Offset ( <NUM_LIT> ) , DPRCSOffset ( <NUM_LIT> ) , GPRCS1Size ( <NUM_LIT> ) , GPRCS2Size ( <NUM_LIT> ) ,

DPRCSSize ( <NUM_LIT> ) , GPRCS1Frames ( <NUM_LIT> ) , GPRCS2Frames ( <NUM_LIT> ) , DPRCSFrames ( <NUM_LIT> ) , JumpTableUId ( <NUM_LIT> ) , PICLabelUId ( <NUM_LIT> ) , VarArgsFrameIndex ( <NUM_LIT> ) , HasITBlocks ( false ) , GlobalBaseReg ( <NUM_LIT> ) {


<s> isThumb ( MF . getTarget ( ) . getSubtarget < <ISA_LIT>Subtarget > ( ) . isThumb ( ) ) , hasThumb2 ( MF . getTarget ( ) . getSubtarget < <ISA_LIT>Subtarget > ( ) . hasThumb2 ( ) ) , VarArgsRegSaveSize ( <NUM_LIT> ) , HasStackFrame ( false ) , LRSpilledForFarJump ( false ) , FramePtrSpillOffset ( <NUM_LIT> ) , GPRCS1Offset ( <NUM_LIT> ) , GPRCS2Offset ( <NUM_LIT> ) , DPRCSOffset ( <NUM_LIT> ) , GPRCS1Size ( <NUM_LIT> ) ,

GPRCS2Size ( <NUM_LIT> ) , DPRCSSize ( <NUM_LIT> ) , GPRCS1Frames ( <NUM_LIT> ) , GPRCS2Frames ( <NUM_LIT> ) , DPRCSFrames ( <NUM_LIT> ) , SpilledCSRegs ( MF . getTarget ( ) . getRegisterInfo ( ) -> getNumRegs ( ) ) , JumpTableUId ( <NUM_LIT> ) , ConstPoolEntryUId ( <NUM_LIT> ) {


<s> isThumb ( false ) , hasThumb2 ( false ) , ArgRegsSaveSize ( <NUM_LIT> ) , ReturnRegsCount ( <NUM_LIT> ) , HasStackFrame ( false ) , RestoreSPFromFP ( false ) , LRSpilledForFarJump ( false ) , FramePtrSpillOffset ( <NUM_LIT> ) , GPRCS1Offset ( <NUM_LIT> ) , GPRCS2Offset ( <NUM_LIT> ) , DPRCSOffset ( <NUM_LIT> )

, GPRCS1Size ( <NUM_LIT> ) , GPRCS2Size ( <NUM_LIT> ) , DPRCSSize ( <NUM_LIT> ) , NumAlignedDPRCS2Regs ( <NUM_LIT> ) , JumpTableUId ( <NUM_LIT> ) , PICLabelUId ( <NUM_LIT> ) , VarArgsFrameIndex ( <NUM_LIT> ) , HasITBlocks ( false ) , GlobalBaseReg ( <NUM_LIT> ) {


<s> static bool GetBranchTargetEnforcement ( MachineFunction & MF ) { const auto & Subtarget = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( ! Subtarget . isMClass ( ) || ! Subtarget . hasV7Ops ( ) ) return false ; const Function & F = MF . getFunction ( ) ; if ( ! F . hasFnAttribute ( <STR_LIT> ) ) { if ( const auto * BTE = mdconst :: extract_or_null < ConstantInt > ( F . getParent ( ) -> getModuleFlag ( <STR_LIT> ) ) ) return BTE -> getZExtValue ( ) ; return false


<s> static std :: pair < bool , bool > GetSignReturnAddress ( const Function & F ) { if ( ! F . hasFnAttribute ( <STR_LIT> ) ) { const Module & M = * F . getParent ( ) ; if ( const auto * Sign = mdconst :: extract_or_null < ConstantInt > ( M . getModuleFlag ( <STR_LIT> ) ) ) { if ( Sign -> getZExtValue ( ) ) { if ( const auto * All = mdconst :: extract_or_null < ConstantInt > ( M . getModuleFlag ( <STR_LIT> ) ) ) return { true , All -> getZExtValue ( ) } ; return { true , false } ; } } return { false , false } ; } StringRef Scope = F . getFnAttribute ( <STR_LIT> ) . getValueAsString ( ) ; if ( Scope . equals ( <STR_LIT> ) ) return { false , false } ; if ( Scope . equals ( <STR_LIT> ) ) return { true , true } ; assert ( Scope . equals


<s> <ISA_LIT>FunctionInfo ( ) : isThumb ( false ) , hasThumb2 ( false ) , VarArgsRegSaveSize ( <NUM_LIT> ) , HasStackFrame ( false ) , RestoreSPFromFP ( false ) , LRSpilledForFarJump ( false ) , FramePtrSpillOffset ( <NUM_LIT> ) , GPRCS1Offset ( <NUM_LIT> ) , GPRCS2Offset ( <NUM_LIT> ) , DPRCSOffset ( <NUM_LIT> ) , GPRCS1Size ( <NUM_LIT> ) , GPRCS2Size ( <NUM_LIT> ) , DPRCSSize ( <NUM_LIT> ) , GPRCS1Frames ( <NUM_LIT> ) , GPRCS2Frames

( <NUM_LIT> ) , DPRCSFrames ( <NUM_LIT> ) , JumpTableUId ( <NUM_LIT> ) , PICLabelUId ( <NUM_LIT> ) , VarArgsFrameIndex ( <NUM_LIT> ) , HasITBlocks ( false ) {


<s> explicit <ISA_LIT>FunctionInfo ( MachineFunction & MF ) : isThumb ( MF . getTarget ( ) . getSubtarget < <ISA_LIT>Subtarget > ( ) . isThumb ( ) ) , hasThumb2 ( MF . getTarget ( ) . getSubtarget < <ISA_LIT>Subtarget > ( ) . hasThumb2 ( ) ) , VarArgsRegSaveSize ( <NUM_LIT> ) , HasStackFrame ( false ) , RestoreSPFromFP ( false ) , LRSpilledForFarJump ( false ) , FramePtrSpillOffset ( <NUM_LIT> ) , GPRCS1Offset ( <NUM_LIT> ) , GPRCS2Offset ( <NUM_LIT> ) , DPRCSOffset ( <NUM_LIT> ) , GPRCS1Size ( <NUM_LIT> ) , GPRCS2Size ( <NUM_LIT> ) , DPRCSSize ( <NUM_LIT> ) , GPRCS1Frames ( <NUM_LIT> ) , GPRCS2Frames ( <NUM_LIT>

) , DPRCSFrames ( <NUM_LIT> ) , SpilledCSRegs ( MF . getTarget ( ) . getRegisterInfo ( ) -> getNumRegs ( ) ) , JumpTableUId ( <NUM_LIT> ) , ConstPoolEntryUId ( <NUM_LIT> ) , VarArgsFrameIndex ( <NUM_LIT> ) , HasITBlocks ( false ) {


<s> <ISA_LIT>FunctionInfo ( ) : isThumb ( false ) , hasThumb2 ( false ) , ArgRegsSaveSize ( <NUM_LIT> ) , HasStackFrame ( false ) , RestoreSPFromFP ( false ) , LRSpilledForFarJump ( false ) , FramePtrSpillOffset ( <NUM_LIT> ) , GPRCS1Offset ( <NUM_LIT> ) , GPRCS2Offset ( <NUM_LIT> ) , DPRCSOffset ( <NUM_LIT> ) , GPRCS1Size ( <NUM_LIT> ) , GPRCS2Size ( <NUM_LIT> ) , DPRCSSize ( <NUM_LIT> ) , NumAlignedDPRCS2Regs ( <NUM_LIT> ) , JumpTableUId ( <NUM_LIT> ) ,

PICLabelUId ( <NUM_LIT> ) , VarArgsFrameIndex ( <NUM_LIT> ) , HasITBlocks ( false ) , GlobalBaseReg ( <NUM_LIT> ) {


<s> unsigned createJumpTableUId ( ) { return JumpTableUId


<s> MCObjectWriter * llvm :: create<ISA_LIT>MachObjectWriter ( raw_pwrite_stream & OS , bool Is64Bit , uint32_t CPUType , uint32_t CPUSubtype ) { return createMachObjectWriter ( new <ISA_LIT>MachObjectWriter ( Is64Bit , CPUType , CPUSubtype ) ,


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return false ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : RelocType = unsigned ( MachO :: <ISA_LIT>_RELOC_BR24 ) ; Log2Size = llvm :: Log2_32 ( <NUM_LIT> ) ; return true ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : RelocType = unsigned ( MachO :: <ISA_LIT>_THUMB_RELOC_BR22 ) ; Log2Size = llvm :: Log2_32 ( <NUM_LIT> ) ; return true ; case <STR_LIT> : RelocType = unsigned ( MachO :: <ISA_LIT>_RELOC_HALF ) ; Log2Size = <NUM_LIT> ; return true ; case <STR_LIT> : RelocType = unsigned ( MachO :: <ISA_LIT>_RELOC_HALF ) ; Log2Size = <NUM_LIT> ; return true ; case <STR_LIT> : RelocType = unsigned ( MachO :: <ISA_LIT>_RELOC_HALF ) ; Log2Size = <NUM_LIT> ; return true ; case


<s> void <ISA_LIT>MachObjectWriter :: Record<ISA_LIT>ScatteredHalfRelocation ( MachObjectWriter * Writer , const MCAssembler & Asm , const MCAsmLayout & Layout , const MCFragment * Fragment , const MCFixup & Fixup , MCValue Target , uint64_t & FixedValue ) { uint32_t FixupOffset = Layout . getFragmentOffset ( Fragment ) + Fixup . getOffset ( ) ; unsigned IsPCRel = Writer -> isFixupKindPCRel ( Asm , Fixup . getKind ( ) ) ; unsigned Type = MachO :: <ISA_LIT>_RELOC_HALF ; const MCSymbol * A = & Target . getSymA ( ) -> getSymbol ( ) ; if ( ! A -> getFragment ( ) ) { Asm . getContext ( ) . reportError ( Fixup . getLoc ( ) , <STR_LIT> + A -> getName ( ) + <STR_LIT> ) ; return ; } uint32_t Value = Writer -> getSymbolAddress ( * A , Layout ) ; uint32_t Value2 = <NUM_LIT> ; uint64_t SecAddr = Writer -> getSectionAddress ( A -> getFragment ( ) -> getParent ( ) ) ; FixedValue += SecAddr ; if ( const MCSymbolRefExpr * B = Target . getSymB ( ) ) { const MCSymbol * SB = & B -> getSymbol ( ) ; if ( ! SB -> getFragment ( ) ) { Asm . getContext ( ) . reportError ( Fixup . getLoc ( ) , <STR_LIT> + B -> getSymbol ( ) . getName ( ) + <STR_LIT> ) ; return ; } Type = MachO :: <ISA_LIT>_RELOC_HALF_SECTDIFF ; Value2 = Writer -> getSymbolAddress ( B -> getSymbol ( ) , Layout ) ; FixedValue -= Writer -> getSectionAddress ( SB -> getFragment ( ) -> getParent ( ) ) ; } unsigned ThumbBit = <NUM_LIT> ; unsigned MovtBit = <NUM_LIT> ; switch ( ( unsigned ) Fixup . getKind ( ) ) { default : break ; case <STR_LIT> : MovtBit = <NUM_LIT> ; if ( Asm . isThumbFunc ( A ) ) FixedValue &= <NUM_LIT> ; break ; case <STR_LIT> : if ( Asm . isThumbFunc ( A ) ) FixedValue &= <NUM_LIT> ; MovtBit = <NUM_LIT> ; case <STR_LIT> : ThumbBit = <NUM_LIT> ; break ; } if ( Type == MachO :: <ISA_LIT>_RELOC_HALF_SECTDIFF ) { uint32_t OtherHalf = MovtBit ? ( FixedValue & <NUM_LIT> ) : ( ( FixedValue & <NUM_LIT> ) >> <NUM_LIT> ) ; MachO :: any_relocation_info MRE ; MRE . r_word0 = ( ( OtherHalf << <NUM_LIT> ) | ( MachO :: <ISA_LIT>_RELOC_PAIR << <NUM_LIT> ) | ( MovtBit << <NUM_LIT> ) | ( ThumbBit << <NUM_LIT> ) | ( IsPCRel << <NUM_LIT> ) | MachO :: R_SCATTERED ) ; MRE . r_word1 = Value2 ; Writer -> addRelocation ( nullptr , Fragment -> getParent ( ) , MRE


<s> if ( IsPCRel && RelocType == MachO :: <ISA_LIT>_RELOC_VANILLA ) Offset += <NUM_LIT> << Log2Size ; if ( Offset && A && ! Writer -> doesSymbolRequireExternRelocation ( * A ) ) return Record<ISA_LIT>ScatteredRelocation ( Writer , Asm , Layout , Fragment , Fixup , Target , RelocType , Log2Size , FixedValue ) ; uint32_t FixupOffset = Layout . getFragmentOffset ( Fragment ) + Fixup . getOffset ( ) ; unsigned Index = <NUM_LIT> ; unsigned Type = <NUM_LIT> ; const MCSymbol * RelSymbol = nullptr ; if ( Target . isAbsolute ( ) ) { report_fatal_error ( <STR_LIT> <STR_LIT> ) ; } else { if ( A -> isVariable ( ) ) { int64_t Res ; if ( A -> getVariableValue ( ) -> evaluateAsAbsolute ( Res , Layout , Writer -> getSectionAddressMap ( ) ) ) { FixedValue = Res ; return ; } } if ( requiresExternRelocation ( Writer , Asm , * Fragment , RelocType , * A , FixedValue ) ) { RelSymbol = A ; if ( ! A -> isUndefined ( ) ) FixedValue -= Layout . getSymbolOffset ( * A ) ; } else { const MCSection & Sec = A -> getSection ( ) ; Index = Sec . getOrdinal ( ) + <NUM_LIT> ; FixedValue += Writer -> getSectionAddress ( & Sec ) ; } if ( IsPCRel ) FixedValue -= Writer -> getSectionAddress ( Fragment -> getParent ( ) ) ; Type = RelocType ; } MachO :: any_relocation_info MRE ; MRE . r_word0 = FixupOffset ; MRE . r_word1 = ( Index << <NUM_LIT> ) | ( IsPCRel << <NUM_LIT> ) | ( Log2Size << <NUM_LIT> ) | ( Type << <NUM_LIT> ) ; if ( Type == MachO :: <ISA_LIT>_RELOC_HALF ) { uint32_t Value = <NUM_LIT> ; switch ( ( unsigned ) Fixup . getKind ( ) ) { default : break ; case <STR_LIT> : case <STR_LIT> : Value = ( FixedValue >> <NUM_LIT> ) & <NUM_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Value = FixedValue & <NUM_LIT> ; break ; } MachO :: any_relocation_info MREPair ; MREPair . r_word0 = Value ; MREPair . r_word1 = ( ( <NUM_LIT> << <NUM_LIT> ) | ( Log2Size << <NUM_LIT> ) | ( MachO :: <ISA_LIT>_RELOC_PAIR << <NUM_LIT> ) ) ; Writer -> addRelocation ( nullptr , Fragment -> getParent ( ) , MREPair ) ; } Writer -> addRelocation ( RelSymbol , Fragment -> getParent ( ) ,


<s> unsigned IsPCRel = Writer -> isFixupKindPCRel ( Asm , Fixup . getKind ( ) ) ; unsigned Type = MachO :: <ISA_LIT>_RELOC_HALF ; const MCSymbol * A = & Target . getSymA ( ) -> getSymbol ( ) ; const MCSymbolData * A_SD = & Asm . getSymbolData ( * A ) ; if ( ! A_SD -> getFragment ( ) ) Asm . getContext ( ) . FatalError ( Fixup . getLoc ( ) , <STR_LIT> + A -> getName ( ) + <STR_LIT> ) ; uint32_t Value = Writer -> getSymbolAddress ( A_SD , Layout ) ; uint32_t Value2 = <NUM_LIT> ; uint64_t SecAddr = Writer -> getSectionAddress ( A_SD -> getFragment ( ) -> getParent ( ) ) ; FixedValue += SecAddr ; if ( const MCSymbolRefExpr * B = Target . getSymB ( ) ) { const MCSymbolData * B_SD = & Asm . getSymbolData ( B -> getSymbol ( ) ) ; if ( ! B_SD -> getFragment ( ) ) Asm . getContext ( ) . FatalError ( Fixup . getLoc ( ) , <STR_LIT> + B -> getSymbol ( ) . getName ( ) + <STR_LIT> ) ; Type = MachO :: <ISA_LIT>_RELOC_HALF_SECTDIFF ; Value2 = Writer -> getSymbolAddress ( B_SD


<s> uint32_t FixupOffset = Layout . getFragmentOffset ( Fragment ) + Fixup . getOffset ( ) ; unsigned IsPCRel = Writer -> isFixupKindPCRel ( Asm , Fixup . getKind ( ) ) ; const MCSymbol * A = & Target . getSymA ( ) -> getSymbol ( ) ; const MCSymbolData * A_SD = & Asm . getSymbolData ( * A ) ; if ( ! A_SD -> getFragment ( ) ) Asm . getContext ( ) . FatalError ( Fixup . getLoc ( ) , <STR_LIT> + A -> getName ( ) + <STR_LIT> ) ; uint32_t Value = Writer -> getSymbolAddress ( A_SD , Layout ) ; uint64_t SecAddr = Writer -> getSectionAddress ( A_SD -> getFragment ( ) -> getParent ( ) ) ; FixedValue += SecAddr ; uint32_t Value2 = <NUM_LIT> ; if ( const MCSymbolRefExpr * B = Target . getSymB ( ) ) { assert ( Type == MachO :: <ISA_LIT>_RELOC_VANILLA && <STR_LIT> ) ; const MCSymbolData * B_SD = & Asm . getSymbolData ( B -> getSymbol ( ) ) ; if ( ! B_SD -> getFragment ( ) ) Asm . getContext ( ) . FatalError ( Fixup . getLoc ( ) , <STR_LIT> + B -> getSymbol ( ) . getName ( ) + <STR_LIT> ) ; Type = MachO :: <ISA_LIT>_RELOC_SECTDIFF ; Value2 = Writer -> getSymbolAddress ( B_SD , Layout ) ; FixedValue -= Writer -> getSectionAddress ( B_SD -> getFragment ( ) -> getParent ( ) ) ; } if ( Type == MachO :: <ISA_LIT>_RELOC_SECTDIFF || Type == MachO :: <ISA_LIT>_RELOC_LOCAL_SECTDIFF ) { MachO :: any_relocation_info MRE ; MRE . r_word0 = ( ( <NUM_LIT> << <NUM_LIT> ) | ( MachO :: <ISA_LIT>_RELOC_PAIR << <NUM_LIT> ) | ( Log2Size << <NUM_LIT> ) | ( IsPCRel << <NUM_LIT> ) | MachO :: R_SCATTERED ) ; MRE . r_word1 = Value2 ; Writer -> addRelocation ( nullptr , Fragment -> getParent ( ) , MRE ) ; } MachO :: any_relocation_info MRE


<s> if ( const MCSymbolRefExpr * B = Target . getSymB ( ) ) { assert ( Type == MachO :: <ISA_LIT>_RELOC_VANILLA && <STR_LIT> ) ; const MCSymbolData * B_SD = & Asm . getSymbolData ( B -> getSymbol ( ) ) ; if ( ! B_SD -> getFragment ( ) ) Asm . getContext ( ) . FatalError ( Fixup . getLoc ( ) , <STR_LIT> + B -> getSymbol ( ) . getName ( ) + <STR_LIT> ) ; Type = MachO :: <ISA_LIT>_RELOC_SECTDIFF ; Value2 = Writer -> getSymbolAddress ( B_SD , Layout ) ; FixedValue -= Writer -> getSectionAddress ( B_SD -> getFragment ( ) -> getParent ( ) ) ; } if ( Type == MachO :: <ISA_LIT>_RELOC_SECTDIFF || Type == MachO :: <ISA_LIT>_RELOC_LOCAL_SECTDIFF ) { MachO :: any_relocation_info MRE ; MRE . r_word0 = ( ( <NUM_LIT> << <NUM_LIT> ) | ( MachO :: <ISA_LIT>_RELOC_PAIR << <NUM_LIT> ) | ( Log2Size << <NUM_LIT> ) | ( IsPCRel << <NUM_LIT> ) | MachO :: R_SCATTERED ) ; MRE . r_word1 = Value2 ; Writer -> addRelocation ( nullptr , Fragment -> getParent ( ) , MRE ) ; } MachO :: any_relocation_info MRE ; MRE . r_word0 = ( ( FixupOffset << <NUM_LIT> ) | ( Type << <NUM_LIT> ) | ( Log2Size << <NUM_LIT> ) | ( IsPCRel << <NUM_LIT> ) | MachO :: R_SCATTERED ) ; MRE . r_word1 = Value ; Writer -> addRelocation ( nullptr , Fragment ->


<s> if ( Offset && SD && ! Writer -> doesSymbolRequireExternRelocation ( SD ) ) return Record<ISA_LIT>ScatteredRelocation ( Writer , Asm , Layout , Fragment , Fixup , Target , RelocType , Log2Size , FixedValue ) ; uint32_t FixupOffset = Layout . getFragmentOffset ( Fragment ) + Fixup . getOffset ( ) ; unsigned Index = <NUM_LIT> ; unsigned Type = <NUM_LIT> ; const MCSymbolData * RelSymbol = nullptr ; if ( Target . isAbsolute ( ) ) { report_fatal_error ( <STR_LIT> <STR_LIT> ) ; } else { if ( SD -> getSymbol ( ) . isVariable ( ) ) { int64_t Res ; if ( SD -> getSymbol ( ) . getVariableValue ( ) -> EvaluateAsAbsolute ( Res , Layout , Writer -> getSectionAddressMap ( ) ) ) { FixedValue = Res ; return ; } } if ( requiresExternRelocation ( Writer , Asm , * Fragment , RelocType , SD , FixedValue ) ) { RelSymbol = SD ; if ( ! SD -> getSymbol ( ) . isUndefined ( ) ) FixedValue -= Layout . getSymbolOffset ( SD ) ; } else { const MCSectionData & SymSD = Asm . getSectionData ( SD -> getSymbol ( ) . getSection ( ) ) ; Index = SymSD . getOrdinal ( ) + <NUM_LIT> ; FixedValue += Writer -> getSectionAddress ( & SymSD ) ; } if ( IsPCRel ) FixedValue -= Writer -> getSectionAddress ( Fragment -> getParent ( ) ) ; Type = RelocType ; } MachO :: any_relocation_info MRE ; MRE . r_word0 = FixupOffset ; MRE . r_word1 = ( Index << <NUM_LIT> ) | ( IsPCRel << <NUM_LIT> ) | ( Log2Size << <NUM_LIT> ) | ( Type << <NUM_LIT> ) ; if ( Type == MachO :: <ISA_LIT>_RELOC_HALF ) { uint32_t Value = <NUM_LIT> ; switch ( ( unsigned ) Fixup . getKind ( ) ) { default : break ; case <STR_LIT> : case <STR_LIT> : Value = ( FixedValue >> <NUM_LIT> ) & <NUM_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Value = FixedValue & <NUM_LIT> ; break ; } MachO :: any_relocation_info MREPair ; MREPair . r_word0 =


<s> int64_t Res ; if ( A -> getVariableValue ( ) -> evaluateAsAbsolute ( Res , Layout , Writer -> getSectionAddressMap ( ) ) ) { FixedValue = Res ; return ; } } if ( requiresExternRelocation ( Writer , Asm , * Fragment , RelocType , * A , FixedValue ) ) { RelSymbol = A ; if ( ! A -> isUndefined ( ) ) FixedValue -= Layout . getSymbolOffset ( * A ) ; } else { const MCSection & Sec = A -> getSection ( ) ; Index = Sec . getOrdinal ( ) + <NUM_LIT> ; FixedValue += Writer -> getSectionAddress ( & Sec ) ; } if ( IsPCRel ) FixedValue -= Writer -> getSectionAddress ( Fragment -> getParent ( ) ) ; Type = RelocType ; } MachO :: any_relocation_info MRE ; MRE . r_word0 = FixupOffset ; MRE . r_word1 = ( Index << <NUM_LIT> ) | ( IsPCRel << <NUM_LIT> ) | ( Log2Size << <NUM_LIT> ) | ( Type << <NUM_LIT> ) ; if ( Type == MachO :: <ISA_LIT>_RELOC_HALF ) { uint32_t Value = <NUM_LIT> ; switch ( ( unsigned ) Fixup . getKind ( ) ) { default : break ; case <STR_LIT> : case <STR_LIT> : Value = ( FixedValue >> <NUM_LIT> ) & <NUM_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Value = FixedValue


<s> return llvm :: make_unique < <ISA_LIT>MachObjectWriter > (


<s> uint32_t FixupOffset = Layout . getFragmentOffset ( Fragment ) + Fixup . getOffset ( ) ; unsigned IsPCRel = Writer -> isFixupKindPCRel ( Asm , Fixup . getKind ( ) ) ; unsigned Type = MachO :: <ISA_LIT>_RELOC_HALF ; const MCSymbol * A = & Target . getSymA ( ) -> getSymbol ( ) ; if ( ! A -> getFragment ( ) ) { Asm . getContext ( ) . reportError ( Fixup . getLoc ( ) , <STR_LIT> + A -> getName ( ) + <STR_LIT> ) ; return ; } uint32_t Value = Writer -> getSymbolAddress ( * A , Layout ) ; uint32_t Value2 = <NUM_LIT> ; uint64_t SecAddr = Writer -> getSectionAddress ( A -> getFragment ( ) -> getParent ( ) ) ; FixedValue += SecAddr ; if ( const MCSymbolRefExpr * B = Target . getSymB ( ) ) { const MCSymbol * SB = & B -> getSymbol ( ) ; if ( ! SB -> getFragment ( ) ) { Asm . getContext ( ) . reportError ( Fixup . getLoc ( ) , <STR_LIT> + B -> getSymbol ( ) . getName ( ) + <STR_LIT> ) ; return ; } Type = MachO :: <ISA_LIT>_RELOC_HALF_SECTDIFF ; Value2 = Writer -> getSymbolAddress ( B -> getSymbol ( ) , Layout ) ; FixedValue -= Writer -> getSectionAddress ( SB -> getFragment ( ) -> getParent ( ) ) ; } unsigned ThumbBit = <NUM_LIT> ; unsigned MovtBit = <NUM_LIT> ; switch ( ( unsigned ) Fixup . getKind ( ) ) { default : break ; case <STR_LIT> : MovtBit = <NUM_LIT> ; if ( Asm . isThumbFunc ( A ) ) FixedValue &= <NUM_LIT> ; break ; case <STR_LIT> : if ( Asm . isThumbFunc ( A ) ) FixedValue &= <NUM_LIT> ; MovtBit = <NUM_LIT> ; LLVM_FALLTHROUGH ; case <STR_LIT>


<s> MovtBit = <NUM_LIT> ; LLVM_FALLTHROUGH ; case <STR_LIT> : ThumbBit = <NUM_LIT> ; break ; } if ( Type == MachO :: <ISA_LIT>_RELOC_HALF_SECTDIFF ) { uint32_t OtherHalf = MovtBit ? ( FixedValue & <NUM_LIT> ) : ( ( FixedValue & <NUM_LIT> ) >> <NUM_LIT> ) ; MachO :: any_relocation_info MRE ; MRE . r_word0 = ( ( OtherHalf << <NUM_LIT> ) | ( MachO :: <ISA_LIT>_RELOC_PAIR << <NUM_LIT> ) | ( MovtBit << <NUM_LIT> ) | ( ThumbBit << <NUM_LIT> ) | ( IsPCRel << <NUM_LIT> ) | MachO :: R_SCATTERED ) ; MRE . r_word1 = Value2 ; Writer -> addRelocation ( nullptr , Fragment -> getParent ( ) , MRE ) ; } MachO :: any_relocation_info MRE ; MRE . r_word0 = ( ( FixupOffset << <NUM_LIT> ) | ( Type << <NUM_LIT> ) | ( MovtBit << <NUM_LIT> ) | ( ThumbBit << <NUM_LIT> ) | ( IsPCRel << <NUM_LIT> ) | MachO :: R_SCATTERED ) ; MRE . r_word1 = Value ; Writer -> addRelocation ( nullptr , Fragment -> getParent (


<s> case MachO :: <ISA_LIT>_RELOC_BR24 : if ( ! S . isTemporary ( ) ) return true ; Value -= <NUM_LIT> ; Range = <NUM_LIT> ; break ; case MachO :: <ISA_LIT>_THUMB_RELOC_BR22 : Value -= <NUM_LIT> ; Range = <NUM_LIT> ; } Value += Writer -> getSectionAddress ( & S . getSection ( ) ) ; Value -= Writer -> getSectionAddress ( Fragment . getParent ( ) ) ; if ( Value > Range || Value < - ( Range + <NUM_LIT> ) )


<s> Value2 = Writer -> getSymbolAddress ( B_SD , Layout ) ; FixedValue -= Writer -> getSectionAddress ( B_SD -> getFragment ( ) -> getParent ( ) ) ; } if ( Type == MachO :: <ISA_LIT>_RELOC_SECTDIFF || Type == MachO :: <ISA_LIT>_RELOC_LOCAL_SECTDIFF ) { MachO :: any_relocation_info MRE ; MRE . r_word0 = ( ( <NUM_LIT> << <NUM_LIT> ) | ( MachO :: <ISA_LIT>_RELOC_PAIR << <NUM_LIT> ) | ( Log2Size << <NUM_LIT> ) | ( IsPCRel << <NUM_LIT> ) | MachO :: R_SCATTERED ) ; MRE . r_word1 = Value2 ; Writer -> addRelocation ( Fragment -> getParent ( ) , MRE ) ; } MachO :: any_relocation_info MRE ; MRE . r_word0 = ( ( FixupOffset << <NUM_LIT> ) | ( Type << <NUM_LIT> ) | ( Log2Size << <NUM_LIT> ) | ( IsPCRel << <NUM_LIT> ) | MachO :: R_SCATTERED ) ; MRE . r_word1 = Value ; Writer -> addRelocation ( Fragment ->


<s> unsigned Type = <NUM_LIT> ; if ( Target . isAbsolute ( ) ) { report_fatal_error ( <STR_LIT> <STR_LIT> ) ; } else { if ( SD -> getSymbol ( ) . isVariable ( ) ) { int64_t Res ; if ( SD -> getSymbol ( ) . getVariableValue ( ) -> EvaluateAsAbsolute ( Res , Layout , Writer -> getSectionAddressMap ( ) ) ) { FixedValue = Res ; return ; } } if ( requiresExternRelocation ( Writer , Asm , * Fragment , RelocType , SD , FixedValue ) ) { IsExtern = <NUM_LIT> ; Index = SD -> getIndex ( ) ; if ( ! SD -> Symbol -> isUndefined ( ) ) FixedValue -= Layout . getSymbolOffset ( SD ) ; } else { const MCSectionData & SymSD = Asm . getSectionData ( SD -> getSymbol ( ) . getSection ( ) ) ; Index = SymSD . getOrdinal ( ) + <NUM_LIT> ; FixedValue += Writer -> getSectionAddress ( & SymSD ) ; } if ( IsPCRel ) FixedValue -= Writer -> getSectionAddress ( Fragment -> getParent ( ) ) ; Type = RelocType ; } MachO :: any_relocation_info MRE ; MRE . r_word0 = FixupOffset ; MRE . r_word1 = ( ( Index << <NUM_LIT> ) | ( IsPCRel << <NUM_LIT> ) | ( Log2Size << <NUM_LIT> ) | ( IsExtern << <NUM_LIT> ) | ( Type << <NUM_LIT> ) ) ; if ( Type == MachO :: <ISA_LIT>_RELOC_HALF ) { uint32_t Value = <NUM_LIT> ; switch ( ( unsigned ) Fixup .