<s> for ( auto & MI : * MBB ) { if ( isLoopStart ( MI ) ) return & MI ; } if ( MBB -> pred_size ( ) == <NUM_LIT> ) return SearchForStart ( * MBB -> pred_begin ( ) ) ; return nullptr ; } ; LowOverheadLoop LoLoop ( * ML , * MLI , * RDA , * TRI , * TII ) ; if ( LoLoop . Preheader ) LoLoop . Start = SearchForStart ( LoLoop . Preheader ) ; else return Changed ; for ( auto * MBB : reverse ( ML -> getBlocks ( ) ) ) { for ( auto & MI : * MBB ) { if ( MI . isDebugValue ( ) ) continue ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . Dec = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . End = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . End = LoLoop . Dec = & MI ; else if ( isLoopStart ( MI ) ) LoLoop . Start = & MI ; else if ( MI . getDesc ( ) . isCall ( ) ) { LoLoop . Revert = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } else { LoLoop . AnalyseMVEInst ( & MI ) ; } } } LLVM_DEBUG ( LoLoop . dump ( ) ) ; if ( ! LoLoop . FoundAllComponents ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return Changed ; } assert ( LoLoop . Start -> getOpcode ( ) != <STR_LIT> && <STR_LIT> ) ;


<s> MachineInstr * VCMP = VCMPOpcodeToVPT ( Divergent -> getOpcode ( ) ) != <NUM_LIT> ? Divergent : nullptr ; if ( DivergentNextIsPredicated ) { if ( ! VCMP ) { MachineInstrBuilder MIB = BuildMI ( * Divergent -> getParent ( ) , Divergent , Divergent -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . addImm ( <NUM_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; LoLoop . BlockMasksToRecompute . insert ( MIB . getInstr ( ) ) ; } else { ReplaceVCMPWithVPT ( VCMP , VCMP ) ; } } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * VPST ) ; LoLoop . ToRemove . insert ( VPST ) ; } else if ( Block . containsVCTP ( ) ) { MachineInstr * VPST = Insts . front ( ) ; if ( Block . size ( ) == <NUM_LIT> ) { assert ( VPST -> getOpcode ( ) == <STR_LIT> && <STR_LIT> ) ; LoLoop . ToRemove . insert ( VPST ) ; } else LoLoop . BlockMasksToRecompute . insert ( VPST ) ; } else if ( Insts . front ( ) -> getOpcode ( ) == <STR_LIT> ) { MachineInstr * VPST = Insts . front ( ) ; auto Next = ++ MachineBasicBlock :: iterator ( VPST ) ; assert ( getVPTInstrPredicate ( * Next ) != <STR_LIT> && <STR_LIT> ) ; ( void ) Next ; MachineInstr * VprDef = RDA -> getUniqueReachingMIDef ( VPST , <STR_LIT> ) ; if ( VprDef && VCMPOpcodeToVPT ( VprDef -> getOpcode ( ) ) && ! LoLoop . ToRemove . contains ( VprDef ) ) { MachineInstr * VCMP = VprDef ; if ( ! std :: any_of ( ++ MachineBasicBlock :: iterator ( VCMP ) , MachineBasicBlock :: iterator ( VPST ) , hasVPRUse ) && RDA -> hasSameReachingDef ( VCMP , VPST , VCMP -> getOperand ( <NUM_LIT> ) . getReg ( ) ) && RDA -> hasSameReachingDef ( VCMP , VPST , VCMP -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { ReplaceVCMPWithVPT ( VCMP , VPST ) ;


<s> unsigned Off = LoLoop . Dec == LoLoop . End ? <NUM_LIT> : <NUM_LIT> ; MIB . add ( End -> getOperand ( Off + <NUM_LIT> ) ) ; MIB . add ( End -> getOperand ( Off + <NUM_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; LoLoop . ToRemove . insert ( LoLoop . Dec ) ; LoLoop . ToRemove . insert ( End ) ; return & * MIB ; } ; auto RemoveDeadBranch = [ ] ( MachineInstr * I ) { MachineBasicBlock * BB = I -> getParent ( ) ; MachineInstr * Terminator = & BB -> instr_back ( ) ; if ( Terminator -> isUnconditionalBranch ( ) && I != Terminator ) { MachineBasicBlock * Succ = Terminator -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( BB -> isLayoutSuccessor ( Succ ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Terminator ) ; Terminator -> eraseFromParent ( ) ;


<s> MachineBasicBlock * MBB = LoLoop . StartInsertBB ; unsigned Opc = LoLoop . getStartOpcode ( ) ; MachineOperand & Count = LoLoop . getLoopStartOperand ( ) ; MachineInstrBuilder MIB = BuildMI ( * MBB , InsertPt , Start -> getDebugLoc ( ) , TII -> get ( Opc ) ) ; MIB . addDef ( <STR_LIT> ) ; MIB . add ( Count ) ; if ( ! isDo ( Start ) ) MIB . add ( Start -> getOperand ( <NUM_LIT> ) ) ; LoLoop . ToRemove . insert ( Start ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; return & * MIB ;


<s> IterationCountDCE ( LoLoop ) ; MachineBasicBlock :: iterator InsertPt = LoLoop . StartInsertPt ; MachineInstr * Start = LoLoop . Start ; MachineBasicBlock * MBB = LoLoop . StartInsertBB ; unsigned Opc = LoLoop . getStartOpcode ( ) ; MachineOperand & Count = LoLoop . getLoopStartOperand ( ) ; MachineInstrBuilder MIB = BuildMI ( * MBB , InsertPt , Start -> getDebugLoc ( ) , TII -> get ( Opc ) ) ;


<s> if ( IsTailPredicationLegal ( ) ) return TPNumElements ;

return isDo ( Start ) ? Start -> getOperand ( <NUM_LIT> ) : Start -> getOperand ( <NUM_LIT> ) ;


<s> unsigned getStartOpcode ( ) const { bool IsDo = isDo ( Start ) ; if ( ! IsTailPredicationLegal ( ) ) return IsDo ? <STR_LIT> : <STR_LIT> ; return VCTPOpcodeToLSTP ( VCTPs . back ( ) -> getOpcode ( ) , IsDo ) ;


<s> return MI -> getOpcode ( ) != <STR_LIT> ;


<s> void <ISA_LIT>LowOverheadLoops :: IterationCountDCE ( LowOverheadLoop & LoLoop ) { if ( ! LoLoop . IsTailPredicationLegal ( ) ) return ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ;

MachineInstr * Def = RDA -> getMIOperand ( LoLoop . Start , isDo ( LoLoop . Start ) ? <NUM_LIT> : <NUM_LIT> ) ;


<s> MachineInstr * Def = RDA -> getMIOperand ( LoLoop . Start , isDo ( LoLoop . Start ) ? <NUM_LIT> : <NUM_LIT> ) ; if ( ! Def ) {


<s> LowOverheadLoop ( MachineLoop & ML , MachineLoopInfo & MLI , ReachingDefAnalysis & RDA , const TargetRegisterInfo & TRI , const <ISA_LIT>BaseInstrInfo & TII ) : ML ( ML ) , MLI ( MLI ) , RDA ( RDA ) , TRI ( TRI ) , TII ( TII ) , TPNumElements ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) { MF = ML . getHeader ( ) -> getParent ( ) ; if ( auto * MBB = ML . getLoopPreheader ( ) ) Preheader = MBB ; else if ( auto * MBB = MLI . findLoopPreheader ( & ML , true ) ) Preheader = MBB ;


<s> auto IsZeroInit = [ ] ( MachineInstr * Def ) { return ! isVectorPredicated ( Def ) && Def -> getOpcode ( ) == <STR_LIT> && Def -> getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> ; } ; bool AllowScalars = isHorizontalReduction ( MI ) ; for ( auto & MO : MI . operands ( ) ) { if ( ! MO . isReg ( ) || ! MO . getReg ( ) ) continue ; if ( ! isRegInClass ( MO , QPRs ) && AllowScalars ) continue ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Defs ; RDA . getGlobalReachingDefs ( & MI , MO . getReg ( ) , Defs ) ; for ( auto * Def : Defs ) { if ( Def == & MI || FalseLanesZero . count ( Def ) || IsZeroInit ( Def ) ) continue ; if ( MO . isUse ( ) && isPredicated ) continue ; return false ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ;


<s> auto IsZeroInit = [ ] ( MachineInstr * Def ) { return ! isVectorPredicated ( Def ) && Def -> getOpcode ( ) == <STR_LIT> && Def -> getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> ; } ; bool AllowScalars = isHorizontalReduction ( MI ) ; for ( auto & MO : MI . operands ( ) ) { if ( ! MO . isReg ( ) || ! MO . getReg ( ) ) continue ; if ( ! isRegInClass ( MO , QPRs ) && AllowScalars ) continue ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Defs ; RDA . getGlobalReachingDefs ( & MI , MO . getReg ( ) , Defs ) ; for ( auto * Def : Defs ) { if ( Def == & MI || FalseLanesZero . count ( Def ) || IsZeroInit ( Def ) ) continue ; if ( MO . isUse ( ) && isPredicated ) continue ;


<s> bool Changed = false ; for ( auto & MBB : * MF ) { SmallVector < MachineInstr * , <NUM_LIT> > Starts ; SmallVector < MachineInstr * , <NUM_LIT> > Decs ; SmallVector < MachineInstr * , <NUM_LIT> > Ends ; SmallVector < MachineInstr * , <NUM_LIT> > EndDecs ; for ( auto & I : MBB ) { if ( isLoopStart ( I ) ) Starts . push_back ( & I ) ; else if ( I . getOpcode ( ) == <STR_LIT> ) Decs . push_back ( & I ) ; else if ( I . getOpcode ( ) == <STR_LIT> ) Ends . push_back ( & I ) ; else if ( I . getOpcode ( ) == <STR_LIT> ) EndDecs . push_back ( & I ) ; } if ( Starts . empty ( ) && Decs . empty ( ) && Ends . empty ( ) && EndDecs . empty ( ) ) continue ; Changed = true ; for ( auto * Start : Starts ) { if ( Start -> getOpcode ( ) == <STR_LIT> ) RevertWhile ( Start ) ;


<s> void <ISA_LIT>LowOverheadLoops :: RevertWhile ( MachineInstr * MI ) const { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; MachineBasicBlock * DestBB = MI -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; unsigned BrOpc = BBUtils -> isBBInRange ( MI , DestBB , <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ;

RevertWhileLoopStart ( MI , TII , BrOpc ) ;


<s> MachineBasicBlock * DestBB = MI -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; unsigned BrOpc = BBUtils -> isBBInRange ( MI , DestBB , <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; RevertWhileLoopStart ( MI , TII , BrOpc ) ;


<s> static bool shouldInspect ( MachineInstr & MI ) { return isDomainMVE ( & MI ) || isVectorPredicate ( & MI ) || hasVPRUse ( MI ) ;


<s> return isDomainMVE ( & MI ) || isVectorPredicate ( & MI ) || hasVPRUse ( MI ) ;


<s> return true ; } ; auto FindStartInsertionPoint = [ ] ( MachineInstr * Start , MachineInstr * Dec , MachineBasicBlock :: iterator & InsertPt , MachineBasicBlock * & InsertBB , ReachingDefAnalysis & RDA , InstSet & ToRemove ) { if ( isDo ( Start ) || Start -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) { InsertPt = MachineBasicBlock :: iterator ( Start ) ; InsertBB = Start -> getParent ( ) ; return true ; } if ( ! RDA . isSafeToDefRegAt ( Start , MCRegister :: from ( <STR_LIT> ) ) ) return false ; InsertPt = MachineBasicBlock :: iterator ( Start ) ; InsertBB = Start -> getParent ( ) ; return true ; } ; if ( ! FindStartInsertionPoint ( Start , Dec , StartInsertPt , StartInsertBB , RDA , ToRemove ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Revert = true ;


<s> return false ; } if ( Start -> getOpcode ( ) == <STR_LIT> && ( BBUtils -> getOffsetOf ( Start ) > BBUtils -> getOffsetOf ( Start -> getOperand ( <NUM_LIT> ) . getMBB ( ) ) || ! BBUtils -> isBBInRange ( Start , Start -> getOperand ( <NUM_LIT> ) . getMBB ( ) , <NUM_LIT> ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } return true ; } ; auto FindStartInsertionPoint = [ ] ( MachineInstr * Start , MachineInstr * Dec , MachineBasicBlock :: iterator & InsertPt , MachineBasicBlock * & InsertBB , ReachingDefAnalysis & RDA , InstSet & ToRemove ) { if ( isDo ( Start ) || Start -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) { InsertPt = MachineBasicBlock :: iterator ( Start ) ; InsertBB = Start -> getParent ( ) ; return true ; }

if ( ! RDA . isSafeToDefRegAt ( Start , MCRegister :: from ( <STR_LIT> ) ) ) return false ;


<s> } if ( isVCTP ( MI ) && ! AddVCTP ( MI ) ) return false ; const MCInstrDesc & MCID = MI -> getDesc ( ) ; bool IsUse = false ; unsigned LastOpIdx = MI -> getNumOperands ( ) - <NUM_LIT> ; for ( auto & Op : enumerate ( reverse ( MCID . operands ( ) ) ) ) { const MachineOperand & MO = MI -> getOperand ( LastOpIdx - Op . index ( ) ) ; if ( ! MO . isReg ( ) || ! MO . isUse ( ) || MO . getReg ( ) != <STR_LIT> ) continue ; if ( <STR_LIT> ( Op . value ( ) . OperandType ) ) { VPTState :: addInst ( MI ) ; IsUse = true ; } else if ( MI -> getOpcode ( ) != <STR_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; return false ; } }

bool RequiresExplicitPredication = ( MCID . TSFlags & <STR_LIT> ) == <NUM_LIT> ;


<s> ElemDef -> removeFromParent ( ) ; StartInsertBB -> insert ( StartInsertPt , ElemDef ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ElemDef ) ; } else if ( RDA . isSafeToMoveBackwards ( & * StartInsertPt , ElemDef ) ) { StartInsertPt -> removeFromParent ( ) ; StartInsertBB -> insertAfter ( MachineBasicBlock :: iterator ( ElemDef ) , & * StartInsertPt ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ElemDef ) ; } else { MachineOperand Operand = ElemDef -> getOperand ( <NUM_LIT> ) ; if ( isMovRegOpcode ( ElemDef -> getOpcode ( ) ) && RDA . getUniqueReachingMIDef ( ElemDef , Operand . getReg ( ) . asMCReg ( ) ) == RDA . getUniqueReachingMIDef ( & * StartInsertPt , Operand . getReg ( ) . asMCReg ( ) ) ) { TPNumElements = Operand ; NumElements = TPNumElements . getReg ( ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return false ; } } } } auto CannotProvideElements = [ this ] ( MachineBasicBlock * MBB , MCRegister NumElements ) { if ( MBB -> empty ( ) ) return false ; if ( RDA . hasLocalDefBefore ( & MBB -> back ( ) , NumElements ) ) return true ; if ( MBB -> pred_size ( ) > <NUM_LIT> ) return true ; return false ; } ; MachineBasicBlock * MBB = Preheader ; while ( MBB && MBB != StartInsertBB ) { if ( CannotProvideElements ( MBB , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MBB = * MBB -> pred_begin ( ) ; } } if ( std :: any_of ( StartInsertPt , StartInsertBB -> end ( ) , shouldInspect ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto IsValidSub = [ ] ( MachineInstr * MI , int ExpectedVecWidth ) { return - getAddSubImmediate ( * MI ) == ExpectedVecWidth ; } ; MachineBasicBlock * MBB = VCTP -> getParent ( ) ; if ( auto * Def = RDA . getUniqueReachingMIDef ( & MBB -> back ( ) , VCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) . asMCReg ( ) ) ) { SmallPtrSet < MachineInstr * , <NUM_LIT> > ElementChain ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Ignore ;

unsigned ExpectedVectorWidth = getTailPredVectorWidth ( VCTP -> getOpcode ( ) ) ;


<s> } assert ( ! VCTPs . empty ( ) && <STR_LIT> ) ; assert ( ML . getBlocks ( ) . size ( ) == <NUM_LIT> && <STR_LIT> ) ; if ( DisableTailPredication ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( ! VPTState :: isValid ( RDA ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( ! ValidateLiveOuts ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } Register StartReg = isDo ( Start ) ? Start -> getOperand ( <NUM_LIT> ) . getReg ( ) : Start -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( StartInsertPt == Start && StartReg == <STR_LIT> ) { if ( auto * IterCount = RDA . getMIOperand ( Start , isDo ( Start ) ? <NUM_LIT> : <NUM_LIT> ) ) { SmallPtrSet < MachineInstr * , <NUM_LIT> > Uses ; RDA . getGlobalUses ( IterCount , MCRegister :: from ( <STR_LIT> ) , Uses ) ; for ( auto * Use : Uses ) { if ( Use != Start && Use != Dec ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Use ) ; return false ; } } } } MachineInstr * VCTP = VCTPs . back ( ) ; if ( Start -> getOpcode ( ) == <STR_LIT> ) { TPNumElements = Start -> getOperand ( <NUM_LIT> ) ; StartInsertPt = Start ; StartInsertBB = Start -> getParent ( ) ; } else { TPNumElements = VCTP -> getOperand ( <NUM_LIT> ) ; MCRegister NumElements = TPNumElements . getReg ( ) . asMCReg ( ) ; if ( RDA . hasLocalDefBefore ( VCTP , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( StartInsertPt != StartInsertBB -> end ( ) && ! RDA . isReachingDefLiveOut ( & * StartInsertPt , NumElements ) ) { if ( auto * ElemDef = RDA . getLocalLiveOutMIDef ( StartInsertBB , NumElements ) ) { if ( RDA . isSafeToMoveForwards ( ElemDef , & * StartInsertPt ) ) { ElemDef -> removeFromParent ( ) ; StartInsertBB -> insert ( StartInsertPt , ElemDef ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ElemDef ) ; } else if ( RDA . isSafeToMoveBackwards ( & * StartInsertPt , ElemDef ) ) {


<s> MachineInstr * <ISA_LIT>LowOverheadLoops :: ExpandLoopStart ( LowOverheadLoop & LoLoop ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; IterationCountDCE ( LoLoop ) ; MachineBasicBlock :: iterator InsertPt = LoLoop . StartInsertPt ; MachineInstr * Start = LoLoop . Start ; MachineBasicBlock * MBB = LoLoop . StartInsertBB ; unsigned Opc = LoLoop . getStartOpcode ( ) ; MachineOperand & Count = LoLoop . getLoopStartOperand ( ) ; MachineInstr * NewStart ; if ( Opc == <STR_LIT> && Count . isReg ( ) && Count . getReg ( ) == <STR_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; NewStart = nullptr ; } else { MachineInstrBuilder MIB = BuildMI ( * MBB , InsertPt , Start -> getDebugLoc ( ) , TII -> get ( Opc ) ) ; MIB . addDef ( <STR_LIT> ) ; MIB . add ( Count ) ; if ( ! isDo ( Start ) ) MIB . add ( Start -> getOperand ( <NUM_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; NewStart = & * MIB ;


<s> MachineOperand & Count = LoLoop . getLoopStartOperand ( ) ; MachineInstr * NewStart ; if ( Opc == <STR_LIT> && Count . isReg ( ) && Count . getReg ( ) == <STR_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; NewStart = nullptr ; } else { MachineInstrBuilder MIB = BuildMI ( * MBB , InsertPt , Start -> getDebugLoc ( ) , TII -> get ( Opc ) ) ; MIB . addDef ( <STR_LIT> ) ; MIB . add ( Count ) ; if ( ! isDo ( Start ) ) MIB . add ( Start -> getOperand ( <NUM_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; NewStart = & * MIB ; }


<s> static bool isDo ( MachineInstr * MI ) { return MI -> getOpcode ( ) != <STR_LIT> ;


<s> else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . End = LoLoop . Dec = & MI ; else if ( isLoopStart ( MI ) ) LoLoop . Start = & MI ; else if ( MI . getDesc ( ) . isCall ( ) ) { LoLoop . Revert = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } else { LoLoop . AnalyseMVEInst ( & MI ) ; } } } LLVM_DEBUG ( LoLoop . dump ( ) ) ; if ( ! LoLoop . FoundAllComponents ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } assert ( LoLoop . Start -> getOpcode ( ) != <STR_LIT> && <STR_LIT> ) ; if ( LoLoop . Dec != LoLoop . End ) { SmallPtrSet < MachineInstr * , <NUM_LIT> > Uses ; RDA -> getReachingLocalUses ( LoLoop . Dec , MCRegister :: from ( <STR_LIT> ) , Uses ) ;

if ( Uses . size ( ) > <NUM_LIT> || ! Uses . count ( LoLoop . End ) ) {


<s> bool <ISA_LIT>LowOverheadLoops :: ProcessLoop ( MachineLoop * ML ) { bool Changed = false ; for ( auto I = ML -> begin ( ) , E = ML -> end ( ) ; I != E ; ++ I ) Changed |= ProcessLoop ( * I ) ; LLVM_DEBUG ( { dbgs ( ) << <STR_LIT> ; if ( auto * Preheader = ML -> getLoopPreheader ( ) ) dbgs ( ) << <STR_LIT> << printMBBReference ( * Preheader ) << <STR_LIT> ; else if ( auto * Preheader = MLI -> findLoopPreheader ( ML , true , true ) ) dbgs ( ) << <STR_LIT> << printMBBReference ( * Preheader ) << <STR_LIT> ; for ( auto * MBB : ML -> getBlocks ( ) ) dbgs ( ) << <STR_LIT> << printMBBReference ( * MBB ) << <STR_LIT> ; } ) ; std :: function < MachineInstr * ( MachineBasicBlock * ) > SearchForStart = [ & SearchForStart ] ( MachineBasicBlock * MBB ) -> MachineInstr * { for ( auto & MI : * MBB ) { if ( isLoopStart ( MI ) ) return & MI ; } if ( MBB -> pred_size ( ) == <NUM_LIT> ) return SearchForStart ( * MBB -> pred_begin ( ) ) ; return nullptr ; } ; LowOverheadLoop LoLoop ( * ML , * MLI , * RDA , * TRI , * TII ) ; if ( LoLoop . Preheader ) LoLoop . Start = SearchForStart ( LoLoop . Preheader ) ; else return false ; for ( auto * MBB : reverse ( ML -> getBlocks ( ) ) ) { for ( auto & MI : * MBB ) { if ( MI . isDebugValue ( ) ) continue ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . Dec = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . End = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . End = LoLoop . Dec = & MI ;

else if ( isLoopStart ( MI ) ) LoLoop . Start = & MI ;


<s> bool Changed = false ; for ( auto & MBB : * MF ) { SmallVector < MachineInstr * , <NUM_LIT> > Starts ; SmallVector < MachineInstr * , <NUM_LIT> > Decs ; SmallVector < MachineInstr * , <NUM_LIT> > Ends ; SmallVector < MachineInstr * , <NUM_LIT> > EndDecs ; for ( auto & I : MBB ) { if ( isLoopStart ( I ) ) Starts . push_back ( & I ) ; else if ( I . getOpcode ( ) == <STR_LIT> ) Decs . push_back ( & I ) ; else if ( I . getOpcode ( ) == <STR_LIT> ) Ends . push_back ( & I ) ; else if ( I . getOpcode ( ) == <STR_LIT> ) EndDecs . push_back ( & I ) ; }

if ( Starts . empty ( ) && Decs . empty ( ) && Ends . empty ( ) && EndDecs . empty ( ) ) continue ;


<s> bool Changed = false ; for ( auto & MBB : * MF ) { SmallVector < MachineInstr * , <NUM_LIT> > Starts ; SmallVector < MachineInstr * , <NUM_LIT> > Decs ; SmallVector < MachineInstr * , <NUM_LIT> > Ends ; SmallVector < MachineInstr * , <NUM_LIT> > EndDecs ; for ( auto & I : MBB ) { if ( isLoopStart ( I ) ) Starts . push_back ( & I ) ; else if ( I . getOpcode ( ) == <STR_LIT> ) Decs . push_back ( & I ) ; else if ( I . getOpcode ( ) == <STR_LIT> ) Ends . push_back ( & I ) ; else if ( I . getOpcode ( ) == <STR_LIT> ) EndDecs . push_back ( & I ) ;


<s> MachineBasicBlock * DestBB = MI -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; unsigned BrOpc = BBUtils -> isBBInRange ( MI , DestBB , <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; RevertWhileLoopStartLR ( MI , TII , BrOpc ) ;


<s> unsigned BrOpc = BBUtils -> isBBInRange ( MI , DestBB , <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; RevertWhileLoopStartLR ( MI , TII , BrOpc ) ;


<s> auto ValidateRanges = [ ] ( MachineInstr * Start , MachineInstr * End , <ISA_LIT>BasicBlockUtils * BBUtils , MachineLoop & ML ) { MachineBasicBlock * TgtBB = End -> getOpcode ( ) == <STR_LIT> ? End -> getOperand ( <NUM_LIT> ) . getMBB ( ) : End -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( TgtBB != ML . getHeader ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( BBUtils -> getOffsetOf ( End ) < BBUtils -> getOffsetOf ( ML . getHeader ( ) ) || ! BBUtils -> isBBInRange ( End , ML . getHeader ( ) , <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( Start -> getOpcode ( ) == <STR_LIT> && ( BBUtils -> getOffsetOf ( Start ) > BBUtils -> getOffsetOf ( Start -> getOperand ( <NUM_LIT> ) . getMBB ( ) ) || ! BBUtils -> isBBInRange ( Start , Start -> getOperand ( <NUM_LIT> ) . getMBB ( ) , <NUM_LIT> ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } return true ; } ; StartInsertPt = MachineBasicBlock :: iterator ( Start ) ;

StartInsertBB = Start -> getParent ( ) ;


<s> TPNumElements = VCTP -> getOperand ( <NUM_LIT> ) ; MCRegister NumElements = TPNumElements . getReg ( ) . asMCReg ( ) ; if ( RDA . hasLocalDefBefore ( VCTP , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( StartInsertPt != StartInsertBB -> end ( ) && ! RDA . isReachingDefLiveOut ( & * StartInsertPt , NumElements ) ) { if ( auto * ElemDef = RDA . getLocalLiveOutMIDef ( StartInsertBB , NumElements ) ) { if ( RDA . isSafeToMoveForwards ( ElemDef , & * StartInsertPt ) ) { ElemDef -> removeFromParent ( ) ; StartInsertBB -> insert ( StartInsertPt , ElemDef ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ElemDef ) ; } else if ( RDA . isSafeToMoveBackwards ( & * StartInsertPt , ElemDef ) ) { StartInsertPt -> removeFromParent ( ) ; StartInsertBB -> insertAfter ( MachineBasicBlock :: iterator ( ElemDef ) , & * StartInsertPt ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ElemDef ) ; } else { MachineOperand Operand = ElemDef -> getOperand ( <NUM_LIT> ) ; if ( isMovRegOpcode ( ElemDef -> getOpcode ( ) ) && RDA . getUniqueReachingMIDef ( ElemDef , Operand . getReg ( ) . asMCReg ( ) ) == RDA . getUniqueReachingMIDef ( & * StartInsertPt , Operand . getReg ( ) . asMCReg ( ) ) ) { TPNumElements = Operand ; NumElements = TPNumElements . getReg ( ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return false ; } } } } auto CannotProvideElements = [ this ] ( MachineBasicBlock * MBB , MCRegister NumElements ) { if ( MBB -> empty ( ) ) return false ; if ( RDA . hasLocalDefBefore ( & MBB -> back ( ) , NumElements ) ) return true ; if ( MBB -> pred_size ( ) > <NUM_LIT> ) return true ; return false ; } ; MachineBasicBlock * MBB = Preheader ; while ( MBB && MBB != StartInsertBB ) { if ( CannotProvideElements ( MBB , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ;


<s> MachineBasicBlock * MBB = End -> getParent ( ) ; MachineInstrBuilder MIB = BuildMI ( * MBB , End , End -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . addDef ( <STR_LIT> ) ; MIB . add ( End -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( End -> getOperand ( <NUM_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; MachineBasicBlock * Latch = End -> getParent ( ) ; MachineInstr * Terminator = & Latch -> instr_back ( ) ; if ( End != Terminator ) { MachineBasicBlock * Exit = ML -> getExitBlock ( ) ; if ( Latch -> isLayoutSuccessor ( Exit ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Terminator ) ; Terminator -> eraseFromParent ( ) ; } } End -> eraseFromParent ( ) ; Dec -> eraseFromParent ( ) ; } ; auto ExpandBranch = [ this ] ( MachineInstr * Dec , MachineInstr * End ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; MachineBasicBlock * MBB = Dec -> getParent ( ) ; MachineInstrBuilder MIB = BuildMI ( * MBB , Dec , Dec -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . addDef ( <STR_LIT> ) ; MIB . add ( Dec -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( Dec -> getOperand ( <NUM_LIT> ) ) ;


<s> MIB . add ( End -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( End -> getOperand ( <NUM_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; MachineBasicBlock * Latch = End -> getParent ( ) ; MachineInstr * Terminator = & Latch -> instr_back ( ) ; if ( End != Terminator ) { MachineBasicBlock * Exit = ML -> getExitBlock ( ) ; if ( Latch -> isLayoutSuccessor ( Exit ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Terminator ) ; Terminator -> eraseFromParent ( ) ; } } End -> eraseFromParent ( ) ; Dec -> eraseFromParent ( ) ; } ; auto ExpandBranch = [ this ] ( MachineInstr * Dec , MachineInstr * End ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; MachineBasicBlock * MBB = Dec -> getParent ( ) ; MachineInstrBuilder MIB = BuildMI ( * MBB , Dec , Dec -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . addDef ( <STR_LIT> ) ; MIB . add ( Dec -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( Dec -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <STR_LIT> ) ;


<s> for ( auto & MI : * MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) Dec = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) End = & MI ; else if ( MI . getDesc ( ) . isCall ( ) ) Revert = true ; if ( ! Dec ) continue ; if ( MI . mayLoad ( ) || MI . mayStore ( ) ) Revert = MI . getOperand ( <NUM_LIT> ) . isReg ( ) && MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ; } if ( Dec && End && Revert ) break ; } if ( Start || Dec || End ) { if ( ! Start || ! Dec || ! End ) report_fatal_error ( <STR_LIT> ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return Changed ; } if ( ! End -> getOperand ( <NUM_LIT> ) . isMBB ( ) || End -> getOperand ( <NUM_LIT> ) . getMBB ( ) != ML -> getHeader ( ) ) report_fatal_error ( <STR_LIT> ) ; if ( ! BBUtils -> isBBInRange ( End , ML -> getHeader ( ) , <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Revert = true ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Start << <STR_LIT> << * Dec << <STR_LIT> << * End ) ; Expand ( ML , Start , Dec , End , Revert ) ; return true ;


<s> bool Revert = false ; if ( auto * Preheader = ML -> getLoopPreheader ( ) ) Start = SearchForStart ( Preheader ) ; for ( auto * MBB : reverse ( ML -> getBlocks ( ) ) ) { for ( auto & MI : * MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) Dec = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) End = & MI ; else if ( MI . getDesc ( ) . isCall ( ) ) Revert = true ; if ( ! Dec ) continue ; if ( MI . mayLoad ( ) || MI . mayStore ( ) ) Revert = MI . getOperand ( <NUM_LIT> ) . isReg ( ) && MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ; } if ( Dec && End && Revert ) break ; } if ( Start || Dec || End ) { if ( ! Start || ! Dec || ! End ) report_fatal_error ( <STR_LIT> ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return Changed ; } if ( ! End -> getOperand ( <NUM_LIT> ) . isMBB ( ) || End -> getOperand ( <NUM_LIT> ) . getMBB ( ) != ML -> getHeader ( ) ) report_fatal_error ( <STR_LIT> ) ; if ( ! BBUtils -> isBBInRange ( End , ML -> getHeader ( ) , <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Revert = true ; }

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Start << <STR_LIT> << * Dec << <STR_LIT> << * End ) ;


<s> bool <ISA_LIT>LowOverheadLoops :: runOnMachineFunction ( MachineFunction & MF ) { if ( ! static_cast < const <ISA_LIT>Subtarget & > ( MF . getSubtarget ( ) ) . hasLOB ( ) ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MF . getName ( ) << <STR_LIT> ) ; auto & MLI = getAnalysis < MachineLoopInfo > ( ) ; MRI = & MF . getRegInfo ( ) ; TII = static_cast < const <ISA_LIT>BaseInstrInfo * > ( MF . getSubtarget ( ) . getInstrInfo ( ) ) ;

BBUtils = std :: unique_ptr < <ISA_LIT>BasicBlockUtils > ( new <ISA_LIT>BasicBlockUtils ( MF ) ) ;


<s> void addVCTP ( MachineInstr * MI ) { if ( ! VCTP ) { VCTP = MI ; FoundOneVCTP = true ;


<s> if ( ! IsTailPredicationLegal ( ) ) return ; Register NumElements = VCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( RDA -> getReachingDef ( VCTP , NumElements ) >= <NUM_LIT> ) { CannotTailPredicate = true ; return ; } MachineBasicBlock * InsertBB = InsertPt -> getParent ( ) ; if ( ! RDA -> hasSameReachingDef ( InsertPt , & InsertBB -> back ( ) , NumElements ) ) { CannotTailPredicate = true ; return ; } auto CannotProvideElements = [ & RDA ] ( MachineBasicBlock * MBB , Register NumElements ) { if ( RDA -> getReachingDef ( & MBB -> back ( ) , NumElements ) >= <NUM_LIT> ) return true ; if ( MBB -> pred_size ( ) > <NUM_LIT> ) return true ; return false ; } ; MachineBasicBlock * MBB = MLI -> findLoopPreheader ( ML , true ) ; if ( ! MBB ) { CannotTailPredicate = true ; return ; } while ( MBB != InsertBB ) { CannotTailPredicate = CannotProvideElements ( MBB , NumElements ) ; if ( CannotTailPredicate ) return ; MBB = * MBB -> pred_begin ( ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ;

for ( auto * MI : VPTUsers ) dbgs ( ) << <STR_LIT> << * MI ; ) ;


<s> if ( ( Flags & <STR_LIT> ) == <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; CannotTailPredicate = true ; }


<s> unsigned Opc = LoLoop . IsTailPredicationLegal ( ) ? <STR_LIT> : <STR_LIT> ; MachineInstrBuilder MIB = BuildMI ( * MBB , End , End -> getDebugLoc ( ) , TII -> get ( Opc ) ) ; MIB . addDef ( <STR_LIT> ) ; MIB . add ( End -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( End -> getOperand ( <NUM_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; LoLoop . End -> eraseFromParent ( ) ; LoLoop . Dec -> eraseFromParent ( ) ; return & * MIB ; } ; auto RemoveDeadBranch = [ ] ( MachineInstr * I ) { MachineBasicBlock * BB = I -> getParent ( ) ; MachineInstr * Terminator = & BB -> instr_back ( ) ; if ( Terminator -> isUnconditionalBranch ( ) && I != Terminator ) { MachineBasicBlock * Succ = Terminator -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( BB -> isLayoutSuccessor ( Succ ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Terminator ) ; Terminator -> eraseFromParent ( ) ; } } } ; if ( LoLoop . Revert ) { if ( LoLoop . Start -> getOpcode ( ) == <STR_LIT> ) RevertWhile ( LoLoop . Start ) ; else LoLoop . Start -> eraseFromParent ( ) ; bool FlagsAlreadySet = RevertLoopDec ( LoLoop . Dec , true ) ; RevertLoopEnd ( LoLoop . End , FlagsAlreadySet ) ; } else { LoLoop . Start = ExpandLoopStart ( LoLoop ) ; RemoveDeadBranch ( LoLoop . Start ) ; LoLoop . End = ExpandLoopEnd ( LoLoop ) ; RemoveDeadBranch ( LoLoop . End ) ;

if ( LoLoop . IsTailPredicationLegal ( ) ) {


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; LoLoop . End -> eraseFromParent ( ) ; LoLoop . Dec -> eraseFromParent ( ) ; return & * MIB ; } ; auto RemoveDeadBranch = [ ] ( MachineInstr * I ) { MachineBasicBlock * BB = I -> getParent ( ) ; MachineInstr * Terminator = & BB -> instr_back ( ) ; if ( Terminator -> isUnconditionalBranch ( ) && I != Terminator ) { MachineBasicBlock * Succ = Terminator -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( BB -> isLayoutSuccessor ( Succ ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Terminator ) ; Terminator -> eraseFromParent ( ) ; } } } ; if ( LoLoop . Revert ) { if ( LoLoop . Start -> getOpcode ( ) == <STR_LIT> ) RevertWhile ( LoLoop . Start ) ; else LoLoop . Start -> eraseFromParent ( ) ; bool FlagsAlreadySet = RevertLoopDec ( LoLoop . Dec , true ) ; RevertLoopEnd ( LoLoop . End , FlagsAlreadySet ) ; } else {

LoLoop . Start = ExpandLoopStart ( LoLoop ) ;


<s> if ( LoLoop . IsTailPredicationLegal ( ) ) { SmallVector < MachineInstr * , <NUM_LIT> > Killed ; SmallVector < MachineInstr * , <NUM_LIT> > Dead ; if ( auto * Def = RDA -> getReachingMIDef ( Start , Start -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { Killed . push_back ( Def ) ; while ( ! Killed . empty ( ) ) { MachineInstr * Def = Killed . back ( ) ; Killed . pop_back ( ) ; Dead . push_back ( Def ) ; for ( auto & MO : Def -> operands ( ) ) { if ( ! MO . isReg ( ) || ! MO . isKill ( ) ) continue ; MachineInstr * Kill = RDA -> getReachingMIDef ( Def , MO . getReg ( ) ) ; if ( Kill && RDA -> getNumUses ( Kill , MO . getReg ( ) ) == <NUM_LIT> ) Killed . push_back ( Kill ) ; } } for ( auto * MI : Dead ) MI -> eraseFromParent ( ) ; }


<s> MachineInstrBuilder MIB = BuildMI ( * MBB , InsertPt , InsertPt -> getDebugLoc ( ) , TII -> get ( Opc ) ) ; MIB . addDef ( <STR_LIT> ) ; MIB . add ( Count ) ; if ( ! IsDo ) MIB . add ( Start -> getOperand ( <NUM_LIT> ) ) ; if ( LoLoop . IsTailPredicationLegal ( ) ) { SmallVector < MachineInstr * , <NUM_LIT> > Killed ; SmallVector < MachineInstr * , <NUM_LIT> > Dead ; if ( auto * Def = RDA -> getReachingMIDef ( Start , Start -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { Killed . push_back ( Def ) ; while ( ! Killed . empty ( ) ) { MachineInstr * Def = Killed . back ( ) ; Killed . pop_back ( ) ; Dead . push_back ( Def ) ; for ( auto & MO : Def -> operands ( ) ) { if ( ! MO . isReg ( ) || ! MO . isKill ( ) ) continue ; MachineInstr * Kill = RDA -> getReachingMIDef ( Def , MO . getReg ( ) ) ; if ( Kill && RDA -> getNumUses ( Kill , MO . getReg ( ) ) == <NUM_LIT> ) Killed . push_back ( Kill ) ; } }

for ( auto * MI : Dead ) MI -> eraseFromParent ( ) ;


<s> return MI -> getOpcode ( ) == <STR_LIT> && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == CountReg && MI -> getOperand ( <NUM_LIT> ) . getImm ( ) == <STR_LIT> ; } ; MachineBasicBlock * MBB = Start -> getParent ( ) ; if ( auto * LRDef = RDA -> getReachingMIDef ( & MBB -> back ( ) , <STR_LIT> ) ) { if ( IsMoveLR ( LRDef ) && RDA -> hasSameReachingDef ( Start , LRDef , CountReg ) ) return LRDef ;


<s> default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ;


<s> LowOverheadLoop ( MachineLoop * ML ) : ML ( ML ) { MF = ML -> getHeader ( ) -> getParent ( ) ;


<s> MF = ML -> getHeader ( ) -> getParent ( ) ;


<s> if ( IsLoopStart ( MI ) ) return & MI ; } if ( MBB -> pred_size ( ) == <NUM_LIT> ) return SearchForStart ( * MBB -> pred_begin ( ) ) ; return nullptr ; } ; LowOverheadLoop LoLoop ( ML ) ; if ( auto * Preheader = ML -> getLoopPreheader ( ) ) LoLoop . Start = SearchForStart ( Preheader ) ; else if ( auto * Preheader = MLI -> findLoopPreheader ( ML , true ) ) LoLoop . Start = SearchForStart ( Preheader ) ; else return false ; for ( auto * MBB : reverse ( ML -> getBlocks ( ) ) ) { for ( auto & MI : * MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . Dec = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . End = & MI ; else if ( IsLoopStart ( MI ) ) LoLoop . Start = & MI ; else if ( IsVCTP ( & MI ) ) LoLoop . addVCTP ( & MI ) ; else if ( MI . getDesc ( ) . isCall ( ) ) { LoLoop . Revert = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } else { if ( LoLoop . FoundOneVCTP ) LoLoop . ScanForVPR ( & MI ) ; LoLoop . CheckTPValidity ( & MI ) ; } if ( ! LoLoop . Dec || LoLoop . End || LoLoop . Revert ) continue ; for ( auto & MO : MI . operands ( ) ) { if ( MI . getOpcode ( ) != <STR_LIT> && MO . isReg ( ) && MO . getReg ( ) == <STR_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; LoLoop . Revert = true ; break ; } } } }


<s> if ( MBB -> pred_size ( ) == <NUM_LIT> ) return SearchForStart ( * MBB -> pred_begin ( ) ) ; return nullptr ; } ; LowOverheadLoop LoLoop ( ML ) ; if ( auto * Preheader = ML -> getLoopPreheader ( ) ) LoLoop . Start = SearchForStart ( Preheader ) ; else if ( auto * Preheader = MLI -> findLoopPreheader ( ML , true ) ) LoLoop . Start = SearchForStart ( Preheader ) ; else return false ; for ( auto * MBB : reverse ( ML -> getBlocks ( ) ) ) { for ( auto & MI : * MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . Dec = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . End = & MI ; else if ( IsLoopStart ( MI ) ) LoLoop . Start = & MI ; else if ( IsVCTP ( & MI ) ) LoLoop . addVCTP ( & MI ) ; else if ( MI . getDesc ( ) . isCall ( ) ) { LoLoop . Revert = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } else { if ( LoLoop . FoundOneVCTP ) LoLoop . ScanForVPR ( & MI ) ; LoLoop . CheckTPValidity ( & MI ) ; } if ( ! LoLoop . Dec || LoLoop . End || LoLoop . Revert ) continue ; for ( auto & MO : MI . operands ( ) ) { if ( MI . getOpcode ( ) != <STR_LIT> && MO . isReg ( ) && MO . getReg ( ) == <STR_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ;


<s> if ( Uses . size ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return ; } Uses . clear ( ) ; RDA -> getAllInstWithUseBefore ( Def , ElemCount , Uses ) ; if ( ! Uses . size ( ) || ( Uses . size ( ) == <NUM_LIT> && Uses [ <NUM_LIT> ] == LoLoop . VCTP ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; Def -> dump ( ) ) ; Def -> eraseFromParent ( ) ;


<s> assert ( ( MI -> getOperand ( OpNum - <NUM_LIT> ) . isImm ( ) && MI -> getOperand ( OpNum - <NUM_LIT> ) . getImm ( ) == <STR_LIT> ) && <STR_LIT> ) ; MI -> getOperand ( OpNum - <NUM_LIT> ) . setImm ( <STR_LIT> ) ; MI -> getOperand ( OpNum ) . setReg ( <NUM_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * LoLoop . VCTP ) ; LoLoop . VCTP -> eraseFromParent ( ) ; for ( auto * MI : LoLoop . VPTUsers ) { if ( MI -> getOpcode ( ) == <STR_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ;


<s> if ( SetFlags && ( RDA -> isRegUsedAfter ( MI , <STR_LIT> ) || ! RDA -> hasSameReachingDef ( MI , & MBB -> back ( ) , <STR_LIT> ) ) ) SetFlags = false ; MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . addDef ( <STR_LIT> ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <NUM_LIT> ) ; if ( SetFlags ) {


<s> for ( auto & I : MBB ) { if ( IsLoopStart ( I ) ) Starts . push_back ( & I ) ; else if ( I . getOpcode ( ) == <STR_LIT> ) Decs . push_back ( & I ) ; else if ( I . getOpcode ( ) == <STR_LIT> ) Ends . push_back ( & I ) ; } if ( Starts . empty ( ) && Decs . empty ( ) && Ends . empty ( ) ) continue ;


<s> if ( ! MO . isReg ( ) || MO . getReg ( ) != <STR_LIT> ) continue ; if ( MO . isUse ( ) ) VPTUsers . push_back ( MI ) ; if ( MO . isDef ( ) ) {


<s> bool <ISA_LIT>LowOverheadLoops :: runOnMachineFunction ( MachineFunction & mf ) { const <ISA_LIT>Subtarget & ST = mf . getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( ! ST . hasLOB ( ) ) return false ; MF = & mf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MF -> getName ( ) << <STR_LIT> ) ; MLI = & getAnalysis < MachineLoopInfo > ( ) ; RDA = & getAnalysis < ReachingDefAnalysis > ( ) ; MF -> getProperties ( ) . set ( MachineFunctionProperties :: Property :: TracksLiveness ) ; MRI = & MF -> getRegInfo ( ) ; TII = static_cast < const <ISA_LIT>BaseInstrInfo * > ( ST . getInstrInfo ( ) ) ; TRI = ST . getRegisterInfo ( ) ; BBUtils = std :: unique_ptr < <ISA_LIT>BasicBlockUtils > ( new <ISA_LIT>BasicBlockUtils ( * MF ) ) ; BBUtils -> computeAllBlockSizes ( ) ; BBUtils -> adjustBBOffsetsAfter ( & MF -> front ( ) ) ; bool Changed = false ; for ( auto ML : * MLI ) { if ( ML -> isOutermost ( ) ) Changed |= ProcessLoop ( ML ) ; }


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MF -> getName ( ) << <STR_LIT> ) ; MLI = & getAnalysis < MachineLoopInfo > ( ) ; RDA = & getAnalysis < ReachingDefAnalysis > ( ) ; MF -> getProperties ( ) . set ( MachineFunctionProperties :: Property :: TracksLiveness ) ; MRI = & MF -> getRegInfo ( ) ; TII = static_cast < const <ISA_LIT>BaseInstrInfo * > ( ST . getInstrInfo ( ) ) ; TRI = ST . getRegisterInfo ( ) ; BBUtils = std :: unique_ptr < <ISA_LIT>BasicBlockUtils > ( new <ISA_LIT>BasicBlockUtils ( * MF ) ) ; BBUtils -> computeAllBlockSizes ( ) ;

BBUtils -> adjustBBOffsetsAfter ( & MF -> front ( ) ) ;


<s> MIB . add ( End -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( End -> getOperand ( <NUM_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; LoLoop . ToRemove . insert ( LoLoop . Dec ) ; LoLoop . ToRemove . insert ( End ) ; return & * MIB ; } ; auto RemoveDeadBranch = [ ] ( MachineInstr * I ) { MachineBasicBlock * BB = I -> getParent ( ) ; MachineInstr * Terminator = & BB -> instr_back ( ) ; if ( Terminator -> isUnconditionalBranch ( ) && I != Terminator ) { MachineBasicBlock * Succ = Terminator -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( BB -> isLayoutSuccessor ( Succ ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Terminator ) ; Terminator -> eraseFromParent ( ) ; } } } ; if ( LoLoop . Revert ) { if ( LoLoop . Start -> getOpcode ( ) == <STR_LIT> ) RevertWhile ( LoLoop . Start ) ; else LoLoop . Start -> eraseFromParent ( ) ; bool FlagsAlreadySet = RevertLoopDec ( LoLoop . Dec ) ; RevertLoopEnd ( LoLoop . End , FlagsAlreadySet ) ; } else { LoLoop . Start = ExpandLoopStart ( LoLoop ) ; RemoveDeadBranch ( LoLoop . Start ) ;

LoLoop . End = ExpandLoopEnd ( LoLoop ) ;


<s> MachineInstr * Terminator = & BB -> instr_back ( ) ; if ( Terminator -> isUnconditionalBranch ( ) && I != Terminator ) { MachineBasicBlock * Succ = Terminator -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( BB -> isLayoutSuccessor ( Succ ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Terminator ) ; Terminator -> eraseFromParent ( ) ; } } } ; if ( LoLoop . Revert ) { if ( LoLoop . Start -> getOpcode ( ) == <STR_LIT> ) RevertWhile ( LoLoop . Start ) ; else LoLoop . Start -> eraseFromParent ( ) ; bool FlagsAlreadySet = RevertLoopDec ( LoLoop . Dec ) ; RevertLoopEnd ( LoLoop . End , FlagsAlreadySet ) ; } else { LoLoop . Start = ExpandLoopStart ( LoLoop ) ; RemoveDeadBranch ( LoLoop . Start ) ; LoLoop . End = ExpandLoopEnd ( LoLoop ) ; RemoveDeadBranch ( LoLoop . End ) ; if ( LoLoop . IsTailPredicationLegal ( ) ) ConvertVPTBlocks ( LoLoop ) ; for ( auto * I : LoLoop . ToRemove ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * I ) ; I -> eraseFromParent ( ) ; } for ( auto * I : LoLoop . BlockMasksToRecompute ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * I ) ; recomputeVPTBlockMask ( * I ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * I ) ; } } PostOrderLoopTraversal DFS ( LoLoop . ML , * MLI ) ; DFS . ProcessLoop ( ) ;

const SmallVectorImpl < MachineBasicBlock * > & PostOrder = DFS . getOrder ( ) ;


<s> LowOverheadLoop ( MachineLoop & ML , MachineLoopInfo & MLI , ReachingDefAnalysis & RDA , const TargetRegisterInfo & TRI ) : ML ( ML ) , MLI ( MLI ) , RDA ( RDA ) , TRI ( TRI ) { MF = ML . getHeader ( ) -> getParent ( ) ;


<s> if ( isLoopStart ( MI ) ) return & MI ; } if ( MBB -> pred_size ( ) == <NUM_LIT> ) return SearchForStart ( * MBB -> pred_begin ( ) ) ; return nullptr ; } ; LowOverheadLoop LoLoop ( * ML , * MLI , * RDA , * TRI ) ; if ( auto * Preheader = ML -> getLoopPreheader ( ) ) LoLoop . Start = SearchForStart ( Preheader ) ; else if ( auto * Preheader = MLI -> findLoopPreheader ( ML , true ) ) LoLoop . Start = SearchForStart ( Preheader ) ; else return false ; for ( auto * MBB : reverse ( ML -> getBlocks ( ) ) ) { for ( auto & MI : * MBB ) { if ( MI . isDebugValue ( ) ) continue ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . Dec = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . End = & MI ; else if ( isLoopStart ( MI ) ) LoLoop . Start = & MI ; else if ( MI . getDesc ( ) . isCall ( ) ) { LoLoop . Revert = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ;


<s> if ( MBB -> pred_size ( ) == <NUM_LIT> ) return SearchForStart ( * MBB -> pred_begin ( ) ) ; return nullptr ; } ; LowOverheadLoop LoLoop ( * ML , * MLI , * RDA , * TRI ) ; if ( auto * Preheader = ML -> getLoopPreheader ( ) ) LoLoop . Start = SearchForStart ( Preheader ) ; else if ( auto * Preheader = MLI -> findLoopPreheader ( ML , true ) ) LoLoop . Start = SearchForStart ( Preheader ) ; else return false ; for ( auto * MBB : reverse ( ML -> getBlocks ( ) ) ) { for ( auto & MI : * MBB ) { if ( MI . isDebugValue ( ) ) continue ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . Dec = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . End = & MI ; else if ( isLoopStart ( MI ) ) LoLoop . Start = & MI ; else if ( MI . getDesc ( ) . isCall ( ) ) { LoLoop . Revert = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } else { LoLoop . AnalyseMVEInst ( & MI ) ; } } } LLVM_DEBUG ( LoLoop . dump ( ) ) ; if ( ! LoLoop . FoundAllComponents ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } SmallPtrSet < MachineInstr * , <NUM_LIT> > Uses ; RDA -> getReachingLocalUses ( LoLoop . Dec , <STR_LIT> , Uses ) ; if ( Uses . size ( ) > <NUM_LIT> || ! Uses . count ( LoLoop . End ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LoLoop . Revert = true ; } LoLoop . CheckLegality ( BBUtils . get ( ) ) ; Expand ( LoLoop ) ;


<s> if ( isVectorPredicated ( & MI ) ) { if ( MI . mayLoad ( ) ) FalseLanesZero . insert ( & MI ) ; Predicated . insert ( & MI ) ; continue ; } if ( MI . getNumDefs ( ) == <NUM_LIT> ) continue ; if ( ! producesFalseLanesZero ( MI , QPRs , RDA , FalseLanesZero ) ) { if ( retainsPreviousHalfElement ( MI ) || isHorizontalReduction ( MI ) ) return false ; FalseLanesUnknown . insert ( & MI ) ; } else if ( ! isHorizontalReduction ( MI ) ) FalseLanesZero . insert ( & MI ) ; } auto HasPredicatedUsers = [ this ] ( MachineInstr * MI , const MachineOperand & MO , SmallPtrSetImpl < MachineInstr * > & Predicated ) { SmallPtrSet < MachineInstr * , <NUM_LIT> > Uses ; RDA . getGlobalUses ( MI , MO . getReg ( ) , Uses ) ; for ( auto * Use : Uses ) { if ( Use != MI && ! Predicated . count ( Use ) ) return false ; }


<s> uint64_t Flags = MCID . TSFlags ; if ( ( Flags & <STR_LIT> ) != <STR_LIT> ) continue ; if ( isVCTP ( & MI ) || isVPTOpcode ( MI . getOpcode ( ) ) ) continue ; if ( isVectorPredicated ( & MI ) ) { if ( MI . mayLoad ( ) ) FalseLanesZero . insert ( & MI ) ; Predicated . insert ( & MI ) ; continue ; } if ( MI . getNumDefs ( ) == <NUM_LIT> ) continue ; if ( ! producesFalseLanesZero ( MI , QPRs , RDA , FalseLanesZero ) ) { if ( retainsPreviousHalfElement ( MI ) || isHorizontalReduction ( MI ) ) return false ; FalseLanesUnknown . insert ( & MI ) ; } else if ( ! isHorizontalReduction ( MI ) ) FalseLanesZero . insert ( & MI ) ; } auto HasPredicatedUsers = [ this ] ( MachineInstr * MI , const MachineOperand & MO , SmallPtrSetImpl < MachineInstr * > & Predicated ) { SmallPtrSet < MachineInstr * , <NUM_LIT> > Uses ; RDA . getGlobalUses ( MI , MO . getReg ( ) , Uses ) ; for ( auto * Use : Uses ) { if ( Use != MI && ! Predicated . count ( Use ) ) return false ; } return true ; } ; for ( auto * MI : reverse ( FalseLanesUnknown ) ) { for ( auto & MO : MI -> operands ( ) ) { if ( ! isRegInClass ( MO , QPRs ) || ! MO . isDef ( ) ) continue ; if ( ! HasPredicatedUsers ( MI , MO , Predicated ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << TRI . getRegAsmName ( MO . getReg ( ) ) << <STR_LIT> << * MI ) ; return false ; } } Predicated . insert ( MI ) ; } SmallSet < Register , <NUM_LIT> > LiveOuts ; SmallVector < MachineBasicBlock * , <NUM_LIT> > ExitBlocks ; ML . getExitBlocks ( ExitBlocks ) ; for ( auto * MBB : ExitBlocks ) for ( const MachineBasicBlock :: RegisterMaskPair & RegMask : MBB -> liveins ( ) ) if ( QPRs -> contains ( RegMask . PhysReg ) ) LiveOuts . insert ( RegMask . PhysReg ) ; SmallPtrSet < MachineInstr * , <NUM_LIT> > LiveMIs ;

assert ( ML . getNumBlocks ( ) == <NUM_LIT> && <STR_LIT> ) ;


<s> } SmallVectorImpl < PredicatedMI > & Insts = Block . getInsts ( ) ; for ( auto & PredMI : Insts ) { if ( PredMI . Predicates . count ( VCTP ) || isVCTP ( PredMI . MI ) || VCMPOpcodeToVPT ( PredMI . MI -> getOpcode ( ) ) != <NUM_LIT> ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * PredMI . MI << <STR_LIT> ; for ( auto * MI : PredMI . Predicates ) dbgs ( ) << <STR_LIT> << * MI ) ; return false ; } } if ( ! ValidateLiveOuts ( ) ) return false ; Register NumElements = VCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( RDA . hasLocalDefBefore ( VCTP , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MachineBasicBlock * InsertBB = StartInsertPt -> getParent ( ) ; if ( ! RDA . isReachingDefLiveOut ( StartInsertPt , NumElements ) ) { if ( auto * ElemDef = RDA . getLocalLiveOutMIDef ( InsertBB , NumElements ) ) { if ( RDA . isSafeToMoveForwards ( ElemDef , StartInsertPt ) ) { ElemDef -> removeFromParent ( ) ; InsertBB -> insert ( MachineBasicBlock :: iterator ( StartInsertPt ) , ElemDef ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ElemDef ) ; } else if ( RDA . isSafeToMoveBackwards ( StartInsertPt , ElemDef ) ) { StartInsertPt -> removeFromParent ( ) ; InsertBB -> insertAfter ( MachineBasicBlock :: iterator ( ElemDef ) , StartInsertPt ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ElemDef ) ; } else {

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ;


<s> auto CannotProvideElements = [ this ] ( MachineBasicBlock * MBB , Register NumElements ) { if ( RDA . hasLocalDefBefore ( & MBB -> back ( ) , NumElements ) ) return true ; if ( MBB -> pred_size ( ) > <NUM_LIT> ) return true ; return false ; } ; MachineBasicBlock * MBB = MLI . findLoopPreheader ( & ML , true ) ; if ( ! MBB ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } while ( MBB != InsertBB ) { if ( CannotProvideElements ( MBB , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MBB = * MBB -> pred_begin ( ) ; } auto IsValidSub = [ ] ( MachineInstr * MI , int ExpectedVecWidth ) { return - getAddSubImmediate ( * MI ) == ExpectedVecWidth ; } ; MBB = VCTP -> getParent ( ) ; if ( auto * Def = RDA . getUniqueReachingMIDef ( & MBB -> back ( ) , NumElements ) ) { SmallPtrSet < MachineInstr * , <NUM_LIT> > ElementChain ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Ignore = { VCTP } ; unsigned ExpectedVectorWidth = getTailPredVectorWidth ( VCTP -> getOpcode ( ) ) ; Ignore . insert ( SecondaryVCTPs . begin ( ) , SecondaryVCTPs . end ( ) ) ; if ( RDA . isSafeToRemove ( Def , ElementChain , Ignore ) ) { bool FoundSub = false ; for ( auto * MI : ElementChain ) { if ( isMovRegOpcode ( MI -> getOpcode ( ) ) ) continue ; if ( isSubImmOpcode ( MI -> getOpcode ( ) ) ) { if ( FoundSub || ! IsValidSub ( MI , ExpectedVectorWidth ) ) return false ; FoundSub = true ; } else return false ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; for ( auto * MI : ElementChain ) dbgs ( ) << <STR_LIT> << * MI ) ; ToRemove . insert ( ElementChain . begin ( ) , ElementChain . end ( ) ) ; }


<s> } ; auto ExpandLoopEnd = [ this ] ( MachineLoop * ML , MachineInstr * Dec , MachineInstr * End ) { MachineBasicBlock * MBB = End -> getParent ( ) ; MachineInstrBuilder MIB = BuildMI ( * MBB , End , End -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . addDef ( <STR_LIT> ) ; MIB . add ( End -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( End -> getOperand ( <NUM_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; End -> eraseFromParent ( ) ; Dec -> eraseFromParent ( ) ; return & * MIB ; } ; auto RemoveDeadBranch = [ ] ( MachineInstr * I ) { MachineBasicBlock * BB = I -> getParent ( ) ; MachineInstr * Terminator = & BB -> instr_back ( ) ; if ( Terminator -> isUnconditionalBranch ( ) && I != Terminator ) { MachineBasicBlock * Succ = Terminator -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( BB -> isLayoutSuccessor ( Succ ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Terminator ) ; Terminator -> eraseFromParent ( ) ; } } } ; if ( Revert ) { if ( Start -> getOpcode ( ) == <STR_LIT> ) RevertWhile ( Start ) ; else Start -> eraseFromParent ( ) ; RevertLoopDec ( Dec ) ; RevertLoopEnd ( End ) ; } else { Start = ExpandLoopStart ( ML , Start ) ;


<s> MachineInstr * InsertPt = Start ; for ( auto & I : MRI -> def_instructions ( <STR_LIT> ) ) { if ( I . getParent ( ) != MBB ) continue ; if ( ! I . getOperand ( <NUM_LIT> ) . isImm ( ) || I . getOperand ( <NUM_LIT> ) . getImm ( ) != <STR_LIT> ) continue ; if ( ! I . getDesc ( ) . isMoveReg ( ) || ! I . getOperand ( <NUM_LIT> ) . isIdenticalTo ( Start -> getOperand ( <NUM_LIT> ) ) ) continue ; InsertPt = & I ; break ; } unsigned Opc = Start -> getOpcode ( ) == <STR_LIT> ? <STR_LIT> : <STR_LIT> ; MachineInstrBuilder MIB = BuildMI ( * MBB , InsertPt , InsertPt -> getDebugLoc ( ) , TII -> get ( Opc ) ) ; MIB . addDef ( <STR_LIT> ) ; MIB . add ( Start -> getOperand ( <NUM_LIT> ) ) ; if ( Opc == <STR_LIT> ) MIB . add ( Start -> getOperand ( <NUM_LIT> ) ) ; if ( InsertPt != Start ) InsertPt -> eraseFromParent ( ) ; Start -> eraseFromParent ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; return & * MIB ; } ; auto ExpandLoopEnd = [ this ] ( MachineLoop * ML , MachineInstr * Dec , MachineInstr * End ) { MachineBasicBlock * MBB = End -> getParent ( ) ; MachineInstrBuilder MIB = BuildMI ( * MBB , End , End -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . addDef ( <STR_LIT> ) ; MIB . add ( End -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( End -> getOperand ( <NUM_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; End -> eraseFromParent ( ) ; Dec -> eraseFromParent ( ) ; return & * MIB ; } ; auto RemoveDeadBranch = [ ] ( MachineInstr * I ) { MachineBasicBlock * BB = I -> getParent ( ) ; MachineInstr * Terminator = & BB -> instr_back ( ) ; if ( Terminator -> isUnconditionalBranch ( ) && I != Terminator ) { MachineBasicBlock * Succ = Terminator -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( BB -> isLayoutSuccessor ( Succ ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Terminator ) ;


<s> return MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ;


<s> return nullptr ; } ; MachineInstr * Start = nullptr ; MachineInstr * Dec = nullptr ; MachineInstr * End = nullptr ; bool Revert = false ; if ( auto * Preheader = ML -> getLoopPreheader ( ) ) { Start = SearchForStart ( Preheader ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; MachineBasicBlock * Pred = nullptr ; for ( auto * MBB : ML -> getHeader ( ) -> predecessors ( ) ) { if ( ! ML -> contains ( MBB ) ) { if ( Pred ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Start = nullptr ; break ; } Pred = MBB ; Start = SearchForStart ( MBB ) ; } } } for ( auto * MBB : reverse ( ML -> getBlocks ( ) ) ) { for ( auto & MI : * MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) Dec = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) End = & MI ; else if ( IsLoopStart ( MI ) ) Start = & MI ; else if ( MI . getDesc ( ) . isCall ( ) ) Revert = true ; if ( ! Dec ) continue ; if ( MI . mayLoad ( ) || MI . mayStore ( ) ) Revert = MI . getOperand ( <NUM_LIT> ) . isReg ( ) && MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ; } if ( Dec && End && Revert ) break ; } LLVM_DEBUG ( if ( Start ) dbgs ( ) << <STR_LIT> << * Start ; if ( Dec ) dbgs ( ) << <STR_LIT> << * Dec ; if ( End ) dbgs ( ) << <STR_LIT> << * End ; ) ; if ( ! Start && ! Dec && ! End ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return Changed ; } else if ( ! ( Start && Dec && End ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( ! End -> getOperand ( <NUM_LIT> ) . isMBB ( ) ) report_fatal_error ( <STR_LIT> ) ;

if ( End -> getOperand ( <NUM_LIT> ) . getMBB ( ) != ML -> getHeader ( ) ) {


<s> MachineBasicBlock * MBB = MI -> getParent ( ) ; MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . addDef ( <STR_LIT> ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <STR_LIT> ) ;


<s> MIB . addReg ( <STR_LIT> ) ; MIB . addImm ( <NUM_LIT> ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT> ) ; MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <STR_LIT> ) ;


<s> MIB . addReg ( <STR_LIT> ) ; MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT> ) ; MI -> eraseFromParent ( ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; bool Changed = false ; for ( auto & MBB : MF ) { SmallVector < MachineInstr * , <NUM_LIT> > Starts ; SmallVector < MachineInstr * , <NUM_LIT> > Decs ; SmallVector < MachineInstr * , <NUM_LIT> > Ends ; for ( auto & I : MBB ) { if ( IsLoopStart ( I ) ) Starts . push_back ( & I ) ;

else if ( I . getOpcode ( ) == <STR_LIT> ) Decs . push_back ( & I ) ;


<s> auto & MLI = getAnalysis < MachineLoopInfo > ( ) ; MRI = & MF . getRegInfo ( ) ; TII = static_cast < const <ISA_LIT>BaseInstrInfo * > ( MF . getSubtarget ( ) . getInstrInfo ( ) ) ; BBUtils = std :: unique_ptr < <ISA_LIT>BasicBlockUtils > ( new <ISA_LIT>BasicBlockUtils ( MF ) ) ; BBUtils -> computeAllBlockSizes ( ) ; BBUtils -> adjustBBOffsetsAfter ( & MF . front ( ) ) ; bool Changed = false ; for ( auto ML : MLI ) {

if ( ! ML -> getParentLoop ( ) ) Changed |= ProcessLoop ( ML ) ;


<s> BBUtils = std :: unique_ptr < <ISA_LIT>BasicBlockUtils > ( new <ISA_LIT>BasicBlockUtils ( MF ) ) ; BBUtils -> computeAllBlockSizes ( ) ; BBUtils -> adjustBBOffsetsAfter ( & MF . front ( ) ) ; bool Changed = false ; for ( auto ML : MLI ) {


<s> void addInst ( MachineInstr * MI , SetVector < MachineInstr * > & Preds ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; if ( ! Divergent && ! set_difference ( Preds , VPST -> Predicates ) . empty ( ) ) { Divergent = & Insts . back ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Divergent -> MI ) ; } Insts . emplace_back ( MI , Preds ) ;

assert ( Insts . size ( ) <= <NUM_LIT> && <STR_LIT> ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return ; } for ( auto & Block : VPTBlocks ) { if ( Block . IsPredicatedOn ( VCTP ) ) continue ; if ( ! Block . HasNonUniformPredicate ( ) || ! isVCTP ( Block . getDivergent ( ) -> MI ) ) { CannotTailPredicate = true ; return ; } SmallVectorImpl < PredicatedMI > & Insts = Block . getInsts ( ) ; for ( auto & PredMI : Insts ) { if ( PredMI . Predicates . count ( VCTP ) || isVCTP ( PredMI . MI ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * PredMI . MI << <STR_LIT> ; for ( auto * MI : PredMI . Predicates ) dbgs ( ) << <STR_LIT> << * MI ; ) ; CannotTailPredicate = true ; return ; } } Register NumElements = VCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( RDA -> getReachingDef ( VCTP , NumElements ) >= <NUM_LIT> ) { CannotTailPredicate = true ; return ; } MachineBasicBlock * InsertBB = InsertPt -> getParent ( ) ; if ( ! RDA -> isReachingDefLiveOut ( InsertPt , NumElements ) ) { if ( auto * ElemDef = RDA -> getLocalLiveOutMIDef ( InsertBB , NumElements ) ) {

if ( IsSafeToMove < MachineBasicBlock :: reverse_iterator > ( ElemDef , InsertPt , RDA ) ) {


<s> if ( ! RecordVPTBlocks ( MI ) ) { CannotTailPredicate = true ; return ; } const MCInstrDesc & MCID = MI -> getDesc ( ) ;


<s> for ( auto & Block : LoLoop . getVPTBlocks ( ) ) { SmallVectorImpl < PredicatedMI > & Insts = Block . getInsts ( ) ; if ( Block . HasNonUniformPredicate ( ) ) { PredicatedMI * Divergent = Block . getDivergent ( ) ; if ( isVCTP ( Divergent -> MI ) ) { uint64_t Size = get<ISA_LIT>VPTBlockMask ( Block . size ( ) - <NUM_LIT> ) ; Block . getVPST ( ) -> getOperand ( <NUM_LIT> ) . setImm ( Size ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } else if ( Block . IsOnlyPredicatedOn ( LoLoop . VCTP ) ) { for ( auto I = ++ MachineBasicBlock :: iterator ( Block . getVPST ( ) ) , E = ++ MachineBasicBlock :: iterator ( Divergent -> MI ) ; I != E ; ++ I ) RemovePredicate ( & * I ) ;


<s> MachineInstr * Terminator = & BB -> instr_back ( ) ; if ( Terminator -> isUnconditionalBranch ( ) && I != Terminator ) { MachineBasicBlock * Succ = Terminator -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( BB -> isLayoutSuccessor ( Succ ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Terminator ) ; Terminator -> eraseFromParent ( ) ; } } } ; if ( LoLoop . Revert ) { if ( LoLoop . Start -> getOpcode ( ) == <STR_LIT> ) RevertWhile ( LoLoop . Start ) ; else LoLoop . Start -> eraseFromParent ( ) ; bool FlagsAlreadySet = RevertLoopDec ( LoLoop . Dec , true ) ; RevertLoopEnd ( LoLoop . End , FlagsAlreadySet ) ; } else {


<s> bool IsOnlyPredicatedOn ( MachineInstr * MI ) const {

return IsPredicatedOn ( MI ) && VPST -> Predicates . size ( ) == <NUM_LIT> ;


<s> return VPST -> Predicates . count ( MI ) ;


<s> bool IsPredicatedOn ( MachineInstr * MI ) const {

return VPST -> Predicates . count ( MI ) ;


<s> else if ( ! RDA -> hasSameReachingDef ( From , To , MO . getReg ( ) ) ) return false ; } for ( auto I = ++ Iterator ( From ) , E = Iterator ( To ) ; I != E ; ++ I ) { for ( auto & MO : I -> operands ( ) ) if ( MO . isReg ( ) && MO . getReg ( ) && MO . isUse ( ) && Defs . count ( MO . getReg ( ) ) ) return false ; } return true ;


<s> for ( auto I = ++ Iterator ( From ) , E = Iterator ( To ) ; I != E ; ++ I ) { for ( auto & MO : I -> operands ( ) ) if ( MO . isReg ( ) && MO . getReg ( ) && MO . isUse ( ) && Defs . count ( MO . getReg ( ) ) ) return false ; } return true ;


Predicates . insert ( Preds . begin ( ) , Preds . end ( ) ) ;


<s> } ; LowOverheadLoop LoLoop ( ML ) ; if ( auto * Preheader = ML -> getLoopPreheader ( ) ) LoLoop . Start = SearchForStart ( Preheader ) ; else if ( auto * Preheader = MLI -> findLoopPreheader ( ML , true ) ) LoLoop . Start = SearchForStart ( Preheader ) ; else return false ; for ( auto * MBB : reverse ( ML -> getBlocks ( ) ) ) { for ( auto & MI : * MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . Dec = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . End = & MI ; else if ( isLoopStart ( MI ) ) LoLoop . Start = & MI ; else if ( MI . getDesc ( ) . isCall ( ) ) { LoLoop . Revert = true ;


<s> for ( auto I = ML -> begin ( ) , E = ML -> end ( ) ; I != E ; ++ I ) Changed |= ProcessLoop ( * I ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; if ( auto * Preheader = ML -> getLoopPreheader ( ) ) dbgs ( ) << <STR_LIT> << Preheader -> getName ( ) << <STR_LIT> ; else if ( auto * Preheader = MLI -> findLoopPreheader ( ML ) ) dbgs ( ) << <STR_LIT> << Preheader -> getName ( ) << <STR_LIT> ; for ( auto * MBB : ML -> getBlocks ( ) ) dbgs ( ) << <STR_LIT> << MBB -> getName ( ) << <STR_LIT> ; ) ; std :: function < MachineInstr * ( MachineBasicBlock * ) > SearchForStart = [ & SearchForStart ] ( MachineBasicBlock * MBB ) -> MachineInstr * { for ( auto & MI : * MBB ) { if ( isLoopStart ( MI ) ) return & MI ; } if ( MBB -> pred_size ( ) == <NUM_LIT> ) return SearchForStart ( * MBB -> pred_begin ( ) ) ; return nullptr ;


<s> bool LowOverheadLoop :: RecordVPTBlocks ( MachineInstr * MI ) { if ( isVCTP ( MI ) && VCTP ) return false ; if ( MI -> getOpcode ( ) == <STR_LIT> ) { VPTBlocks . emplace_back ( MI , CurrentPredicate ) ; CurrentBlock = & VPTBlocks . back ( ) ; return true ; } if ( isVCTP ( MI ) ) VCTP = MI ; unsigned VPROpNum = MI -> getNumOperands ( ) - <NUM_LIT> ; bool IsUse = false ; if ( MI -> getOperand ( VPROpNum ) . isReg ( ) && MI -> getOperand ( VPROpNum ) . getReg ( ) == <STR_LIT> && MI -> getOperand ( VPROpNum ) . isUse ( ) ) { if ( ! MI -> getOperand ( VPROpNum - <NUM_LIT> ) . isImm ( ) || MI -> getOperand ( VPROpNum - <NUM_LIT> ) . getImm ( ) != <STR_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; return false ; } CurrentBlock -> addInst ( MI , CurrentPredicate ) ; IsUse = true ; } bool IsDef = false ; for ( unsigned i = <NUM_LIT> ; i < MI -> getNumOperands ( ) - <NUM_LIT> ; ++ i ) { const MachineOperand & MO = MI -> getOperand ( i ) ;

if ( ! MO . isReg ( ) || MO . getReg ( ) != <STR_LIT> ) continue ;


<s> VPTBlocks . emplace_back ( MI , CurrentPredicate ) ; CurrentBlock = & VPTBlocks . back ( ) ; return true ; } if ( isVCTP ( MI ) ) VCTP = MI ; unsigned VPROpNum = MI -> getNumOperands ( ) - <NUM_LIT> ; bool IsUse = false ; if ( MI -> getOperand ( VPROpNum ) . isReg ( ) && MI -> getOperand ( VPROpNum ) . getReg ( ) == <STR_LIT> && MI -> getOperand ( VPROpNum ) . isUse ( ) ) { if ( ! MI -> getOperand ( VPROpNum - <NUM_LIT> ) . isImm ( ) || MI -> getOperand ( VPROpNum - <NUM_LIT> ) . getImm ( ) != <STR_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; return false ; } CurrentBlock -> addInst ( MI , CurrentPredicate ) ; IsUse = true ; } bool IsDef = false ; for ( unsigned i = <NUM_LIT> ; i < MI -> getNumOperands ( ) - <NUM_LIT> ; ++ i ) { const MachineOperand & MO = MI -> getOperand ( i ) ; if ( ! MO . isReg ( ) || MO . getReg ( ) != <STR_LIT> ) continue ; if ( MO . isDef ( ) ) { CurrentPredicate . insert ( MI ) ; IsDef = true ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; return false ; } } if ( IsDef && ! IsUse && VCTP && ! isVCTP ( MI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ;


<s> VPTBlock ( MachineInstr * MI , SetVector < MachineInstr * > & Preds ) {

VPST = std :: make_unique < PredicatedMI > ( MI , Preds ) ;


<s> VPTBlock ( MachineInstr * MI , SetVector < MachineInstr * > & Preds ) { VPST = std :: make_unique < PredicatedMI > ( MI , Preds ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } else if ( Block . IsOnlyPredicatedOn ( LoLoop . VCTP ) ) { for ( auto I = ++ MachineBasicBlock :: iterator ( Block . getVPST ( ) ) , E = ++ MachineBasicBlock :: iterator ( Divergent -> MI ) ; I != E ; ++ I ) RemovePredicate ( & * I ) ; unsigned Size = <NUM_LIT> ; auto E = MachineBasicBlock :: reverse_iterator ( Divergent -> MI ) ; auto I = MachineBasicBlock :: reverse_iterator ( Insts . back ( ) . MI ) ; MachineInstr * InsertAt = nullptr ; while ( I != E ) { InsertAt = & * I ; ++ Size ; ++ I ; } MachineInstrBuilder MIB = BuildMI ( * InsertAt -> getParent ( ) , InsertAt , InsertAt -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . addImm ( get<ISA_LIT>VPTBlockMask ( Size ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Block . getVPST ( ) ) ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ;


<s> LoLoop . ToRemove . insert ( LoLoop . Dec ) ; LoLoop . ToRemove . insert ( End ) ; return & * MIB ; } ; auto RemoveDeadBranch = [ ] ( MachineInstr * I ) { MachineBasicBlock * BB = I -> getParent ( ) ; MachineInstr * Terminator = & BB -> instr_back ( ) ; if ( Terminator -> isUnconditionalBranch ( ) && I != Terminator ) { MachineBasicBlock * Succ = Terminator -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( BB -> isLayoutSuccessor ( Succ ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Terminator ) ; Terminator -> eraseFromParent ( ) ; } } } ; if ( LoLoop . Revert ) { if ( LoLoop . Start -> getOpcode ( ) == <STR_LIT> ) RevertWhile ( LoLoop . Start ) ; else LoLoop . Start -> eraseFromParent ( ) ; bool FlagsAlreadySet = RevertLoopDec ( LoLoop . Dec ) ; RevertLoopEnd ( LoLoop . End , FlagsAlreadySet ) ; } else { LoLoop . Start = ExpandLoopStart ( LoLoop ) ; RemoveDeadBranch ( LoLoop . Start ) ; LoLoop . End = ExpandLoopEnd ( LoLoop ) ; RemoveDeadBranch ( LoLoop . End ) ; if ( LoLoop . IsTailPredicationLegal ( ) ) ConvertVPTBlocks ( LoLoop ) ; for ( auto * I : LoLoop . ToRemove ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * I ) ; I -> eraseFromParent ( ) ; }


<s> void <ISA_LIT>LowOverheadLoops :: Expand ( LowOverheadLoop & LoLoop ) { auto ExpandLoopEnd = [ this ] ( LowOverheadLoop & LoLoop ) { MachineInstr * End = LoLoop . End ; MachineBasicBlock * MBB = End -> getParent ( ) ; unsigned Opc = LoLoop . IsTailPredicationLegal ( ) ? <STR_LIT> : <STR_LIT> ; MachineInstrBuilder MIB = BuildMI ( * MBB , End , End -> getDebugLoc ( ) , TII -> get ( Opc ) ) ; MIB . addDef ( <STR_LIT> ) ; MIB . add ( End -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( End -> getOperand ( <NUM_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; LoLoop . ToRemove . insert ( LoLoop . Dec ) ; LoLoop . ToRemove . insert ( End ) ; return & * MIB ; } ; auto RemoveDeadBranch = [ ] ( MachineInstr * I ) { MachineBasicBlock * BB = I -> getParent ( ) ; MachineInstr * Terminator = & BB -> instr_back ( ) ; if ( Terminator -> isUnconditionalBranch ( ) && I != Terminator ) { MachineBasicBlock * Succ = Terminator -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( BB -> isLayoutSuccessor ( Succ ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Terminator ) ; Terminator -> eraseFromParent ( ) ; } } } ; if ( LoLoop . Revert ) { if ( LoLoop . Start -> getOpcode ( ) == <STR_LIT> ) RevertWhile ( LoLoop . Start ) ; else LoLoop . Start -> eraseFromParent ( ) ; bool FlagsAlreadySet = RevertLoopDec ( LoLoop . Dec ) ; RevertLoopEnd ( LoLoop . End , FlagsAlreadySet ) ; } else { LoLoop . Start = ExpandLoopStart ( LoLoop ) ; RemoveDeadBranch ( LoLoop . Start ) ; LoLoop . End = ExpandLoopEnd ( LoLoop ) ; RemoveDeadBranch ( LoLoop . End ) ; if ( LoLoop . IsTailPredicationLegal ( ) ) ConvertVPTBlocks ( LoLoop ) ; for ( auto * I : LoLoop . ToRemove ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * I ) ; I -> eraseFromParent ( ) ; } } PostOrderLoopTraversal DFS ( LoLoop . ML , * MLI ) ; DFS . ProcessLoop ( ) ; const SmallVectorImpl < MachineBasicBlock * > & PostOrder = DFS . getOrder ( ) ;


<s> if ( ! isRegInClass ( MO , QPRs ) || ! MO . isUse ( ) || MO . getReg ( ) != DefReg ) continue ; if ( auto * OpDef = RDA . getMIOperand ( & MI , MO ) ) { if ( KnownFalseZeros . count ( OpDef ) ) { KnownFalseZeros . insert ( & MI ) ; break ; } } } if ( ! KnownFalseZeros . count ( & MI ) ) UnknownFalseLanes . insert ( & MI ) ; } auto HasKnownUsers = [ this ] ( MachineInstr * MI , const MachineOperand & MO , SmallPtrSetImpl < MachineInstr * > & Knowns ) { SmallPtrSet < MachineInstr * , <NUM_LIT> > Uses ; RDA . getGlobalUses ( MI , MO . getReg ( ) , Uses ) ; for ( auto * Use : Uses ) { if ( Use != MI && ! Knowns . count ( Use ) ) return false ; } return true ; } ; for ( auto * MI : reverse ( UnknownFalseLanes ) ) { for ( auto & MO : MI -> operands ( ) ) { if ( ! isRegInClass ( MO , QPRs ) || ! MO . isDef ( ) ) continue ; if ( ! HasKnownUsers ( MI , MO , KnownFalseZeros ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << TRI . getRegAsmName ( MO . getReg ( ) ) << <STR_LIT> << * MI ) ; return false ; } }


<s> } if ( MI . getNumDefs ( ) == <NUM_LIT> ) continue ; assert ( ( MI . getNumDefs ( ) == <NUM_LIT> && MI . defs ( ) . begin ( ) -> isReg ( ) ) && <STR_LIT> ) ; Register DefReg = MI . defs ( ) . begin ( ) -> getReg ( ) ; for ( auto & MO : MI . operands ( ) ) { if ( ! isRegInClass ( MO , QPRs ) || ! MO . isUse ( ) || MO . getReg ( ) != DefReg ) continue ; if ( auto * OpDef = RDA . getMIOperand ( & MI , MO ) ) { if ( KnownFalseZeros . count ( OpDef ) ) { KnownFalseZeros . insert ( & MI ) ; break ; } } } if ( ! KnownFalseZeros . count ( & MI ) ) UnknownFalseLanes . insert ( & MI ) ; } auto HasKnownUsers = [ this ] ( MachineInstr * MI , const MachineOperand & MO , SmallPtrSetImpl < MachineInstr * > & Knowns ) { SmallPtrSet < MachineInstr * , <NUM_LIT> > Uses ; RDA . getGlobalUses ( MI , MO . getReg ( ) , Uses ) ; for ( auto * Use : Uses ) { if ( Use != MI && ! Knowns . count ( Use ) ) return false ; } return true ; } ; for ( auto * MI : reverse ( UnknownFalseLanes ) ) { for ( auto & MO : MI -> operands ( ) ) { if ( ! isRegInClass ( MO , QPRs ) || ! MO . isDef ( ) ) continue ;

if ( ! HasKnownUsers ( MI , MO , KnownFalseZeros ) ) {


<s> if ( ! MO . isReg ( ) || MO . getReg ( ) != <STR_LIT> ) continue ; if ( MO . isDef ( ) ) { CurrentPredicate . insert ( MI ) ; IsDef = true ; } else if ( <STR_LIT> ( MCID . OpInfo [ i ] . OperandType ) ) { CurrentBlock -> addInst ( MI , CurrentPredicate ) ; IsUse = true ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; return false ; } } if ( IsDef && ! IsUse && VCTP && ! isVCTP ( MI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; return false ; } uint64_t Flags = MCID . TSFlags ; if ( ( Flags & <STR_LIT> ) != <STR_LIT> ) return true ; if ( ( Flags & <STR_LIT> ) == <NUM_LIT> && ! IsUse ) {


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Block . getDivergent ( ) -> MI ) ; return false ; } SmallVectorImpl < PredicatedMI > & Insts = Block . getInsts ( ) ; for ( auto & PredMI : Insts ) { if ( PredMI . Predicates . count ( VCTP ) || isVCTP ( PredMI . MI ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * PredMI . MI << <STR_LIT> ; for ( auto * MI : PredMI . Predicates ) dbgs ( ) << <STR_LIT> << * MI ) ; return false ; } } if ( ! ValidateLiveOuts ( ) ) return false ; Register NumElements = VCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( RDA . hasLocalDefBefore ( VCTP , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MachineBasicBlock * InsertBB = StartInsertPt -> getParent ( ) ; if ( ! RDA . isReachingDefLiveOut ( StartInsertPt , NumElements ) ) { if ( auto * ElemDef = RDA . getLocalLiveOutMIDef ( InsertBB , NumElements ) ) { if ( RDA . isSafeToMoveForwards ( ElemDef , StartInsertPt ) ) { ElemDef -> removeFromParent ( ) ; InsertBB -> insert ( MachineBasicBlock :: iterator ( StartInsertPt ) , ElemDef ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ElemDef ) ; } else if ( RDA . isSafeToMoveBackwards ( StartInsertPt , ElemDef ) ) { StartInsertPt -> removeFromParent ( ) ; InsertBB -> insertAfter ( MachineBasicBlock :: iterator ( ElemDef ) , StartInsertPt ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ElemDef ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return false ; } } } auto CannotProvideElements = [ this ] ( MachineBasicBlock * MBB , Register NumElements ) { if ( RDA . hasLocalDefBefore ( & MBB -> back ( ) , NumElements ) ) return true ; if ( MBB -> pred_size ( ) > <NUM_LIT> ) return true ; return false ; } ; MachineBasicBlock * MBB = MLI . findLoopPreheader ( & ML , true ) ; if ( ! MBB ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } while ( MBB != InsertBB ) { if ( CannotProvideElements ( MBB , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MBB = * MBB -> pred_begin ( ) ; } auto IsValidSub = [ ] ( MachineInstr * MI , unsigned ExpectedVecWidth ) { unsigned ImmOpIdx = <NUM_LIT> ; switch ( MI -> getOpcode ( ) ) {


<s> void <ISA_LIT>LowOverheadLoops :: Expand ( LowOverheadLoop & LoLoop ) { auto ExpandLoopEnd = [ this ] ( LowOverheadLoop & LoLoop ) { MachineInstr * End = LoLoop . End ; MachineBasicBlock * MBB = End -> getParent ( ) ; unsigned Opc = LoLoop . IsTailPredicationLegal ( ) ? <STR_LIT> : <STR_LIT> ; MachineInstrBuilder MIB = BuildMI ( * MBB , End , End -> getDebugLoc ( ) , TII -> get ( Opc ) ) ; MIB . addDef ( <STR_LIT> ) ; MIB . add ( End -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( End -> getOperand ( <NUM_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; LoLoop . Dec -> eraseFromParent ( ) ; End -> eraseFromParent ( ) ; return & * MIB ; } ; auto RemoveDeadBranch = [ ] ( MachineInstr * I ) { MachineBasicBlock * BB = I -> getParent ( ) ; MachineInstr * Terminator = & BB -> instr_back ( ) ; if ( Terminator -> isUnconditionalBranch ( ) && I != Terminator ) { MachineBasicBlock * Succ = Terminator -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( BB -> isLayoutSuccessor ( Succ ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Terminator ) ; Terminator -> eraseFromParent ( ) ; } } } ; if ( LoLoop . Revert ) { if ( LoLoop . Start -> getOpcode ( ) == <STR_LIT> ) RevertWhile ( LoLoop . Start ) ; else LoLoop . Start -> eraseFromParent ( ) ; bool FlagsAlreadySet = RevertLoopDec ( LoLoop . Dec ) ; RevertLoopEnd ( LoLoop . End , FlagsAlreadySet ) ; } else { LoLoop . Start = ExpandLoopStart ( LoLoop ) ; RemoveDeadBranch ( LoLoop . Start ) ; LoLoop . End = ExpandLoopEnd ( LoLoop ) ; RemoveDeadBranch ( LoLoop . End ) ; if ( LoLoop . IsTailPredicationLegal ( ) ) ConvertVPTBlocks ( LoLoop ) ; for ( auto * I : LoLoop . ToRemove ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * I ) ; I -> eraseFromParent ( ) ; } } PostOrderLoopTraversal DFS ( LoLoop . ML , * MLI ) ; DFS . ProcessLoop ( ) ; const SmallVectorImpl < MachineBasicBlock * > & PostOrder = DFS . getOrder ( ) ; for ( auto * MBB : PostOrder ) { recomputeLiveIns ( * MBB ) ;


<s> ITBlocks [ IT ] . insert ( & MI ) ; Predicates [ & MI ] = IT ; } } SmallPtrSet < MachineInstr * , <NUM_LIT> > ModifiedITs ; SmallPtrSet < MachineInstr * , <NUM_LIT> > DeadITs ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Killed ; RDA -> collectLocalKilledOperands ( Def , Killed ) ; for ( auto * MI : Killed ) { if ( ! Predicates . count ( MI ) ) continue ; MachineInstr * IT = Predicates [ MI ] ; auto & CurrentBlock = ITBlocks [ IT ] ; CurrentBlock . erase ( MI ) ; ModifiedITs . insert ( IT ) ; if ( CurrentBlock . empty ( ) ) { DeadITs . insert ( IT ) ; ModifiedITs . erase ( IT ) ; } } if ( ModifiedITs . empty ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; for ( auto * MI : Killed ) dbgs ( ) << <STR_LIT> << * MI ;

for ( auto * MI : DeadITs ) dbgs ( ) << <STR_LIT> << * MI ) ;


<s> MachineBasicBlock * MBB = MI -> getParent ( ) ; MachineInstr * Last = & MBB -> back ( ) ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Ignore ; if ( Last -> getOpcode ( ) == <STR_LIT> ) Ignore . insert ( Last ) ; bool SetFlags = RDA -> isSafeToDefRegAt ( MI , <STR_LIT> , Ignore ) ; MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . addDef ( <STR_LIT> ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <NUM_LIT> ) ;


<s> MachineInstr * Last = & MBB -> back ( ) ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Ignore ; if ( Last -> getOpcode ( ) == <STR_LIT> ) Ignore . insert ( Last ) ; bool SetFlags = RDA -> isSafeToDefRegAt ( MI , <STR_LIT> , Ignore ) ; MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . addDef ( <STR_LIT> ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <NUM_LIT> ) ; if ( SetFlags ) { MIB . addReg ( <STR_LIT> ) ; MIB -> getOperand ( <NUM_LIT> ) . setIsDef ( true ) ; } else MIB . addReg ( <NUM_LIT> ) ;


<s> unsigned Off = LoLoop . Dec == LoLoop . End ? <NUM_LIT> : <NUM_LIT> ; MIB . add ( End -> getOperand ( Off + <NUM_LIT> ) ) ; MIB . add ( End -> getOperand ( Off + <NUM_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; LoLoop . ToRemove . insert ( LoLoop . Dec ) ; LoLoop . ToRemove . insert ( End ) ; return & * MIB ; } ; auto RemoveDeadBranch = [ ] ( MachineInstr * I ) { MachineBasicBlock * BB = I -> getParent ( ) ; MachineInstr * Terminator = & BB -> instr_back ( ) ; if ( Terminator -> isUnconditionalBranch ( ) && I != Terminator ) { MachineBasicBlock * Succ = Terminator -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( BB -> isLayoutSuccessor ( Succ ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Terminator ) ; Terminator -> eraseFromParent ( ) ; } } } ; if ( LoLoop . Revert ) {

if ( isWhileLoopStart ( * LoLoop . Start ) ) RevertWhile ( LoLoop . Start ) ;


<s> SmallVector < MachineInstr * , <NUM_LIT> > Decs ; SmallVector < MachineInstr * , <NUM_LIT> > Ends ; SmallVector < MachineInstr * , <NUM_LIT> > EndDecs ; for ( auto & I : MBB ) { if ( isLoopStart ( I ) ) Starts . push_back ( & I ) ; else if ( I . getOpcode ( ) == <STR_LIT> ) Decs . push_back ( & I ) ; else if ( I . getOpcode ( ) == <STR_LIT> ) Ends . push_back ( & I ) ; else if ( I . getOpcode ( ) == <STR_LIT> ) EndDecs . push_back ( & I ) ; } if ( Starts . empty ( ) && Decs . empty ( ) && Ends . empty ( ) && EndDecs . empty ( ) ) continue ; Changed = true ; for ( auto * Start : Starts ) { if ( isWhileLoopStart ( * Start ) ) RevertWhile ( Start ) ; else RevertDo ( Start ) ; } for ( auto * Dec : Decs ) RevertLoopDec ( Dec ) ; for ( auto * End : Ends ) RevertLoopEnd ( End ) ; for ( auto * End : EndDecs ) RevertLoopEndDec ( End ) ;


<s> SmallVector < MachineInstr * , <NUM_LIT> > EndDecs ; for ( auto & I : MBB ) { if ( isLoopStart ( I ) ) Starts . push_back ( & I ) ; else if ( I . getOpcode ( ) == <STR_LIT> ) Decs . push_back ( & I ) ; else if ( I . getOpcode ( ) == <STR_LIT> ) Ends . push_back ( & I ) ; else if ( I . getOpcode ( ) == <STR_LIT> ) EndDecs . push_back ( & I ) ; } if ( Starts . empty ( ) && Decs . empty ( ) && Ends . empty ( ) && EndDecs . empty ( ) ) continue ; Changed = true ; for ( auto * Start : Starts ) { if ( isWhileLoopStart ( * Start ) ) RevertWhile ( Start ) ; else RevertDo ( Start ) ; } for ( auto * Dec : Decs ) RevertLoopDec ( Dec ) ; for ( auto * End : Ends ) RevertLoopEnd ( End ) ; for ( auto * End : EndDecs ) RevertLoopEndDec ( End ) ;


<s> const TargetRegisterClass * QPRs = TRI . getRegClass ( <STR_LIT> ) ; SetVector < MachineInstr * > FalseLanesUnknown ; SmallPtrSet < MachineInstr * , <NUM_LIT> > FalseLanesZero ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Predicated ; MachineBasicBlock * Header = ML . getHeader ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; for ( auto & MI : * Header ) { if ( ! shouldInspect ( MI ) ) continue ;

if ( isVCTP ( & MI ) || isVPTOpcode ( MI . getOpcode ( ) ) ) continue ;


<s> bool LowOverheadLoop :: ValidateMVEInst ( MachineInstr * MI ) { if ( CannotTailPredicate ) return false ; if ( ! shouldInspect ( * MI ) ) return true ; if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { return false ; } if ( isVCTP ( MI ) && ! AddVCTP ( MI ) ) return false ; const MCInstrDesc & MCID = MI -> getDesc ( ) ; bool IsUse = false ; unsigned LastOpIdx = MI -> getNumOperands ( ) - <NUM_LIT> ; for ( auto & Op : enumerate ( reverse ( MCID . operands ( ) ) ) ) { const MachineOperand & MO = MI -> getOperand ( LastOpIdx - Op . index ( ) ) ; if ( ! MO . isReg ( ) || ! MO . isUse ( ) || MO . getReg ( ) != <STR_LIT> ) continue ;

if ( <STR_LIT> ( Op . value ( ) . OperandType ) ) {


<s> } auto IsValidSub = [ ] ( MachineInstr * MI , int ExpectedVecWidth ) { return - getAddSubImmediate ( * MI ) == ExpectedVecWidth ; } ; MachineBasicBlock * MBB = VCTP -> getParent ( ) ; if ( auto * Def = RDA . getUniqueReachingMIDef ( & MBB -> back ( ) , VCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) . asMCReg ( ) ) ) { SmallPtrSet < MachineInstr * , <NUM_LIT> > ElementChain ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Ignore ; unsigned ExpectedVectorWidth = getTailPredVectorWidth ( VCTP -> getOpcode ( ) ) ; Ignore . insert ( VCTPs . begin ( ) , VCTPs . end ( ) ) ; if ( TryRemove ( Def , RDA , ElementChain , Ignore ) ) { bool FoundSub = false ; for ( auto * MI : ElementChain ) { if ( isMovRegOpcode ( MI -> getOpcode ( ) ) ) continue ; if ( isSubImmOpcode ( MI -> getOpcode ( ) ) ) { if ( FoundSub || ! IsValidSub ( MI , ExpectedVectorWidth ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> << * MI ) ; return false ; } FoundSub = true ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> << * MI ) ; return false ; } } ToRemove . insert ( ElementChain . begin ( ) , ElementChain . end ( ) ) ; } } if ( ( Start -> getOpcode ( ) == <STR_LIT> || Start -> getOpcode ( ) == <STR_LIT> ) && Preheader && ! Preheader -> empty ( ) && ! RDA . hasLocalDefBefore ( VCTP , VCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { if ( auto * Def = RDA . getUniqueReachingMIDef ( & Preheader -> back ( ) , VCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) . asMCReg ( ) ) ) { SmallPtrSet < MachineInstr * , <NUM_LIT> > Ignore ; Ignore . insert ( VCTPs . begin ( ) , VCTPs . end ( ) ) ; TryRemove ( Def , RDA , ToRemove , Ignore ) ; }


<s> MI -> getOperand ( PIdx ) . setImm ( <STR_LIT> ) ; MI -> getOperand ( PIdx + <NUM_LIT> ) . setReg ( <NUM_LIT> ) ; } else llvm_unreachable ( <STR_LIT> ) ; } ; for ( auto & Block : LoLoop . getVPTBlocks ( ) ) { SmallVectorImpl < PredicatedMI > & Insts = Block . getInsts ( ) ; if ( Block . HasNonUniformPredicate ( ) ) { PredicatedMI * Divergent = Block . getDivergent ( ) ; if ( isVCTP ( Divergent -> MI ) ) { LoLoop . BlockMasksToRecompute . insert ( Block . getPredicateThen ( ) ) ; } else if ( Block . isVPST ( ) && Block . IsOnlyPredicatedOn ( LoLoop . VCTP ) ) { for ( auto I = ++ MachineBasicBlock :: iterator ( Block . getPredicateThen ( ) ) , E = ++ MachineBasicBlock :: iterator ( Divergent -> MI ) ; I != E ; ++ I ) RemovePredicate ( & * I ) ; MachineInstr * VCMP = VCMPOpcodeToVPT ( Divergent -> MI -> getOpcode ( ) ) != <NUM_LIT> ? Divergent -> MI : nullptr ; unsigned Size = <NUM_LIT> ; auto E = MachineBasicBlock :: reverse_iterator ( Divergent -> MI ) ; auto I = MachineBasicBlock :: reverse_iterator ( Insts . back ( ) . MI ) ; MachineInstr * InsertAt = nullptr ; while ( I != E ) { InsertAt = & * I ; ++ Size ; ++ I ; } MachineInstrBuilder MIB ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Block . getPredicateThen ( ) ) ; if ( VCMP ) { MIB = BuildMI ( * InsertAt -> getParent ( ) , InsertAt , InsertAt -> getDebugLoc ( ) , TII -> get ( VCMPOpcodeToVPT ( VCMP -> getOpcode ( ) ) ) ) ; MIB . addImm ( <STR_LIT> ) ; MIB . add ( VCMP -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( VCMP -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( VCMP -> getOperand ( <NUM_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; LoLoop . ToRemove . insert ( VCMP ) ; } else { MIB = BuildMI ( * InsertAt -> getParent ( ) , InsertAt , InsertAt -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . addImm ( <NUM_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; } LoLoop . ToRemove . insert ( Block . getPredicateThen ( ) ) ; LoLoop . BlockMasksToRecompute . insert ( MIB . getInstr ( ) ) ; } else if ( Block . isVPT ( ) ) { bool RemovedVCTP = false ; for ( PredicatedMI & Elt : Block . getInsts ( ) ) { MachineInstr * MI = Elt . MI ;


<s> MIB . addDef ( <STR_LIT> ) ; MIB . add ( Count ) ; if ( ! IsDo ) MIB . add ( Start -> getOperand ( <NUM_LIT> ) ) ; if ( InsertPt != Start ) LoLoop . ToRemove . insert ( InsertPt ) ; LoLoop . ToRemove . insert ( Start ) ;


<s> MF = ML . getHeader ( ) -> getParent ( ) ; if ( auto * MBB = ML . getLoopPreheader ( ) ) Preheader = MBB ; else if ( auto * MBB = MLI . findLoopPreheader ( & ML , true ) ) Preheader = MBB ;


<s> if ( auto * MBB = ML . getLoopPreheader ( ) ) Preheader = MBB ; else if ( auto * MBB = MLI . findLoopPreheader ( & ML , true ) ) Preheader = MBB ;


<s> const TargetRegisterClass * QPRs = TRI . getRegClass ( <STR_LIT> ) ; SetVector < MachineInstr * > FalseLanesUnknown ; SmallPtrSet < MachineInstr * , <NUM_LIT> > FalseLanesZero ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Predicated ; MachineBasicBlock * Header = ML . getHeader ( ) ; for ( auto & MI : * Header ) { const MCInstrDesc & MCID = MI . getDesc ( ) ; uint64_t Flags = MCID . TSFlags ; if ( ( Flags & <STR_LIT> ) != <STR_LIT> ) continue ; if ( isVCTP ( & MI ) || isVPTOpcode ( MI . getOpcode ( ) ) ) continue ; bool isPredicated = isVectorPredicated ( & MI ) ; bool retainsOrReduces = retainsPreviousHalfElement ( MI ) || isHorizontalReduction ( MI ) ; if ( isPredicated ) Predicated . insert ( & MI ) ; if ( producesFalseLanesZero ( MI , QPRs , RDA , FalseLanesZero ) ) FalseLanesZero . insert ( & MI ) ; else if ( MI . getNumDefs ( ) == <NUM_LIT> ) continue ; else if ( ! isPredicated && retainsOrReduces ) return false ; else if ( ! isPredicated ) FalseLanesUnknown . insert ( & MI ) ; } auto HasPredicatedUsers = [ this ] ( MachineInstr * MI , const MachineOperand & MO , SmallPtrSetImpl < MachineInstr * > & Predicated ) { SmallPtrSet < MachineInstr * , <NUM_LIT> > Uses ; RDA . getGlobalUses ( MI , MO . getReg ( ) , Uses ) ; for ( auto * Use : Uses ) { if ( Use != MI && ! Predicated . count ( Use ) ) return false ; } return true ; } ; SmallPtrSet < MachineInstr * , <NUM_LIT> > NonPredicated ; for ( auto * MI : reverse ( FalseLanesUnknown ) ) { for ( auto & MO : MI -> operands ( ) ) { if ( ! isRegInClass ( MO , QPRs ) || ! MO . isDef ( ) ) continue ; if ( ! HasPredicatedUsers ( MI , MO , Predicated ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << TRI . getRegAsmName ( MO . getReg ( ) ) << <STR_LIT> << * MI ) ; NonPredicated . insert ( MI ) ; break ; } } if ( ! NonPredicated . contains ( MI ) ) Predicated . insert ( MI ) ; } SmallPtrSet < MachineInstr * , <NUM_LIT> > LiveOutMIs ; SmallVector < MachineBasicBlock * , <NUM_LIT> > ExitBlocks ; ML . getExitBlocks ( ExitBlocks ) ; assert ( ML . getNumBlocks ( ) == <NUM_LIT> && <STR_LIT> ) ; assert ( ExitBlocks . size ( ) == <NUM_LIT> && <STR_LIT> ) ;

MachineBasicBlock * ExitBB = ExitBlocks . front ( ) ;


<s> CurrentPredicate . insert ( MI ) ; IsDef = true ; } else if ( <STR_LIT> ( MCID . OpInfo [ i ] . OperandType ) ) { CurrentBlock -> addInst ( MI , CurrentPredicate ) ; IsUse = true ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; return false ; } } if ( IsDef ) { if ( ! isVectorPredicated ( MI ) ) CurrentPredicate . clear ( ) ; CurrentPredicate . insert ( MI ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; } if ( IsDef && ! IsUse && VCTP && ! isVCTP ( MI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; return false ; } if ( ( Flags & <STR_LIT> ) == <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; return IsUse ; } return ! IsUse && MI -> mayStore ( ) ? false : true ;


<s> } if ( isVCTP ( MI ) ) { if ( VCTP ) { if ( ! VCTP -> getOperand ( <NUM_LIT> ) . isIdenticalTo ( MI -> getOperand ( <NUM_LIT> ) ) || ! RDA . hasSameReachingDef ( VCTP , MI , MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; SecondaryVCTPs . insert ( MI ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; VCTP = MI ; } } else if ( isVPTOpcode ( MI -> getOpcode ( ) ) ) { if ( MI -> getOpcode ( ) != <STR_LIT> ) { assert ( MI -> findRegisterDefOperandIdx ( <STR_LIT> ) != - <NUM_LIT> && <STR_LIT> ) ; CurrentPredicate . clear ( ) ; CurrentPredicate . insert ( MI ) ; } VPTBlocks . emplace_back ( MI , CurrentPredicate ) ; CurrentBlock = & VPTBlocks . back ( ) ; return true ; } bool IsUse = false ; bool IsDef = false ; for ( int i = MI -> getNumOperands ( ) - <NUM_LIT> ; i >= <NUM_LIT> ; -- i ) { const MachineOperand & MO = MI -> getOperand ( i ) ; if ( ! MO . isReg ( ) || MO . getReg ( ) != <STR_LIT> ) continue ; if ( MO . isDef ( ) ) { CurrentPredicate . insert ( MI ) ; IsDef = true ; } else if ( <STR_LIT> ( MCID . OpInfo [ i ] . OperandType ) ) { CurrentBlock -> addInst ( MI , CurrentPredicate ) ; IsUse = true ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; return false ; } } if ( IsDef ) { if ( ! isVectorPredicated ( MI ) ) CurrentPredicate . clear ( ) ; CurrentPredicate . insert ( MI ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; } if ( IsDef && ! IsUse && VCTP && ! isVCTP ( MI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; return false ; } if ( ( Flags & <STR_LIT> ) == <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; return IsUse ; } return ! IsUse && MI -> mayStore ( ) ? false : true ;


<s> MachineInstr * InsertAt = nullptr ; while ( I != E ) { InsertAt = & * I ; ++ Size ; ++ I ; } MachineInstrBuilder MIB = BuildMI ( * InsertAt -> getParent ( ) , InsertAt , InsertAt -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . addImm ( ( uint64_t ) get<ISA_LIT>VPTBlockMask ( Size ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Block . getVPST ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; LoLoop . ToRemove . insert ( Block . getVPST ( ) ) ; } } else if ( Block . IsOnlyPredicatedOn ( LoLoop . VCTP ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Block . getVPST ( ) ) ; LoLoop . ToRemove . insert ( Block . getVPST ( ) ) ; for ( auto & PredMI : Insts ) RemovePredicate ( PredMI . MI ) ;


<s> uint64_t Flags = MCID . TSFlags ; if ( ( Flags & <STR_LIT> ) != <STR_LIT> ) continue ; if ( isVCTP ( & MI ) || MI . getOpcode ( ) == <STR_LIT> ) continue ; if ( isVectorPredicated ( & MI ) ) { if ( MI . mayLoad ( ) ) FalseLanesZero . insert ( & MI ) ; Predicated . insert ( & MI ) ; continue ; } if ( MI . getNumDefs ( ) == <NUM_LIT> ) continue ; if ( ! producesFalseLanesZero ( MI , QPRs , RDA , FalseLanesZero ) ) { if ( retainsPreviousHalfElement ( MI ) || isHorizontalReduction ( MI ) ) return false ; FalseLanesUnknown . insert ( & MI ) ; } else if ( ! isHorizontalReduction ( MI ) ) FalseLanesZero . insert ( & MI ) ; } auto HasPredicatedUsers = [ this ] ( MachineInstr * MI , const MachineOperand & MO , SmallPtrSetImpl < MachineInstr * > & Predicated ) { SmallPtrSet < MachineInstr * , <NUM_LIT> > Uses ; RDA . getGlobalUses ( MI , MO . getReg ( ) , Uses ) ; for ( auto * Use : Uses ) { if ( Use != MI && ! Predicated . count ( Use ) ) return false ; } return true ; } ; for ( auto * MI : reverse ( FalseLanesUnknown ) ) { for ( auto & MO : MI -> operands ( ) ) { if ( ! isRegInClass ( MO , QPRs ) || ! MO . isDef ( ) ) continue ; if ( ! HasPredicatedUsers ( MI , MO , Predicated ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << TRI . getRegAsmName ( MO . getReg ( ) ) << <STR_LIT> << * MI ) ; return false ; } } Predicated . insert ( MI ) ; } SmallSet < Register , <NUM_LIT> > LiveOuts ; SmallVector < MachineBasicBlock * , <NUM_LIT> > ExitBlocks ; ML . getExitBlocks ( ExitBlocks ) ; for ( auto * MBB : ExitBlocks ) for ( const MachineBasicBlock :: RegisterMaskPair & RegMask : MBB -> liveins ( ) ) if ( QPRs -> contains ( RegMask . PhysReg ) ) LiveOuts . insert ( RegMask . PhysReg ) ; SmallPtrSet < MachineInstr * , <NUM_LIT> > LiveMIs ;

assert ( ML . getNumBlocks ( ) == <NUM_LIT> && <STR_LIT> ) ;


<s> for ( auto & PredMI : Insts ) { if ( PredMI . Predicates . count ( VCTP ) || isVCTP ( PredMI . MI ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * PredMI . MI << <STR_LIT> ; for ( auto * MI : PredMI . Predicates ) dbgs ( ) << <STR_LIT> << * MI ) ; return false ; } } if ( ! ValidateLiveOuts ( ) ) return false ; Register NumElements = VCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( RDA . hasLocalDefBefore ( VCTP , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MachineBasicBlock * InsertBB = StartInsertPt -> getParent ( ) ; if ( ! RDA . isReachingDefLiveOut ( StartInsertPt , NumElements ) ) { if ( auto * ElemDef = RDA . getLocalLiveOutMIDef ( InsertBB , NumElements ) ) { if ( RDA . isSafeToMoveForwards ( ElemDef , StartInsertPt ) ) { ElemDef -> removeFromParent ( ) ; InsertBB -> insert ( MachineBasicBlock :: iterator ( StartInsertPt ) , ElemDef ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ElemDef ) ; } else if ( RDA . isSafeToMoveBackwards ( StartInsertPt , ElemDef ) ) { StartInsertPt -> removeFromParent ( ) ; InsertBB -> insertAfter ( MachineBasicBlock :: iterator ( ElemDef ) , StartInsertPt ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ElemDef ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return false ; } } } auto CannotProvideElements = [ this ] ( MachineBasicBlock * MBB , Register NumElements ) { if ( RDA . hasLocalDefBefore ( & MBB -> back ( ) , NumElements ) ) return true ; if ( MBB -> pred_size ( ) > <NUM_LIT> ) return true ; return false ; } ; MachineBasicBlock * MBB = MLI . findLoopPreheader ( & ML , true ) ; if ( ! MBB ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } while ( MBB != InsertBB ) {

if ( CannotProvideElements ( MBB , NumElements ) ) {


<s> InsertPt = Revert ? nullptr : IsSafeToDefineLR ( RDA ) ; if ( ! InsertPt ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Revert = true ; return ; } else LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * InsertPt ) ; if ( ! IsTailPredicationLegal ( ) ) { LLVM_DEBUG ( if ( ! VCTP ) dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ) ; return ; } assert ( ML -> getBlocks ( ) . size ( ) == <NUM_LIT> && <STR_LIT> ) ; CannotTailPredicate = ! ValidateTailPredicate ( InsertPt , RDA , MLI ) ;

LLVM_DEBUG ( if ( CannotTailPredicate ) dbgs ( ) << <STR_LIT> ) ;


<s> PredicatedMI * Divergent = Block . getDivergent ( ) ; if ( isVCTP ( Divergent -> MI ) ) { uint64_t Size = get<ISA_LIT>VPTBlockMask ( Block . size ( ) - <NUM_LIT> ) ; Block . getVPST ( ) -> getOperand ( <NUM_LIT> ) . setImm ( Size ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } else if ( Block . IsOnlyPredicatedOn ( LoLoop . VCTP ) ) { for ( auto I = ++ MachineBasicBlock :: iterator ( Block . getVPST ( ) ) , E = ++ MachineBasicBlock :: iterator ( Divergent -> MI ) ; I != E ; ++ I ) RemovePredicate ( & * I ) ; unsigned Size = <NUM_LIT> ; auto E = MachineBasicBlock :: reverse_iterator ( Divergent -> MI ) ; auto I = MachineBasicBlock :: reverse_iterator ( Insts . back ( ) . MI ) ; MachineInstr * InsertAt = nullptr ; while ( I != E ) {


<s> for ( auto * MBB : ML -> getBlocks ( ) ) dbgs ( ) << <STR_LIT> << MBB -> getName ( ) << <STR_LIT> ; ) ; std :: function < MachineInstr * ( MachineBasicBlock * ) > SearchForStart = [ & SearchForStart ] ( MachineBasicBlock * MBB ) -> MachineInstr * { for ( auto & MI : * MBB ) { if ( isLoopStart ( MI ) ) return & MI ; } if ( MBB -> pred_size ( ) == <NUM_LIT> ) return SearchForStart ( * MBB -> pred_begin ( ) ) ; return nullptr ; } ; LowOverheadLoop LoLoop ( ML ) ; if ( auto * Preheader = ML -> getLoopPreheader ( ) ) LoLoop . Start = SearchForStart ( Preheader ) ; else if ( auto * Preheader = MLI -> findLoopPreheader ( ML , true ) ) LoLoop . Start = SearchForStart ( Preheader ) ; else return false ; for ( auto * MBB : reverse ( ML -> getBlocks ( ) ) ) { for ( auto & MI : * MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . Dec = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . End = & MI ; else if ( isLoopStart ( MI ) ) LoLoop . Start = & MI ; else if ( MI . getDesc ( ) . isCall ( ) ) { LoLoop . Revert = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } else { LoLoop . AnalyseMVEInst ( & MI ) ; } if ( ! LoLoop . Dec || LoLoop . End || LoLoop . Revert ) continue ; for ( auto & MO : MI . operands ( ) ) { if ( MI . getOpcode ( ) != <STR_LIT> && MO . isReg ( ) && MO . getReg ( ) == <STR_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ;


<s> CurrentBlock = & VPTBlocks . back ( ) ; return true ; } else if ( isVCTP ( MI ) ) VCTP = MI ; else if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) return false ; bool IsUse = false ; bool IsDef = false ; const MCInstrDesc & MCID = MI -> getDesc ( ) ; for ( int i = MI -> getNumOperands ( ) - <NUM_LIT> ; i >= <NUM_LIT> ; -- i ) { const MachineOperand & MO = MI -> getOperand ( i ) ; if ( ! MO . isReg ( ) || MO . getReg ( ) != <STR_LIT> ) continue ; if ( MO . isDef ( ) ) { CurrentPredicate . insert ( MI ) ; IsDef = true ; } else if ( <STR_LIT> ( MCID . OpInfo [ i ] . OperandType ) ) {

CurrentBlock -> addInst ( MI , CurrentPredicate ) ;


<s> if ( CannotTailPredicate ) return false ; if ( isVCTP ( MI ) && VCTP ) return false ; if ( MI -> getOpcode ( ) == <STR_LIT> ) { VPTBlocks . emplace_back ( MI , CurrentPredicate ) ; CurrentBlock = & VPTBlocks . back ( ) ; return true ; } else if ( isVCTP ( MI ) ) VCTP = MI ; else if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) return false ; bool IsUse = false ; bool IsDef = false ; const MCInstrDesc & MCID = MI -> getDesc ( ) ; for ( int i = MI -> getNumOperands ( ) - <NUM_LIT> ; i >= <NUM_LIT> ; -- i ) { const MachineOperand & MO = MI -> getOperand ( i ) ; if ( ! MO . isReg ( ) || MO . getReg ( ) != <STR_LIT> ) continue ; if ( MO . isDef ( ) ) { CurrentPredicate . insert ( MI ) ; IsDef = true ; } else if ( <STR_LIT> ( MCID . OpInfo [ i ] . OperandType ) ) { CurrentBlock -> addInst ( MI , CurrentPredicate ) ; IsUse = true ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; return false ; } } if ( IsDef && ! IsUse && VCTP && ! isVCTP ( MI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ;