<s> const Function * Func = MF -> getFunction ( ) ; unsigned ReqAlign = STI -> hasV6Ops ( ) ? TD -> getABITypeAlignment ( Type :: getInt64Ty ( Func -> getContext ( ) ) ) : <NUM_LIT> ; if ( Align < ReqAlign ) return false ; int OffImm = getMemoryOpOffset ( Op0 ) ; if ( isT2 ) { int Limit = ( <NUM_LIT> << <NUM_LIT> ) * Scale ; if ( OffImm >= Limit || ( OffImm <= - Limit ) || ( OffImm & ( Scale - <NUM_LIT> ) ) ) return false ; Offset = OffImm ; } else { <STR_LIT> AddSub = <STR_LIT> ; if ( OffImm < <NUM_LIT> ) { AddSub = <STR_LIT> ; OffImm = - OffImm ; } int Limit = ( <NUM_LIT> << <NUM_LIT> ) * Scale ; if ( OffImm >= Limit || ( OffImm & ( Scale - <NUM_LIT> ) ) ) return false ; Offset = <STR_LIT> ( AddSub , OffImm ) ; } FirstReg = Op0 -> getOperand ( <NUM_LIT> ) . getReg ( ) ; SecondReg = Op1 -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( FirstReg == SecondReg ) return false ; BaseReg = Op0 -> getOperand ( <NUM_LIT> ) . getReg ( ) ; Pred = getInstrPredicate ( * Op0 , PredReg ) ; dl = Op0 -> getDebugLoc ( ) ;


<s> if ( ! Errata602117 && ! NonConsecutiveRegs ) return false ; bool isT2 = Opcode == <STR_LIT> || Opcode == <STR_LIT> ; bool isLd = Opcode == <STR_LIT> || Opcode == <STR_LIT> ; bool EvenDeadKill = isLd ? MI -> getOperand ( <NUM_LIT> ) . isDead ( ) : MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; bool EvenUndef = MI -> getOperand ( <NUM_LIT> ) . isUndef ( ) ; bool OddDeadKill = isLd ? MI -> getOperand ( <NUM_LIT> ) . isDead ( ) : MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; bool OddUndef = MI -> getOperand ( <NUM_LIT> ) . isUndef ( ) ; bool BaseKill = BaseOp . isKill ( ) ; bool BaseUndef = BaseOp . isUndef ( ) ; bool OffKill = isT2 ? false : MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; bool OffUndef = isT2 ? false : MI -> getOperand ( <NUM_LIT> ) . isUndef ( ) ; int OffImm = getMemoryOpOffset ( MI ) ; unsigned PredReg = <NUM_LIT> ; <STR_LIT> Pred = getInstrPredicate ( * MI , PredReg ) ; if ( OddRegNum > EvenRegNum && OffImm == <NUM_LIT> ) { unsigned NewOpc = ( isLd ) ? ( isT2 ? <STR_LIT> : <STR_LIT> ) : ( isT2 ? <STR_LIT> : <STR_LIT> ) ; if ( isLd ) { BuildMI ( MBB , MBBI , MBBI -> getDebugLoc ( ) , TII -> get ( NewOpc ) ) . addReg ( BaseReg , getKillRegState ( BaseKill ) ) . addImm ( Pred ) . addReg ( PredReg ) . addReg ( EvenReg , getDefRegState ( isLd ) | getDeadRegState ( EvenDeadKill ) ) . addReg ( OddReg , getDefRegState ( isLd ) | getDeadRegState ( OddDeadKill ) ) ; ++ NumLDRD2LDM ; } else { BuildMI ( MBB , MBBI , MBBI -> getDebugLoc ( ) , TII -> get ( NewOpc ) ) . addReg ( BaseReg , getKillRegState ( BaseKill ) ) . addImm ( Pred ) . addReg ( PredReg ) . addReg ( EvenReg , getKillRegState ( EvenDeadKill ) | getUndefRegState ( EvenUndef ) ) . addReg ( OddReg , getKillRegState ( OddDeadKill ) | getUndefRegState ( OddUndef ) ) ; ++ NumSTRD2STM ; } } else { unsigned NewOpc = ( isLd ) ? ( isT2 ? ( OffImm < <NUM_LIT> ? <STR_LIT> : <STR_LIT> ) : <STR_LIT> ) : ( isT2 ? ( OffImm < <NUM_LIT> ? <STR_LIT> : <STR_LIT> ) : <STR_LIT> ) ; unsigned NewOpc2 = ( isLd ) ? ( isT2 ? ( OffImm + <NUM_LIT> < <NUM_LIT> ? <STR_LIT> : <STR_LIT> ) : <STR_LIT> ) : ( isT2 ? ( OffImm + <NUM_LIT> < <NUM_LIT> ? <STR_LIT> : <STR_LIT> ) : <STR_LIT> ) ; DebugLoc dl = MBBI -> getDebugLoc ( ) ; if ( isLd && ( BaseKill || OffKill ) && ( TRI -> regsOverlap ( EvenReg , BaseReg ) ) ) {

assert ( ! TRI -> regsOverlap ( OddReg , BaseReg ) ) ;


<s> bool PartOfLSDouble = CanMergeToLSDouble && Count <= <NUM_LIT> ; if ( ! PartOfLSMulti && ! PartOfLSDouble ) break ; CanMergeToLSMulti &= PartOfLSMulti ; CanMergeToLSDouble &= PartOfLSDouble ; unsigned Position = MemOps [ I ] . Position ; if ( Position < MemOps [ Latest ] . Position ) Latest = I ; else if ( Position > MemOps [ Earliest ] . Position ) Earliest = I ; Offset += Size ; PRegNum = RegNum ; } MergeCandidate * Candidate = new ( Allocator . Allocate ( ) ) MergeCandidate ; for ( unsigned C = SIndex , CE = SIndex + Count ; C < CE ; ++ C ) Candidate -> Instrs . push_back ( MemOps [ C ] . MI ) ; Candidate -> LatestMIIdx = Latest - SIndex ; Candidate -> EarliestMIIdx = Earliest - SIndex ; Candidate -> InsertPos = MemOps [ Latest ] . Position ; if ( Count == <NUM_LIT> ) CanMergeToLSMulti = CanMergeToLSDouble = false ; Candidate -> CanMergeToLSMulti = CanMergeToLSMulti ; Candidate -> CanMergeToLSDouble = CanMergeToLSDouble ; Candidates . push_back ( Candidate ) ; SIndex += Count ; } while ( SIndex < EIndex ) ;


<s> if ( STI -> isCortexM3 ( ) && isi32Load ( Opcode ) && PReg == getLoadStoreBaseOp ( * MI ) . getReg ( ) ) CanMergeToLSDouble = false ; bool CanMergeToLSMulti = true ; if ( STI -> isSwift ( ) && ! isNotVFP && ( PRegNum % <NUM_LIT> ) == <NUM_LIT> ) CanMergeToLSMulti = false ; if ( PReg == <STR_LIT> || PReg == <STR_LIT> ) CanMergeToLSMulti = CanMergeToLSDouble = false ; if ( AssumeMisalignedLoadStores && ! mayCombineMisaligned ( * STI , * MI ) ) CanMergeToLSMulti = CanMergeToLSDouble = false ; for ( unsigned I = SIndex + <NUM_LIT> ; I < EIndex ; ++ I , ++ Count ) { int NewOffset = MemOps [ I ] . Offset ; if ( NewOffset != Offset + ( int ) Size ) break ; const MachineOperand & MO = getLoadStoreRegOp ( * MemOps [ I ] . MI ) ; unsigned Reg = MO . getReg ( ) ; if ( Reg == <STR_LIT> || Reg == <STR_LIT> ) break ; unsigned RegNum = MO . isUndef ( ) ? UINT_MAX : TRI -> getEncodingValue ( Reg ) ; bool PartOfLSMulti = CanMergeToLSMulti ; if ( PartOfLSMulti ) { if ( RegNum <= PRegNum ) PartOfLSMulti = false ; else if ( ! isNotVFP && RegNum != PRegNum + <NUM_LIT> ) PartOfLSMulti = false ; } bool PartOfLSDouble = CanMergeToLSDouble && Count <= <NUM_LIT> ; if ( ! PartOfLSMulti && ! PartOfLSDouble ) break ; CanMergeToLSMulti &= PartOfLSMulti ; CanMergeToLSDouble &= PartOfLSDouble ; unsigned Position = MemOps [ I ] . Position ; if ( Position < MemOps [ Latest ] . Position ) Latest = I ; else if ( Position > MemOps [ Earliest ] . Position ) Earliest = I ; Offset += Size ; PRegNum = RegNum ; } MergeCandidate * Candidate = new ( Allocator . Allocate ( ) ) MergeCandidate ; for ( unsigned C = SIndex , CE = SIndex + Count ; C < CE ; ++ C ) Candidate -> Instrs . push_back ( MemOps [ C ] . MI ) ; Candidate -> LatestMIIdx = Latest - SIndex ; Candidate -> EarliestMIIdx = Earliest - SIndex ; Candidate -> InsertPos = MemOps [ Latest ] . Position ;

if ( Count == <NUM_LIT> ) CanMergeToLSMulti = CanMergeToLSDouble = false ;


<s> MemOpQueue MemOps ; unsigned CurrBase = <NUM_LIT> ; unsigned CurrOpc = ~ <NUM_LIT> ; <STR_LIT> CurrPred = <STR_LIT> ; unsigned Position = <NUM_LIT> ; assert ( Candidates . size ( ) == <NUM_LIT> ) ; assert ( MergeBaseCandidates . size ( ) == <NUM_LIT> ) ; LiveRegsValid = false ; for ( MachineBasicBlock :: iterator I = MBB . end ( ) , MBBI ; I != MBB . begin ( ) ; I = MBBI ) { MBBI = std :: prev ( I ) ; if ( FixInvalidRegPairOp ( MBB , MBBI ) ) continue ; ++ Position ; if ( isMemoryOp ( * MBBI ) ) { unsigned Opcode = MBBI -> getOpcode ( ) ; const MachineOperand & MO = MBBI -> getOperand ( <NUM_LIT> ) ; unsigned Reg = MO . getReg ( ) ; unsigned Base = getLoadStoreBaseOp ( * MBBI ) . getReg ( ) ; unsigned PredReg = <NUM_LIT> ; <STR_LIT> Pred = getInstrPredicate ( * MBBI , PredReg ) ; int Offset = getMemoryOpOffset ( MBBI ) ; if ( CurrBase == <NUM_LIT> ) { CurrBase = Base ; CurrOpc = Opcode ; CurrPred = Pred ; MemOps . push_back ( MemOpQueueEntry ( MBBI , Offset , Position ) ) ; continue ; } if ( CurrOpc == Opcode && CurrBase == Base && CurrPred == Pred ) { bool Overlap = false ; if ( isLoadSingle ( Opcode ) ) { Overlap = ( Base == Reg ) ; if ( ! Overlap ) { for ( const MemOpQueueEntry & E : MemOps ) { if ( TRI -> regsOverlap ( Reg , E . MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { Overlap = true ; break ; } } } } if ( ! Overlap ) { if ( Offset > MemOps . back ( ) . Offset ) { MemOps . push_back ( MemOpQueueEntry ( MBBI , Offset , Position ) ) ; continue ; } else { MemOpQueue :: iterator MI , ME ; for ( MI = MemOps . begin ( ) , ME = MemOps . end ( ) ; MI != ME ; ++ MI ) { if ( Offset < MI -> Offset ) { break ; } if ( Offset == MI -> Offset ) { MI = ME ; break ; } } if ( MI != MemOps . end ( ) ) { MemOps . insert ( MI , MemOpQueueEntry ( MBBI , Offset , Position ) ) ; continue ; } } } } MBBI = I ; -- Position ; } else if ( MBBI -> isDebugValue ( ) ) { continue ; } else if ( MBBI -> getOpcode ( ) == <STR_LIT> || MBBI -> getOpcode ( ) == <STR_LIT> ) { MergeBaseCandidates . push_back ( MBBI ) ; } if ( MemOps . size ( ) > <NUM_LIT> ) { FormCandidates ( MemOps ) ; CurrBase = <NUM_LIT> ; CurrOpc = ~ <NUM_LIT> ;


<s> } else { MergeInstr = findIncDecAfter ( MBBI , Base , Pred , PredReg , Offset ) ; if ( ( ( Mode != <STR_LIT> && Mode != <STR_LIT> ) || Offset != Bytes ) && ( ( Mode != <STR_LIT> && Mode != <STR_LIT> ) || Offset != - Bytes ) ) return false ; } MBB . erase ( MergeInstr ) ; unsigned NewOpc = getUpdatingLSMultipleOpcode ( Opcode , Mode ) ; MachineInstrBuilder MIB = BuildMI ( MBB , MBBI , DL , TII -> get ( NewOpc ) ) . addReg ( Base , getDefRegState ( true ) ) . addReg ( Base , getKillRegState ( BaseKill ) ) . addImm ( Pred ) . addReg ( PredReg ) ; for ( unsigned OpNum = <NUM_LIT> , e = MI -> getNumOperands ( ) ; OpNum != e ; ++ OpNum ) MIB . addOperand ( MI -> getOperand ( OpNum ) ) ;

MIB -> setMemRefs ( MI -> memoperands_begin ( ) , MI -> memoperands_end ( ) ) ;


<s> if ( MBBI != MBB . begin ( ) && ( MBBI -> getOpcode ( ) == <STR_LIT> || MBBI -> getOpcode ( ) == <STR_LIT> || MBBI -> getOpcode ( ) == <STR_LIT> ) ) { MachineBasicBlock :: iterator PrevI = std :: prev ( MBBI ) ; while ( PrevI -> isDebugValue ( ) && PrevI != MBB . begin ( ) ) -- PrevI ; MachineInstr * PrevMI = PrevI ; unsigned Opcode = PrevMI -> getOpcode ( ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { MachineOperand & MO = PrevMI -> getOperand ( PrevMI -> getNumOperands ( ) - <NUM_LIT> ) ; if ( MO . getReg ( ) != <STR_LIT> ) return false ; unsigned NewOpc = ( isThumb2 ? <STR_LIT> : <STR_LIT> ) ; assert ( ( ( isThumb2 && Opcode == <STR_LIT> ) || Opcode == <STR_LIT> ) && <STR_LIT> ) ; PrevMI -> setDesc ( TII -> get ( NewOpc ) ) ; MO . setReg ( <STR_LIT> ) ; PrevMI -> copyImplicitOps ( * MBB . getParent ( ) , * MBBI ) ; MBB . erase ( MBBI ) ; return true ;


<s> while ( MBBI != E ) { for ( ; MBBI != E ; ++ MBBI ) { MachineInstr * MI = MBBI ; if ( MI -> isCall ( ) || MI -> isTerminator ( ) ) { ++ MBBI ; break ; } if ( ! MI -> isDebugValue ( ) ) MI2LocMap [ MI ] = ++ Loc ; if ( ! isMemoryOp ( MI ) ) continue ; unsigned PredReg = <NUM_LIT> ; if ( llvm :: getInstrPredicate ( MI , PredReg ) != <STR_LIT> ) continue ; int Opc = MI -> getOpcode ( ) ; bool isLd = isi32Load ( Opc ) || Opc == <STR_LIT> || Opc == <STR_LIT> ; unsigned Base = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; int Offset = getMemoryOpOffset ( MI ) ; bool StopHere = false ; if ( isLd ) { DenseMap < unsigned , SmallVector < MachineInstr * , <NUM_LIT> > > :: iterator BI = Base2LdsMap . find ( Base ) ; if ( BI != Base2LdsMap . end ( ) ) { for ( unsigned i = <NUM_LIT> , e = BI -> second . size ( ) ; i != e ; ++ i ) { if ( Offset == getMemoryOpOffset ( BI -> second [ i ] ) ) { StopHere = true ; break ; } } if ( ! StopHere ) BI -> second . push_back ( MI ) ; } else { SmallVector < MachineInstr * , <NUM_LIT> > MIs ; MIs . push_back ( MI ) ; Base2LdsMap [ Base ] = MIs ; LdBases . push_back ( Base ) ; } } else { DenseMap < unsigned , SmallVector < MachineInstr * , <NUM_LIT> > > :: iterator BI = Base2StsMap . find ( Base ) ; if ( BI != Base2StsMap . end ( ) ) { for ( unsigned i = <NUM_LIT> , e = BI -> second . size ( ) ; i != e ; ++ i ) { if ( Offset == getMemoryOpOffset ( BI -> second [ i ] ) ) { StopHere = true ; break ; } } if ( ! StopHere ) BI -> second . push_back ( MI ) ; } else { SmallVector < MachineInstr * , <NUM_LIT> > MIs ; MIs . push_back ( MI ) ; Base2StsMap [ Base ] = MIs ; StBases . push_back ( Base ) ;


<s> NewOpc = getPostIndexedLoadStoreOpcode ( Opcode ) ; } else if ( isMatchingIncrement ( NextMBBI , Base , Bytes , Pred , PredReg ) ) { DoMerge = true ; NewOpc = getPostIndexedLoadStoreOpcode ( Opcode ) ; } if ( DoMerge ) { if ( NextMBBI == I ) { Advance = true ; ++ I ; } MBB . erase ( NextMBBI ) ; } } if ( ! DoMerge ) return false ; bool isDPR = NewOpc == <STR_LIT> || NewOpc == <STR_LIT> ; unsigned Offset = isAM2 ? <STR_LIT> ( AddSub , Bytes , <STR_LIT> ) : <STR_LIT> ( ( AddSub == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> , true , isDPR ? <NUM_LIT> : <NUM_LIT> ) ; if ( isLd ) { if ( isAM2 ) BuildMI ( MBB , MBBI , dl , TII -> get ( NewOpc ) , MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) . addReg ( Base , true ) . addReg ( Base ) . addReg ( <NUM_LIT> ) . addImm ( Offset ) . addImm ( Pred ) . addReg ( PredReg ) ; else BuildMI ( MBB , MBBI , dl , TII -> get ( NewOpc ) ) . addReg ( Base , false , false , BaseKill ) . addImm ( Offset ) . addImm ( Pred ) . addReg ( PredReg ) . addReg ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) , true ) ; } else { MachineOperand & MO = MI -> getOperand ( <NUM_LIT> ) ; if ( isAM2 ) BuildMI ( MBB , MBBI , dl , TII -> get ( NewOpc ) , Base ) . addReg ( MO . getReg ( ) , false , false , MO . isKill ( ) ) . addReg ( Base ) . addReg ( <NUM_LIT> ) . addImm ( Offset ) . addImm ( Pred ) . addReg ( PredReg ) ; else BuildMI ( MBB , MBBI , dl , TII -> get ( NewOpc ) ) . addReg ( Base ) . addImm ( Offset ) . addImm ( Pred ) . addReg ( PredReg ) . addReg ( MO . getReg ( ) , false , false , MO . isKill ( ) ) ;


<s> else if ( isAM4 && Offset == - <NUM_LIT> * ( int ) NumRegs ) Mode = <STR_LIT> ; else if ( Offset != <NUM_LIT> ) { if ( NumRegs <= <NUM_LIT> ) return false ; unsigned NewBase ; if ( Opcode == <STR_LIT> ) NewBase = Regs [ NumRegs - <NUM_LIT> ] . first ; else { NewBase = Scratch ; if ( NewBase == <NUM_LIT> ) return false ; } int BaseOpc = <STR_LIT> ; if ( Offset < <NUM_LIT> ) { BaseOpc = <STR_LIT> ; Offset = - Offset ; } int ImmedOffset = <STR_LIT> ( Offset ) ; if ( ImmedOffset == - <NUM_LIT> ) return false ; BuildMI ( MBB , MBBI , dl , TII -> get ( BaseOpc ) , NewBase ) . addReg ( Base , false , false , BaseKill ) . addImm ( ImmedOffset ) . addImm ( Pred ) . addReg ( PredReg ) . addReg ( <NUM_LIT> ) ; Base = NewBase ; BaseKill = true ; } bool isDPR = Opcode == <STR_LIT> || Opcode == <STR_LIT> ; bool isDef = Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ; Opcode = getLoadStoreMultipleOpcode ( Opcode ) ; MachineInstrBuilder MIB = ( isAM4 ) ? BuildMI ( MBB , MBBI , dl , TII -> get ( Opcode ) ) . addReg ( Base , false , false , BaseKill ) . addImm ( <STR_LIT> ( Mode ) ) . addImm ( Pred ) . addReg ( PredReg ) : BuildMI ( MBB , MBBI , dl , TII -> get ( Opcode ) ) . addReg ( Base , false , false , BaseKill ) . addImm ( <STR_LIT> ( Mode , false , isDPR ? NumRegs << <NUM_LIT> : NumRegs ) ) . addImm ( Pred ) . addReg ( PredReg ) ; for ( unsigned i = <NUM_LIT> ; i != NumRegs ; ++ i ) MIB = MIB . addReg ( Regs [ i ] . first , isDef , false , Regs [ i ] . second ) ;


<s> } MBB . erase ( NextMBBI ) ; } } if ( ! DoMerge ) return false ; bool isDPR = NewOpc == <STR_LIT> || NewOpc == <STR_LIT> ; unsigned Offset = <NUM_LIT> ; if ( isAM5 ) Offset = <STR_LIT> ( ( AddSub == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> , true , ( isDPR ? <NUM_LIT> : <NUM_LIT> ) ) ; else if ( isAM2 ) Offset = <STR_LIT> ( AddSub , Bytes , <STR_LIT> ) ; else Offset = AddSub == <STR_LIT> ? - Bytes : Bytes ; if ( isLd ) { if ( isAM5 ) BuildMI ( MBB , MBBI , dl , TII -> get ( NewOpc ) ) . addReg ( Base , getKillRegState ( BaseKill ) ) . addImm ( Offset ) . addImm ( Pred ) . addReg ( PredReg ) . addReg ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) , RegState :: Define ) ; else if ( isAM2 ) BuildMI ( MBB , MBBI , dl , TII -> get ( NewOpc ) , MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) . addReg ( Base , RegState :: Define ) . addReg ( Base ) . addReg ( <NUM_LIT> ) . addImm ( Offset ) . addImm ( Pred ) . addReg ( PredReg ) ; else BuildMI ( MBB , MBBI , dl , TII -> get ( NewOpc ) , MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) . addReg ( Base , RegState :: Define ) . addReg ( Base ) . addImm ( Offset ) . addImm ( Pred ) . addReg ( PredReg ) ; } else { MachineOperand & MO = MI -> getOperand ( <NUM_LIT> ) ; if ( isAM5 ) BuildMI ( MBB , MBBI , dl , TII -> get ( NewOpc ) ) . addReg ( Base ) . addImm ( Offset ) . addImm ( Pred ) . addReg ( PredReg ) . addReg ( MO . getReg ( ) , getKillRegState ( MO . isKill ( ) ) ) ;

else if ( isAM2 ) BuildMI ( MBB , MBBI , dl , TII -> get ( NewOpc ) , Base ) . addReg ( MO . getReg ( ) , getKillRegState ( MO . isKill ( ) ) ) . addReg ( Base ) . addReg ( <NUM_LIT> ) . addImm ( Offset ) . addImm ( Pred ) . addReg ( PredReg ) ;


<s> MI -> getOperand ( <NUM_LIT> ) . setImm ( <STR_LIT> ( <STR_LIT> , true ) ) ; MBB . erase ( PrevMBBI ) ; return true ; } } if ( MBBI != MBB . end ( ) ) { MachineBasicBlock :: iterator NextMBBI = llvm :: next ( MBBI ) ; if ( ( Mode == <STR_LIT> || Mode == <STR_LIT> ) && isMatchingIncrement ( NextMBBI , Base , Bytes , <NUM_LIT> , Pred , PredReg ) ) { MI -> getOperand ( <NUM_LIT> ) . setImm ( <STR_LIT> ( Mode , true ) ) ; if ( NextMBBI == I ) { Advance = true ; ++ I ; } MBB . erase ( NextMBBI ) ; return true ; } else if ( ( Mode == <STR_LIT> || Mode == <STR_LIT> ) && isMatchingDecrement ( NextMBBI , Base , Bytes , <NUM_LIT> , Pred , PredReg ) ) { MI -> getOperand ( <NUM_LIT> ) . setImm ( <STR_LIT> ( Mode , true ) ) ; if ( NextMBBI == I ) { Advance = true ; ++ I ; } MBB . erase ( NextMBBI ) ; return true ; } } } else { if ( <STR_LIT> ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ) return false ; <STR_LIT> Mode = <STR_LIT> ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ; unsigned Offset = <STR_LIT> ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ; if ( MBBI != MBB . begin ( ) ) { MachineBasicBlock :: iterator PrevMBBI = prior ( MBBI ) ; if ( Mode == <STR_LIT> && isMatchingDecrement ( PrevMBBI , Base , Bytes , <NUM_LIT> , Pred , PredReg ) ) { MI -> getOperand ( <NUM_LIT> ) . setImm ( <STR_LIT> ( <STR_LIT> , true , Offset ) ) ; MBB . erase ( PrevMBBI ) ; return true ; } }


<s> if ( isLd ) { BuildMI ( MBB , MBBI , MBBI -> getDebugLoc ( ) , TII -> get ( NewOpc ) ) . addReg ( BaseReg , getKillRegState ( BaseKill ) ) . addImm ( Pred ) . addReg ( PredReg ) . addReg ( EvenReg , getDefRegState ( isLd ) | getDeadRegState ( EvenDeadKill ) ) . addReg ( OddReg , getDefRegState ( isLd ) | getDeadRegState ( OddDeadKill ) ) ; ++ NumLDRD2LDM ; } else { BuildMI ( MBB , MBBI , MBBI -> getDebugLoc ( ) , TII -> get ( NewOpc ) ) . addReg ( BaseReg , getKillRegState ( BaseKill ) ) . addImm ( Pred ) . addReg ( PredReg ) . addReg ( EvenReg , getKillRegState ( EvenDeadKill ) | getUndefRegState ( EvenUndef ) ) . addReg ( OddReg , getKillRegState ( OddDeadKill ) | getUndefRegState ( OddUndef ) ) ; ++ NumSTRD2STM ; } NewBBI = std :: prev ( MBBI ) ; } else { unsigned NewOpc = ( isLd ) ? ( isT2 ? ( OffImm < <NUM_LIT> ? <STR_LIT> : <STR_LIT> ) : <STR_LIT> ) : ( isT2 ? ( OffImm < <NUM_LIT> ? <STR_LIT> : <STR_LIT> ) : <STR_LIT> ) ; unsigned NewOpc2 = ( isLd ) ? ( isT2 ? ( OffImm + <NUM_LIT> < <NUM_LIT> ? <STR_LIT> : <STR_LIT> ) : <STR_LIT> ) : ( isT2 ? ( OffImm + <NUM_LIT> < <NUM_LIT> ? <STR_LIT> : <STR_LIT> ) : <STR_LIT> ) ; DebugLoc dl = MBBI -> getDebugLoc ( ) ; if ( isLd && ( BaseKill || OffKill ) && ( TRI -> regsOverlap ( EvenReg , BaseReg ) ) ) { assert ( ! TRI -> regsOverlap ( OddReg , BaseReg ) ) ; InsertLDR_STR ( MBB , MBBI , OffImm + <NUM_LIT> , isLd , dl , NewOpc2 , OddReg , OddDeadKill , false , BaseReg , false , BaseUndef , false , OffUndef , Pred , PredReg , TII , isT2 ) ; NewBBI = std :: prev ( MBBI ) ; InsertLDR_STR ( MBB , MBBI , OffImm , isLd , dl , NewOpc , EvenReg , EvenDeadKill , false , BaseReg , BaseKill , BaseUndef , OffKill , OffUndef , Pred , PredReg , TII , isT2 ) ; } else { if ( OddReg == EvenReg && EvenDeadKill ) { EvenDeadKill = false ; OddDeadKill = true ;


<s> unsigned PredReg = <NUM_LIT> ; <STR_LIT> Pred = getInstrPredicate ( MBBI , PredReg ) ; int Offset = getMemoryOpOffset ( MBBI ) ; bool Clobber = isi32Load ( Opcode ) && Base == MBBI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool Overlap = false ; for ( MemOpQueueIter I = MemOps . begin ( ) , E = MemOps . end ( ) ; I != E ; ++ I ) { if ( TRI -> regsOverlap ( Reg , I -> MBBI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { Overlap = true ; break ; } } if ( CurrBase == <NUM_LIT> && ! Clobber ) { CurrBase = Base ; CurrOpc = Opcode ; CurrSize = Size ; CurrPred = Pred ; CurrPredReg = PredReg ; MemOps . push_back ( MemOpQueueEntry ( Offset , Reg , isKill , Position , MBBI ) ) ; ++ NumMemOps ; Advance = true ; } else if ( ! Overlap ) { if ( Clobber ) { TryMerge = true ; Advance = true ; } if ( CurrOpc == Opcode && CurrBase == Base && CurrPred == Pred ) { if ( Offset > MemOps . back ( ) . Offset ) { MemOps . push_back ( MemOpQueueEntry ( Offset , Reg , isKill , Position , MBBI ) ) ; ++ NumMemOps ; Advance = true ; } else { for ( MemOpQueueIter I = MemOps . begin ( ) , E = MemOps . end ( ) ; I != E ; ++ I ) { if ( Offset < I -> Offset ) { MemOps . insert ( I , MemOpQueueEntry ( Offset , Reg , isKill , Position , MBBI ) ) ; ++ NumMemOps ; Advance = true ; break ; } else if ( Offset == I -> Offset ) { break ; } }


<s> bool isLd = isi32Load ( Opcode ) || Opcode == <STR_LIT> || Opcode == <STR_LIT> ; if ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == Base ) return false ; unsigned PredReg = <NUM_LIT> ; <STR_LIT> Pred = getInstrPredicate ( MI , PredReg ) ; bool DoMerge = false ; <STR_LIT> AddSub = <STR_LIT> ; unsigned NewOpc = <NUM_LIT> ; unsigned Limit = isAM5 ? <NUM_LIT> : ( isAM2 ? <NUM_LIT> : <NUM_LIT> ) ; MachineBasicBlock :: iterator BeginMBBI = MBB . begin ( ) ; if ( MBBI != BeginMBBI ) { MachineBasicBlock :: iterator PrevMBBI = std :: prev ( MBBI ) ; while ( PrevMBBI != BeginMBBI && PrevMBBI -> isDebugValue ( ) ) -- PrevMBBI ; if ( isMatchingDecrement ( PrevMBBI , Base , Bytes , Limit , Pred , PredReg ) ) { DoMerge = true ; AddSub = <STR_LIT> ; } else if ( ! isAM5 && isMatchingIncrement ( PrevMBBI , Base , Bytes , Limit , Pred , PredReg ) ) { DoMerge = true ; } if ( DoMerge ) { NewOpc = getPreIndexedLoadStoreOpcode ( Opcode , AddSub ) ; MBB . erase ( PrevMBBI ) ; } } MachineBasicBlock :: iterator EndMBBI = MBB . end ( ) ; if ( ! DoMerge && MBBI != EndMBBI ) { MachineBasicBlock :: iterator NextMBBI = std :: next ( MBBI ) ; while ( NextMBBI != EndMBBI && NextMBBI -> isDebugValue ( ) ) ++ NextMBBI ; if ( ! isAM5 && isMatchingDecrement ( NextMBBI , Base , Bytes , Limit , Pred , PredReg ) ) { DoMerge = true ; AddSub = <STR_LIT> ; } else if ( isMatchingIncrement ( NextMBBI , Base , Bytes , Limit , Pred , PredReg ) ) { DoMerge = true ; } if ( DoMerge ) { NewOpc = getPostIndexedLoadStoreOpcode ( Opcode , AddSub ) ; if ( NextMBBI == I ) { Advance = true ; ++ I ; } MBB . erase ( NextMBBI ) ; } } if ( ! DoMerge ) return false ; if ( isAM5 ) { MachineOperand & MO = MI -> getOperand ( <NUM_LIT> ) ; BuildMI ( MBB , MBBI , dl , TII -> get ( NewOpc ) ) . addReg ( Base , getDefRegState ( true ) ) . addReg ( Base , getKillRegState ( isLd ? BaseKill : false ) ) . addImm ( Pred ) . addReg ( PredReg ) . addReg ( MO . getReg ( ) , ( isLd ? getDefRegState ( true ) : getKillRegState ( MO . isKill ( ) ) ) ) ; } else if ( isLd ) { if ( isAM2 ) { if ( NewOpc == <STR_LIT> || NewOpc == <STR_LIT> ) {

int Offset = AddSub == <STR_LIT> ? - Bytes : Bytes ;


<s> bool BaseKill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; unsigned Bytes = getLSMultipleTransferSize ( MI ) ; unsigned PredReg = <NUM_LIT> ; <STR_LIT> Pred = getInstrPredicate ( MI , PredReg ) ; unsigned Opcode = MI -> getOpcode ( ) ; DebugLoc dl = MI -> getDebugLoc ( ) ; for ( unsigned i = <NUM_LIT> , e = MI -> getNumOperands ( ) ; i != e ; ++ i ) if ( MI -> getOperand ( i ) . getReg ( ) == Base ) return false ; bool DoMerge = false ; <STR_LIT> Mode = getLoadStoreMultipleSubMode ( Opcode ) ; MachineBasicBlock :: iterator BeginMBBI = MBB . begin ( ) ; if ( MBBI != BeginMBBI ) { MachineBasicBlock :: iterator PrevMBBI = std :: prev ( MBBI ) ; while ( PrevMBBI != BeginMBBI && PrevMBBI -> isDebugValue ( ) ) -- PrevMBBI ; if ( Mode == <STR_LIT> && isMatchingDecrement ( PrevMBBI , Base , Bytes , <NUM_LIT> , Pred , PredReg ) ) { Mode = <STR_LIT> ; DoMerge = true ; } else if ( Mode == <STR_LIT> && isMatchingDecrement ( PrevMBBI , Base , Bytes , <NUM_LIT> , Pred , PredReg ) ) { Mode = <STR_LIT> ; DoMerge = true ; } if ( DoMerge ) MBB . erase ( PrevMBBI ) ; } MachineBasicBlock :: iterator EndMBBI = MBB . end ( ) ; if ( ! DoMerge && MBBI != EndMBBI ) { MachineBasicBlock :: iterator NextMBBI = std :: next ( MBBI ) ; while ( NextMBBI != EndMBBI && NextMBBI -> isDebugValue ( ) ) ++ NextMBBI ; if ( ( Mode == <STR_LIT> || Mode == <STR_LIT> ) && isMatchingIncrement ( NextMBBI , Base , Bytes , <NUM_LIT> , Pred , PredReg ) ) { DoMerge = true ; } else if ( ( Mode == <STR_LIT> || Mode == <STR_LIT> ) && isMatchingDecrement ( NextMBBI , Base , Bytes , <NUM_LIT> , Pred , PredReg ) ) { DoMerge = true ; } if ( DoMerge ) { if ( NextMBBI == I ) {


<s> case <STR_LIT> : Limit = <NUM_LIT> ; break ; case <STR_LIT> : Limit = <NUM_LIT> ; break ; case <STR_LIT> : Limit = <NUM_LIT> ; break ; } for ( unsigned i = SIndex + <NUM_LIT> , e = MemOps . size ( ) ; i != e ; ++ i ) { int NewOffset = MemOps [ i ] . Offset ; const MachineOperand & MO = MemOps [ i ] . MBBI -> getOperand ( <NUM_LIT> ) ; unsigned Reg = MO . getReg ( ) ; unsigned RegNum = MO . isUndef ( ) ? UINT_MAX : TRI -> getEncodingValue ( Reg ) ; if ( Reg != <STR_LIT> && NewOffset == Offset + ( int ) Size && ( ( isNotVFP && RegNum > PRegNum ) || ( ( Count < Limit ) && RegNum == PRegNum + <NUM_LIT> ) ) && ( ! STI -> isSwift ( ) || isNotVFP || Count != <NUM_LIT> || ! ( PRegNum & <NUM_LIT> ) ) ) { Offset += Size ; PRegNum = RegNum ; ++ Count ; } else { BaseKill = Loc -> killsRegister ( Base ) ; MergeOpsUpdate ( MBB , MemOps , SIndex , i , insertAfter , SOffset , Base , BaseKill , Opcode , Pred , PredReg , Scratch , dl , Merges ) ; MergeLDR_STR ( MBB , i , Base , Opcode , Size , Pred , PredReg , Scratch , MemOps , Merges ) ; return ; } if ( MemOps [ i ] . Position > MemOps [ insertAfter ] . Position ) { insertAfter = i ; Loc = MemOps [ i ] . MBBI ; }


<s> } } SmallVector < std :: pair < unsigned , bool > , <NUM_LIT> > Regs ; SmallVector < unsigned , <NUM_LIT> > ImpDefs ; SmallVector < MachineOperand * , <NUM_LIT> > UsesOfImpDefs ; for ( unsigned i = memOpsBegin ; i < memOpsEnd ; ++ i ) { unsigned Reg = memOps [ i ] . Reg ; bool isKill = memOps [ i ] . isKill || KilledRegs . count ( Reg ) ; Regs . push_back ( std :: make_pair ( Reg , isKill ) ) ; for ( const MachineOperand & MO : memOps [ i ] . MBBI -> operands ( ) ) { if ( ! MO . isReg ( ) || ! MO . isDef ( ) || ! MO . isImplicit ( ) || MO . isDead ( ) ) continue ; unsigned DefReg = MO . getReg ( ) ; if ( std :: find ( ImpDefs . begin ( ) , ImpDefs . end ( ) , DefReg ) == ImpDefs . end ( ) ) ImpDefs . push_back ( DefReg ) ; findUsesOfImpDef ( UsesOfImpDefs , memOps , DefReg , memOps [ i ] . Position , insertPos ) ; } } MachineBasicBlock :: iterator Loc = memOps [ insertAfter ] . MBBI ; ++ Loc ; if ( ! MergeOps ( MBB , Loc , Offset , Base , BaseKill , Opcode , Pred , PredReg , Scratch , dl , Regs , ImpDefs ) ) return ; Merges . push_back ( std :: prev ( Loc ) ) ; for ( SmallVectorImpl < MachineOperand * > :: iterator I = UsesOfImpDefs . begin ( ) , E = UsesOfImpDefs . end ( ) ; I != E ; ++ I ) ( * I ) -> setIsUndef ( ) ; for ( unsigned i = memOpsBegin ; i < memOpsEnd ; ++ i ) { if ( Regs [ i - memOpsBegin ] . second ) { unsigned Reg = Regs [ i - memOpsBegin ] . first ; if ( KilledRegs . count ( Reg ) ) { unsigned j = Killer [ Reg ] ;

int Idx = memOps [ j ] . MBBI -> findRegisterUseOperandIdx ( Reg , true ) ;


<s> bool IsStore = Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ; if ( IsLoad || IsStore ) { MachineOperand & MO = MBBI -> getOperand ( MBBI -> getDesc ( ) . getNumOperands ( ) - <NUM_LIT> ) ; Offset = MO . getImm ( ) - WordOffset * getImmScale ( Opc ) ; unsigned InstrSrcReg = MBBI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( Offset >= <NUM_LIT> && ! ( IsStore && InstrSrcReg == Base ) ) MO . setImm ( Offset ) ; else InsertSub = true ; } else if ( ( Opc == <STR_LIT> || Opc == <STR_LIT> ) && ! definesCPSR ( MBBI ) ) { MachineOperand & MO = MBBI -> getOperand ( MBBI -> getDesc ( ) . getNumOperands ( ) - <NUM_LIT> ) ; Offset = ( Opc == <STR_LIT> ) ? MO . getImm ( ) + WordOffset * <NUM_LIT> : MO . getImm ( ) - WordOffset * <NUM_LIT> ; if ( Offset >= <NUM_LIT> && TL -> isLegalAddImmediate ( Offset ) ) { MO . setImm ( Offset ) ; return ; } else { InsertSub = true ; } } else { InsertSub = true ; } } else if ( definesCPSR ( MBBI ) || MBBI -> isCall ( ) || MBBI -> isBranch ( ) ) { InsertSub = true ; }


<s> int OffImm = getMemoryOpOffset ( MI ) ; unsigned PredReg = <NUM_LIT> ; <STR_LIT> Pred = getInstrPredicate ( MI , PredReg ) ; if ( OddRegNum > EvenRegNum && OffImm == <NUM_LIT> ) { unsigned NewOpc = ( isLd ) ? ( isT2 ? <STR_LIT> : <STR_LIT> ) : ( isT2 ? <STR_LIT> : <STR_LIT> ) ; if ( isLd ) { BuildMI ( MBB , MBBI , MBBI -> getDebugLoc ( ) , TII -> get ( NewOpc ) ) . addReg ( BaseReg , getKillRegState ( BaseKill ) ) . addImm ( Pred ) . addReg ( PredReg ) . addReg ( EvenReg , getDefRegState ( isLd ) | getDeadRegState ( EvenDeadKill ) ) . addReg ( OddReg , getDefRegState ( isLd ) | getDeadRegState ( OddDeadKill ) ) ; ++ NumLDRD2LDM ; } else { BuildMI ( MBB , MBBI , MBBI -> getDebugLoc ( ) , TII -> get ( NewOpc ) ) . addReg ( BaseReg , getKillRegState ( BaseKill ) ) . addImm ( Pred ) . addReg ( PredReg ) . addReg ( EvenReg , getKillRegState ( EvenDeadKill ) | getUndefRegState ( EvenUndef ) ) . addReg ( OddReg , getKillRegState ( OddDeadKill ) | getUndefRegState ( OddUndef ) ) ; ++ NumSTRD2STM ; } NewBBI = llvm :: prior ( MBBI ) ; } else { unsigned NewOpc = ( isLd ) ? ( isT2 ? ( OffImm < <NUM_LIT> ? <STR_LIT> : <STR_LIT> ) : <STR_LIT> ) : ( isT2 ? ( OffImm < <NUM_LIT> ? <STR_LIT> : <STR_LIT> ) : <STR_LIT> ) ; DebugLoc dl = MBBI -> getDebugLoc ( ) ; if ( isLd && ( BaseKill || OffKill ) && ( TRI -> regsOverlap ( EvenReg , BaseReg ) ) ) { assert ( ! TRI -> regsOverlap ( OddReg , BaseReg ) ) ; InsertLDR_STR ( MBB , MBBI , OffImm + <NUM_LIT> , isLd , dl , NewOpc , OddReg , OddDeadKill , false , BaseReg , false , BaseUndef , false , OffUndef , Pred , PredReg , TII , isT2 ) ; NewBBI = llvm :: prior ( MBBI ) ;

if ( isT2 && NewOpc == <STR_LIT> && OffImm + <NUM_LIT> >= <NUM_LIT> ) NewOpc = <STR_LIT> ;


<s> ( isThumb1 && Offset < <NUM_LIT> ) ? <STR_LIT> : isThumb1 ? <STR_LIT> : <STR_LIT> ; if ( Offset < <NUM_LIT> ) { Offset = - Offset ; BaseOpc = isThumb2 ? <STR_LIT> : ( isThumb1 && Offset < <NUM_LIT> && Base != <STR_LIT> ) ? <STR_LIT> : isThumb1 ? <STR_LIT> : <STR_LIT> ; } if ( ! TL -> isLegalAddImmediate ( Offset ) ) return nullptr ; bool KillOldBase = BaseKill && ( ! isi32Store ( Opcode ) || ! ContainsReg ( Regs , Base ) ) ; if ( isThumb1 ) { if ( Base != NewBase && ( BaseOpc == <STR_LIT> || BaseOpc == <STR_LIT> ) ) { if ( is<ISA_LIT>LowRegister ( NewBase ) && is<ISA_LIT>LowRegister ( Base ) && ! STI -> hasV6Ops ( ) ) { if ( Pred != <STR_LIT> ) return nullptr ; BuildMI ( MBB , InsertBefore , DL , TII -> get ( <STR_LIT> ) , NewBase ) . addReg ( Base , getKillRegState ( KillOldBase ) ) ; } else BuildMI ( MBB , InsertBefore , DL , TII -> get ( <STR_LIT> ) , NewBase ) . addReg ( Base , getKillRegState ( KillOldBase ) ) . add ( predOps ( Pred , PredReg ) ) ; Base = NewBase ; KillOldBase = true ; } if ( BaseOpc == <STR_LIT> ) { assert ( Offset % <NUM_LIT> == <NUM_LIT> && <STR_LIT> ) ; BuildMI ( MBB , InsertBefore , DL , TII -> get ( BaseOpc ) , NewBase ) . addReg ( Base , getKillRegState ( KillOldBase ) ) . addImm ( Offset / <NUM_LIT> ) . add ( predOps ( Pred , PredReg ) ) ; } else BuildMI ( MBB , InsertBefore , DL , TII -> get ( BaseOpc ) , NewBase ) . add ( t1CondCodeOp ( true ) ) . addReg ( Base , getKillRegState ( KillOldBase ) ) . addImm ( Offset ) . add ( predOps ( Pred , PredReg ) ) ; } else { BuildMI ( MBB , InsertBefore , DL , TII -> get ( BaseOpc ) , NewBase ) . addReg ( Base , getKillRegState ( KillOldBase ) ) . addImm ( Offset ) . add ( predOps ( Pred , PredReg ) ) . add ( condCodeOp ( ) ) ; } Base = NewBase ; BaseKill = true ; } bool isDef = isLoadSingle ( Opcode ) ; Opcode = getLoadStoreMultipleOpcode ( Opcode , Mode ) ; if ( ! Opcode ) return nullptr ; if ( isThumb1 && ! SafeToClobberCPSR && Writeback && ! BaseKill ) return nullptr ; MachineInstrBuilder MIB ; if ( Writeback ) { assert ( isThumb1 && <STR_LIT> ) ; if ( Opcode == <STR_LIT> ) { assert ( ! ( ContainsReg ( Regs , Base ) ) && <STR_LIT> ) ; Opcode = <STR_LIT> ; }

MIB = BuildMI ( MBB , InsertBefore , DL , TII -> get ( Opcode ) ) ;


<s> } else { BuildMI ( MBB , MBBI , MBBI -> getDebugLoc ( ) , TII -> get ( NewOpc ) ) . addReg ( BaseReg , getKillRegState ( BaseKill ) ) . addImm ( Pred ) . addReg ( PredReg ) . addReg ( EvenReg , getKillRegState ( EvenDeadKill ) | getUndefRegState ( EvenUndef ) ) . addReg ( OddReg , getKillRegState ( OddDeadKill ) | getUndefRegState ( OddUndef ) ) . cloneMemRefs ( * MI ) ; ++ NumSTRD2STM ; } } else { unsigned NewOpc = ( isLd ) ? ( isT2 ? ( OffImm < <NUM_LIT> ? <STR_LIT> : <STR_LIT> ) : <STR_LIT> ) : ( isT2 ? ( OffImm < <NUM_LIT> ? <STR_LIT> : <STR_LIT> ) : <STR_LIT> ) ; unsigned NewOpc2 = ( isLd ) ? ( isT2 ? ( OffImm + <NUM_LIT> < <NUM_LIT> ? <STR_LIT> : <STR_LIT> ) : <STR_LIT> ) : ( isT2 ? ( OffImm + <NUM_LIT> < <NUM_LIT> ? <STR_LIT> : <STR_LIT> ) : <STR_LIT> ) ; if ( isLd && TRI -> regsOverlap ( EvenReg , BaseReg ) ) { assert ( ! TRI -> regsOverlap ( OddReg , BaseReg ) ) ; InsertLDR_STR ( MBB , MBBI , OffImm + <NUM_LIT> , isLd , NewOpc2 , OddReg , OddDeadKill , false , BaseReg , false , BaseUndef , Pred , PredReg , TII , MI ) ; InsertLDR_STR ( MBB , MBBI , OffImm , isLd , NewOpc , EvenReg , EvenDeadKill , false , BaseReg , BaseKill , BaseUndef , Pred , PredReg , TII , MI ) ; } else { if ( OddReg == EvenReg && EvenDeadKill ) { EvenDeadKill = false ; OddDeadKill = true ; } if ( EvenReg == BaseReg ) EvenDeadKill = false ; InsertLDR_STR ( MBB , MBBI , OffImm , isLd , NewOpc , EvenReg , EvenDeadKill , EvenUndef , BaseReg , false , BaseUndef , Pred , PredReg , TII , MI ) ; InsertLDR_STR ( MBB , MBBI , OffImm + <NUM_LIT> , isLd , NewOpc2 , OddReg , OddDeadKill , OddUndef , BaseReg , BaseKill , BaseUndef , Pred , PredReg , TII , MI ) ; } if ( isLd ) ++ NumLDRD2LDR ; else ++ NumSTRD2STR ;


<s> MachineBasicBlock & MBB = * MI -> getParent ( ) ; MachineBasicBlock :: iterator MBBI ( MI ) ; int Offset ; MachineBasicBlock :: iterator MergeInstr = findIncDecBefore ( MBBI , Base , Pred , PredReg , Offset ) ; unsigned NewOpc ; if ( ! isAM5 && Offset == Bytes ) { NewOpc = getPreIndexedLoadStoreOpcode ( Opcode , <STR_LIT> ) ; } else if ( Offset == - Bytes ) { NewOpc = getPreIndexedLoadStoreOpcode ( Opcode , <STR_LIT> ) ; } else { MergeInstr = findIncDecAfter ( MBBI , Base , Pred , PredReg , Offset ) ; if ( Offset == Bytes ) { NewOpc = getPostIndexedLoadStoreOpcode ( Opcode , <STR_LIT> ) ; } else if ( ! isAM5 && Offset == - Bytes ) { NewOpc = getPostIndexedLoadStoreOpcode ( Opcode , <STR_LIT> ) ; } else return false ; } MBB . erase ( MergeInstr ) ; <STR_LIT> AddSub = Offset < <NUM_LIT> ? <STR_LIT> : <STR_LIT> ; bool isLd = isLoadSingle ( Opcode ) ; if ( isAM5 ) { MachineOperand & MO = MI -> getOperand ( <NUM_LIT> ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( NewOpc ) ) . addReg ( Base , getDefRegState ( true ) ) . addReg ( Base , getKillRegState ( isLd ? BaseKill : false ) ) . addImm ( Pred ) . addReg ( PredReg ) . addReg ( MO . getReg ( ) , ( isLd ? getDefRegState ( true ) : getKillRegState ( MO . isKill ( ) ) ) ) . cloneMemRefs ( * MI ) ; } else if ( isLd ) { if ( isAM2 ) { if ( NewOpc == <STR_LIT> || NewOpc == <STR_LIT> ) { BuildMI ( MBB , MBBI , DL , TII -> get ( NewOpc ) , MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) . addReg ( Base , RegState :: Define ) . addReg ( Base ) . addImm ( Offset ) . addImm ( Pred ) . addReg ( PredReg ) . cloneMemRefs ( * MI ) ; } else { int Imm = <STR_LIT> ( AddSub , Bytes , <STR_LIT> ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( NewOpc ) , MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) . addReg ( Base , RegState :: Define ) . addReg ( Base ) . addReg ( <NUM_LIT> ) . addImm ( Imm ) . add ( predOps ( Pred , PredReg ) ) . cloneMemRefs ( * MI ) ; }


<s> int Offset ; MachineBasicBlock :: iterator MergeInstr = findIncDecBefore ( MBBI , Base , Pred , PredReg , Offset ) ; unsigned NewOpc ; if ( Offset == <NUM_LIT> || Offset == - <NUM_LIT> ) { NewOpc = Opcode == <STR_LIT> ? <STR_LIT> : <STR_LIT> ; } else { MergeInstr = findIncDecAfter ( MBBI , Base , Pred , PredReg , Offset ) ; if ( Offset == <NUM_LIT> || Offset == - <NUM_LIT> ) { NewOpc = Opcode == <STR_LIT> ? <STR_LIT> : <STR_LIT> ; } else return false ; } MBB . erase ( MergeInstr ) ; DebugLoc DL = MI . getDebugLoc ( ) ; MachineInstrBuilder MIB = BuildMI ( MBB , MBBI , DL , TII -> get ( NewOpc ) ) ; if ( NewOpc == <STR_LIT> || NewOpc == <STR_LIT> ) { MIB . add ( Reg0Op ) . add ( Reg1Op ) . addReg ( BaseOp . getReg ( ) , RegState :: Define ) ; } else { assert ( NewOpc == <STR_LIT> || NewOpc == <STR_LIT> ) ; MIB . addReg ( BaseOp . getReg ( ) , RegState :: Define ) . add ( Reg0Op ) . add ( Reg1Op ) ; } MIB . addReg ( BaseOp . getReg ( ) , RegState :: Kill ) . addImm ( Offset ) . addImm ( Pred ) . addReg ( PredReg ) ;

assert ( TII -> get ( Opcode ) . getNumOperands ( ) == <NUM_LIT> && TII -> get ( NewOpc ) . getNumOperands ( ) == <NUM_LIT> && <STR_LIT> ) ;


<s> if ( isThumb1 ) return false ; const MachineOperand & BaseOP = MI -> getOperand ( <NUM_LIT> ) ; unsigned Base = BaseOP . getReg ( ) ; bool BaseKill = BaseOP . isKill ( ) ; unsigned PredReg = <NUM_LIT> ; <STR_LIT> Pred = getInstrPredicate ( * MI , PredReg ) ; unsigned Opcode = MI -> getOpcode ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; for ( unsigned i = <NUM_LIT> , e = MI -> getNumOperands ( ) ; i != e ; ++ i ) if ( MI -> getOperand ( i ) . getReg ( ) == Base ) return false ; int Bytes = getLSMultipleTransferSize ( MI ) ; MachineBasicBlock & MBB = * MI -> getParent ( ) ; MachineBasicBlock :: iterator MBBI ( MI ) ; int Offset ; MachineBasicBlock :: iterator MergeInstr = findIncDecBefore ( MBBI , Base , Pred , PredReg , Offset ) ; <STR_LIT> Mode = getLoadStoreMultipleSubMode ( Opcode ) ; if ( Mode == <STR_LIT> && Offset == - Bytes ) { Mode = <STR_LIT> ; } else if ( Mode == <STR_LIT> && Offset == - Bytes ) { Mode = <STR_LIT> ; } else { MergeInstr = findIncDecAfter ( MBBI , Base , Pred , PredReg , Offset ) ; if ( ( ( Mode != <STR_LIT> && Mode != <STR_LIT> ) || Offset != Bytes ) && ( ( Mode != <STR_LIT> && Mode != <STR_LIT> ) || Offset != - Bytes ) ) {

if ( ! STI -> hasMinSize ( ) || ! BaseKill ) return false ;


<s> <STR_LIT> Pred = getInstrPredicate ( * MI , PredReg ) ; unsigned Opcode = MI -> getOpcode ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; for ( unsigned i = <NUM_LIT> , e = MI -> getNumOperands ( ) ; i != e ; ++ i ) if ( MI -> getOperand ( i ) . getReg ( ) == Base ) return false ; int Bytes = getLSMultipleTransferSize ( MI ) ; MachineBasicBlock & MBB = * MI -> getParent ( ) ; MachineBasicBlock :: iterator MBBI ( MI ) ; int Offset ; MachineBasicBlock :: iterator MergeInstr = findIncDecBefore ( MBBI , Base , Pred , PredReg , Offset ) ; <STR_LIT> Mode = getLoadStoreMultipleSubMode ( Opcode ) ; if ( Mode == <STR_LIT> && Offset == - Bytes ) { Mode = <STR_LIT> ; } else if ( Mode == <STR_LIT> && Offset == - Bytes ) { Mode = <STR_LIT> ; } else { MergeInstr = findIncDecAfter ( MBBI , Base , Pred , PredReg , Offset ) ; if ( ( ( Mode != <STR_LIT> && Mode != <STR_LIT> ) || Offset != Bytes ) && ( ( Mode != <STR_LIT> && Mode != <STR_LIT> ) || Offset != - Bytes ) ) { if ( ! STI -> hasMinSize ( ) || ! BaseKill ) return false ; bool HighRegsUsed = false ; for ( unsigned i = <NUM_LIT> , e = MI -> getNumOperands ( ) ; i != e ; ++ i ) if ( MI -> getOperand ( i ) . getReg ( ) >= <STR_LIT> ) { HighRegsUsed = true ; break ; } if ( ! HighRegsUsed ) MergeInstr = MBB . end ( ) ; else return false ; } }

if ( MergeInstr != MBB . end ( ) ) MBB . erase ( MergeInstr ) ;


<s> } } using iterator = MachineBasicBlock :: iterator ; MachineInstr * LatestMI = Cand . Instrs [ Cand . LatestMIIdx ] ; iterator InsertBefore = std :: next ( iterator ( LatestMI ) ) ; MachineBasicBlock & MBB = * LatestMI -> getParent ( ) ; unsigned Offset = getMemoryOpOffset ( * First ) ; unsigned Base = getLoadStoreBaseOp ( * First ) . getReg ( ) ; bool BaseKill = LatestMI -> killsRegister ( Base ) ; unsigned PredReg = <NUM_LIT> ; <STR_LIT> Pred = getInstrPredicate ( * First , PredReg ) ; DebugLoc DL = First -> getDebugLoc ( ) ; MachineInstr * Merged = nullptr ; if ( Cand . CanMergeToLSDouble ) Merged = CreateLoadStoreDouble ( MBB , InsertBefore , Offset , Base , BaseKill , Opcode , Pred , PredReg , DL , Regs , Cand . Instrs ) ; if ( ! Merged && Cand . CanMergeToLSMulti ) Merged = CreateLoadStoreMulti ( MBB , InsertBefore , Offset , Base , BaseKill , Opcode , Pred , PredReg , DL , Regs , Cand . Instrs ) ; if ( ! Merged ) return nullptr ; iterator EarliestI ( Cand . Instrs [ Cand . EarliestMIIdx ] ) ; bool EarliestAtBegin = false ; if ( EarliestI == MBB . begin ( ) ) { EarliestAtBegin = true ; } else { EarliestI = std :: prev ( EarliestI ) ; } for ( MachineInstr * MI : Cand . Instrs ) MBB . erase ( MI ) ; if ( EarliestAtBegin ) EarliestI = MBB . begin ( ) ; else EarliestI = std :: next ( EarliestI ) ; auto FixupRange = make_range ( EarliestI , iterator ( Merged ) ) ; if ( isLoadSingle ( Opcode ) ) { for ( MachineInstr & MI : FixupRange ) { for ( unsigned & ImpDefReg : ImpDefs ) { for ( MachineOperand & MO : MI . implicit_operands ( ) ) { if ( ! MO . isReg ( ) || MO . getReg ( ) != ImpDefReg ) continue ; if ( MO . readsReg ( ) ) MO . setIsUndef ( ) ; else if ( MO . isDef ( ) ) ImpDefReg = <NUM_LIT> ;


<s> for ( ; MBBI != E ; ++ MBBI ) { MachineInstr & MI = * MBBI ; if ( MI . isCall ( ) || MI . isTerminator ( ) ) { ++ MBBI ; break ; } if ( ! MI . isDebugInstr ( ) ) MI2LocMap [ & MI ] = ++ Loc ; if ( ! isMemoryOp ( MI ) ) continue ; unsigned PredReg = <NUM_LIT> ; if ( getInstrPredicate ( MI , PredReg ) != <STR_LIT> ) continue ; int Opc = MI . getOpcode ( ) ; bool isLd = isLoadSingle ( Opc ) ; unsigned Base = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; int Offset = getMemoryOpOffset ( MI ) ; bool StopHere = false ; auto FindBases = [ & ] ( Base2InstMap & Base2Ops , BaseVec & Bases ) { MapIt BI = Base2Ops . find ( Base ) ; if ( BI == Base2Ops . end ( ) ) { Base2Ops [ Base ] . push_back ( & MI ) ; Bases . push_back ( Base ) ; return ; } for ( unsigned i = <NUM_LIT> , e = BI -> second . size ( ) ; i != e ; ++ i ) { if ( Offset == getMemoryOpOffset ( * BI -> second [ i ] ) ) { StopHere = true ; break ; } } if ( ! StopHere ) BI -> second . push_back ( & MI ) ; } ; if ( isLd ) FindBases ( Base2LdsMap , LdBases ) ; else FindBases ( Base2StsMap , StBases ) ; if ( StopHere ) { -- Loc ; break ; } } for ( unsigned i = <NUM_LIT> , e = LdBases . size ( ) ; i != e ; ++ i ) { unsigned Base = LdBases [ i ] ; SmallVectorImpl < MachineInstr * > & Lds = Base2LdsMap [ Base ] ; if ( Lds . size ( ) > <NUM_LIT> ) RetVal |= RescheduleOps ( MBB , Lds , Base , true , MI2LocMap ) ; } for ( unsigned i = <NUM_LIT> , e = StBases . size ( ) ; i != e ; ++ i ) { unsigned Base = StBases [ i ] ; SmallVectorImpl < MachineInstr * > & Sts = Base2StsMap [ Base ] ; if ( Sts . size ( ) > <NUM_LIT> ) RetVal |= RescheduleOps ( MBB , Sts , Base , false , MI2LocMap ) ; } if ( MBBI != E ) { Base2LdsMap . clear ( ) ;


<s> if ( ! SafeToClobberCPSR ) return false ; unsigned NewBase ; if ( isi32Load ( Opcode ) ) { NewBase = Regs [ NumRegs - <NUM_LIT> ] . first ; } else { NewBase = Scratch ; if ( NewBase == <NUM_LIT> ) return false ; } int BaseOpc = isThumb2 ? <STR_LIT> : ( isThumb1 && Offset < <NUM_LIT> ) ? <STR_LIT> : isThumb1 ? <STR_LIT> : <STR_LIT> ; if ( Offset < <NUM_LIT> ) { Offset = - Offset ; BaseOpc = isThumb2 ? <STR_LIT> : ( isThumb1 && Offset < <NUM_LIT> ) ? <STR_LIT> : isThumb1 ? <STR_LIT> : <STR_LIT> ; } if ( ! TL -> isLegalAddImmediate ( Offset ) ) return false ; if ( isThumb1 ) { if ( Base != NewBase && Offset >= <NUM_LIT> ) { BuildMI ( MBB , MBBI , dl , TII -> get ( <STR_LIT> ) , NewBase ) . addReg ( Base , getKillRegState ( BaseKill ) ) . addImm ( Pred ) . addReg ( PredReg ) ; Base = NewBase ; BaseKill = false ; } AddDefaultT1CC ( BuildMI ( MBB , MBBI , dl , TII -> get ( BaseOpc ) , NewBase ) , true ) . addReg ( Base , getKillRegState ( BaseKill ) ) . addImm ( Offset ) . addImm ( Pred ) . addReg ( PredReg ) ; } else { BuildMI ( MBB , MBBI , dl , TII -> get ( BaseOpc ) , NewBase ) . addReg ( Base , getKillRegState ( BaseKill ) ) . addImm ( Offset ) . addImm ( Pred ) . addReg ( PredReg ) . addReg ( <NUM_LIT> ) ; } Base = NewBase ; BaseKill = true ; } bool isDef = ( isi32Load ( Opcode ) || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) ; Opcode = getLoadStoreMultipleOpcode ( Opcode , Mode ) ; if ( ! Opcode ) return false ; bool Writeback = isThumb1 ; if ( Opcode == <STR_LIT> && isThumb1 ) for ( unsigned I = <NUM_LIT> ; I < NumRegs ; ++ I ) if ( Base == Regs [ I ] . first ) { Writeback = false ; break ; } if ( isThumb1 && Writeback && ! BaseKill ) return false ; MachineInstrBuilder MIB ; if ( Writeback ) { if ( Opcode == <STR_LIT> ) Opcode = <STR_LIT> ; MIB = BuildMI ( MBB , MBBI , dl , TII -> get ( Opcode ) ) ;

MIB . addReg ( Base , getDefRegState ( true ) ) . addReg ( Base , getKillRegState ( BaseKill ) ) ;


<s> TII = Fn . getSubtarget ( ) . getInstrInfo ( ) ; TRI = Fn . getSubtarget ( ) . getRegisterInfo ( ) ; STI = & Fn . getTarget ( ) . getSubtarget < <ISA_LIT>Subtarget > ( ) ; MRI = & Fn . getRegInfo ( ) ; MF = & Fn ; bool Modified = false ; for ( MachineFunction :: iterator MFI = Fn . begin ( ) , E = Fn . end ( ) ; MFI != E ;

++ MFI ) Modified |= RescheduleLoadStoreInstrs ( MFI ) ;


<s> while ( InsertPos != MBB -> end ( ) && ( MemOps . count ( InsertPos ) || InsertPos -> isDebugValue ( ) ) ) ++ InsertPos ; MachineInstr * Op0 = Ops . back ( ) ; MachineInstr * Op1 = Ops [ Ops . size ( ) - <NUM_LIT> ] ; unsigned EvenReg = <NUM_LIT> , OddReg = <NUM_LIT> ; unsigned BaseReg = <NUM_LIT> , PredReg = <NUM_LIT> ; <STR_LIT> Pred = <STR_LIT> ; bool isT2 = false ; unsigned NewOpc = <NUM_LIT> ; int Offset = <NUM_LIT> ; DebugLoc dl ; if ( NumMove == <NUM_LIT> && CanFormLdStDWord ( Op0 , Op1 , dl , NewOpc , EvenReg , OddReg , BaseReg , Offset , PredReg , Pred , isT2 ) ) { Ops . pop_back ( ) ; Ops . pop_back ( ) ; const MCInstrDesc & MCID = TII -> get ( NewOpc ) ; const TargetRegisterClass * TRC = TII -> getRegClass ( MCID , <NUM_LIT> , TRI ) ; MRI -> constrainRegClass ( EvenReg , TRC ) ; MRI -> constrainRegClass ( OddReg , TRC ) ; if ( isLd ) { MachineInstrBuilder MIB = BuildMI ( * MBB , InsertPos , dl , MCID ) . addReg ( EvenReg , RegState :: Define ) . addReg ( OddReg , RegState :: Define ) . addReg ( BaseReg ) ; if ( ! isT2 ) MIB . addReg ( <NUM_LIT> ) ; MIB . addImm ( Offset ) . addImm ( Pred ) . addReg ( PredReg ) ; ++ NumLDRDFormed ; } else { MachineInstrBuilder MIB = BuildMI ( * MBB , InsertPos , dl , MCID ) . addReg ( EvenReg ) . addReg ( OddReg ) . addReg ( BaseReg ) ; if ( ! isT2 ) MIB . addReg ( <NUM_LIT> ) ; MIB . addImm ( Offset ) . addImm ( Pred ) . addReg ( PredReg ) ; ++ NumSTRDFormed ; } MBB -> erase ( Op0 ) ; MBB -> erase ( Op1 ) ; MRI -> setRegAllocationHint ( EvenReg , <STR_LIT> , OddReg ) ; MRI -> setRegAllocationHint ( OddReg , <STR_LIT> , EvenReg ) ; } else { for ( unsigned i = <NUM_LIT> ; i != NumMove ; ++ i ) { MachineInstr * Op = Ops . back ( ) ; Ops . pop_back ( ) ; MBB -> splice ( InsertPos , MBB , Op ) ;


<s> unsigned Reg = memOps [ i ] . Reg ; bool isKill = memOps [ i ] . isKill || KilledRegs . count ( Reg ) ; Regs . push_back ( std :: make_pair ( Reg , isKill ) ) ; for ( MIOperands MO ( memOps [ i ] . MBBI ) ; MO . isValid ( ) ; ++ MO ) { if ( ! MO -> isReg ( ) || ! MO -> isDef ( ) || ! MO -> isImplicit ( ) || MO -> isDead ( ) ) continue ; unsigned DefReg = MO -> getReg ( ) ; if ( std :: find ( ImpDefs . begin ( ) , ImpDefs . end ( ) , DefReg ) == ImpDefs . end ( ) ) ImpDefs . push_back ( DefReg ) ; findUsesOfImpDef ( UsesOfImpDefs , memOps , DefReg , memOps [ i ] . Position , insertPos ) ; } } MachineBasicBlock :: iterator Loc = memOps [ insertAfter ] . MBBI ; ++ Loc ; if ( ! MergeOps ( MBB , Loc , Offset , Base , BaseKill , Opcode , Pred , PredReg , Scratch , dl , Regs , ImpDefs ) ) return ; Merges . push_back ( prior ( Loc ) ) ; for ( SmallVectorImpl < MachineOperand * > :: iterator I = UsesOfImpDefs . begin ( ) , E = UsesOfImpDefs . end ( ) ; I != E ; ++ I ) ( * I ) -> setIsUndef ( ) ; for ( unsigned i = memOpsBegin ; i < memOpsEnd ; ++ i ) { if ( Regs [ i - memOpsBegin ] . second ) { unsigned Reg = Regs [ i - memOpsBegin ] . first ; if ( KilledRegs . count ( Reg ) ) { unsigned j = Killer [ Reg ] ; int Idx = memOps [ j ] . MBBI -> findRegisterUseOperandIdx ( Reg , true ) ; assert ( Idx >= <NUM_LIT> && <STR_LIT> ) ; memOps [ j ] . MBBI -> getOperand ( Idx ) . setIsKill ( false ) ; memOps [ j ] . isKill = false ; }


<s> default : return <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return ( MI -> getNumOperands ( ) - <NUM_LIT> ) * <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) * <NUM_LIT> ;


<s> if ( DoMerge ) MBB . erase ( PrevMBBI ) ; } if ( ! DoMerge && MBBI != MBB . end ( ) ) { MachineBasicBlock :: iterator NextMBBI = llvm :: next ( MBBI ) ; if ( isAM4 ) { if ( ( Mode == <STR_LIT> || Mode == <STR_LIT> ) && isMatchingIncrement ( NextMBBI , Base , Bytes , <NUM_LIT> , Pred , PredReg ) ) { DoMerge = true ; } else if ( ( Mode == <STR_LIT> || Mode == <STR_LIT> ) && isMatchingDecrement ( NextMBBI , Base , Bytes , <NUM_LIT> , Pred , PredReg ) ) { DoMerge = true ; } } else { if ( Mode == <STR_LIT> && isMatchingIncrement ( NextMBBI , Base , Bytes , <NUM_LIT> , Pred , PredReg ) ) { DoMerge = true ; } } if ( DoMerge ) { if ( NextMBBI == I ) { Advance = true ; ++ I ; } MBB . erase ( NextMBBI ) ; } } if ( ! DoMerge ) return false ; unsigned NewOpc = getUpdatingLSMultipleOpcode ( Opcode ) ; MachineInstrBuilder MIB = BuildMI ( MBB , MBBI , dl , TII -> get ( NewOpc ) ) . addReg ( Base , getDefRegState ( true ) ) . addReg ( Base , getKillRegState ( BaseKill ) ) ; if ( isAM4 ) { MIB . addImm ( <STR_LIT> ( Mode ) ) . addImm ( Pred ) . addReg ( PredReg ) ; } else { MIB . addImm ( <STR_LIT> ( Mode , Offset ) ) . addImm ( Pred ) . addReg ( PredReg ) ; } for ( unsigned OpNum = <NUM_LIT> , e = MI -> getNumOperands ( ) ; OpNum != e ; ++ OpNum ) MIB . addOperand ( MI -> getOperand ( OpNum ) ) ; ( * MIB ) . setMemRefs ( MI -> memoperands_begin ( ) , MI -> memoperands_end ( ) ) ; MBB . erase ( MBBI ) ;


<s> if ( NumRegs <= <NUM_LIT> ) return false ; unsigned NewBase ; if ( isi32Load ( Opcode ) ) NewBase = Regs [ NumRegs - <NUM_LIT> ] . first ; else { NewBase = Scratch ; if ( NewBase == <NUM_LIT> ) return false ; } int BaseOpc = ! isThumb2 ? <STR_LIT> : ( ( Base == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) ; if ( Offset < <NUM_LIT> ) { BaseOpc = ! isThumb2 ? <STR_LIT> : ( ( Base == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) ; Offset = - Offset ; } int ImmedOffset = isThumb2 ? <STR_LIT> ( Offset ) : <STR_LIT> ( Offset ) ; if ( ImmedOffset == - <NUM_LIT> ) return false ; BuildMI ( MBB , MBBI , dl , TII -> get ( BaseOpc ) , NewBase ) . addReg ( Base , getKillRegState ( BaseKill ) ) . addImm ( Offset ) . addImm ( Pred ) . addReg ( PredReg ) . addReg ( <NUM_LIT> ) ;


<s> MachineBasicBlock :: iterator MBBI = MBB . getLastNonDebugInstr ( ) ; if ( MBBI != MBB . begin ( ) && ( MBBI -> getOpcode ( ) == <STR_LIT> || MBBI -> getOpcode ( ) == <STR_LIT> || MBBI -> getOpcode ( ) == <STR_LIT> ) ) { MachineInstr * PrevMI = std :: prev ( MBBI ) ; unsigned Opcode = PrevMI -> getOpcode ( ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { MachineOperand & MO = PrevMI -> getOperand ( PrevMI -> getNumOperands ( ) - <NUM_LIT> ) ; if ( MO . getReg ( ) != <STR_LIT> ) return false ; unsigned NewOpc = ( isThumb2 ? <STR_LIT> : <STR_LIT> ) ; assert ( ( ( isThumb2 && Opcode == <STR_LIT> ) || Opcode == <STR_LIT> ) && <STR_LIT> ) ; PrevMI -> setDesc ( TII -> get ( NewOpc ) ) ; MO . setReg ( <STR_LIT> ) ;

PrevMI -> copyImplicitOps ( * MBB . getParent ( ) , & * MBBI ) ;


<s> MachineInstr * PrevMI = std :: prev ( MBBI ) ; unsigned Opcode = PrevMI -> getOpcode ( ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { MachineOperand & MO = PrevMI -> getOperand ( PrevMI -> getNumOperands ( ) - <NUM_LIT> ) ; if ( MO . getReg ( ) != <STR_LIT> ) return false ; unsigned NewOpc = ( isThumb2 ? <STR_LIT> : <STR_LIT> ) ; assert ( ( ( isThumb2 && Opcode == <STR_LIT> ) || Opcode == <STR_LIT> ) && <STR_LIT> ) ; PrevMI -> setDesc ( TII -> get ( NewOpc ) ) ; MO . setReg ( <STR_LIT> ) ;


<s> if ( ! DoMerge && MBBI != MBB . end ( ) ) { MachineBasicBlock :: iterator NextMBBI = llvm :: next ( MBBI ) ; if ( ! isAM5 && isMatchingDecrement ( NextMBBI , Base , Bytes , Limit , Pred , PredReg ) ) { DoMerge = true ; AddSub = <STR_LIT> ; } else if ( isMatchingIncrement ( NextMBBI , Base , Bytes , Limit , Pred , PredReg ) ) { DoMerge = true ; } if ( DoMerge ) { NewOpc = getPostIndexedLoadStoreOpcode ( Opcode ) ; if ( NextMBBI == I ) { Advance = true ; ++ I ; } MBB . erase ( NextMBBI ) ; } } if ( ! DoMerge ) return false ; bool isDPR = NewOpc == <STR_LIT> || NewOpc == <STR_LIT> ; unsigned Offset = <NUM_LIT> ; if ( isAM5 ) Offset = <STR_LIT> ( AddSub == <STR_LIT> ? <STR_LIT> : <STR_LIT> , true , ( isDPR ? <NUM_LIT> : <NUM_LIT> ) ) ; else if ( isAM2 ) Offset = <STR_LIT> ( AddSub , Bytes , <STR_LIT> ) ; else Offset = AddSub == <STR_LIT> ? - Bytes : Bytes ; if ( isAM5 ) { MachineOperand & MO = MI -> getOperand ( <NUM_LIT> ) ; BuildMI ( MBB , MBBI , dl , TII -> get ( NewOpc ) ) . addReg ( Base , getDefRegState ( true ) ) . addReg ( Base , getKillRegState ( isLd ? BaseKill : false ) ) . addImm ( Offset ) . addImm ( Pred ) . addReg ( PredReg ) . addReg ( MO . getReg ( ) , ( isLd ? getDefRegState ( true ) : getKillRegState ( MO . isKill ( ) ) ) ) ; } else if ( isLd ) { if ( isAM2 ) BuildMI ( MBB , MBBI , dl , TII -> get ( NewOpc ) , MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) . addReg ( Base , RegState :: Define ) . addReg ( Base ) . addReg ( <NUM_LIT> ) . addImm ( Offset ) . addImm ( Pred ) . addReg ( PredReg ) ; else BuildMI ( MBB , MBBI , dl , TII -> get ( NewOpc ) , MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) . addReg ( Base , RegState :: Define ) . addReg ( Base ) . addImm ( Offset ) . addImm ( Pred ) . addReg ( PredReg ) ;


<s> bool <ISA_LIT>LoadStoreOpt :: MergeBaseUpdateLoadStore ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator MBBI , const TargetInstrInfo * TII , bool & Advance , MachineBasicBlock :: iterator & I ) { MachineInstr * MI = MBBI ; unsigned Base = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool BaseKill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; unsigned Bytes = getLSMultipleTransferSize ( MI ) ; int Opcode = MI -> getOpcode ( ) ; DebugLoc dl = MI -> getDebugLoc ( ) ; bool isAM5 = ( Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) ; bool isAM2 = ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) ; if ( isAM2 && <STR_LIT> ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) != <NUM_LIT> ) return false ; if ( isAM5 && <STR_LIT> ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) != <NUM_LIT> ) return false ; if ( isT2i32Load ( Opcode ) || isT2i32Store ( Opcode ) ) if ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ) return false ; bool isLd = isi32Load ( Opcode ) || Opcode == <STR_LIT> || Opcode == <STR_LIT> ; if ( isLd && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == Base ) return false ; unsigned PredReg = <NUM_LIT> ; <STR_LIT> Pred = llvm :: getInstrPredicate ( MI , PredReg ) ; bool DoMerge = false ; <STR_LIT> AddSub = <STR_LIT> ; unsigned NewOpc = <NUM_LIT> ; unsigned Limit = isAM5 ? <NUM_LIT> : ( isAM2 ? <NUM_LIT> : <NUM_LIT> ) ; if ( MBBI != MBB . begin ( ) ) { MachineBasicBlock :: iterator PrevMBBI = prior ( MBBI ) ; if ( isMatchingDecrement ( PrevMBBI , Base , Bytes , Limit , Pred , PredReg ) ) { DoMerge = true ; AddSub = <STR_LIT> ; } else if ( ! isAM5 && isMatchingIncrement ( PrevMBBI , Base , Bytes , Limit , Pred , PredReg ) ) { DoMerge = true ; } if ( DoMerge ) { NewOpc = getPreIndexedLoadStoreOpcode ( Opcode ) ; MBB . erase ( PrevMBBI ) ; } } if ( ! DoMerge && MBBI != MBB . end ( ) ) { MachineBasicBlock :: iterator NextMBBI = llvm :: next ( MBBI ) ; if ( ! isAM5 && isMatchingDecrement ( NextMBBI , Base , Bytes , Limit , Pred , PredReg ) ) { DoMerge = true ; AddSub = <STR_LIT> ; } else if ( isMatchingIncrement ( NextMBBI , Base , Bytes , Limit , Pred , PredReg ) ) { DoMerge = true ; } if ( DoMerge ) {

NewOpc = getPostIndexedLoadStoreOpcode ( Opcode ) ;


<s> MachineInstr * MI = MBBI ; unsigned Base = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool BaseKill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; unsigned Bytes = getLSMultipleTransferSize ( MI ) ; unsigned PredReg = <NUM_LIT> ; <STR_LIT> Pred = llvm :: getInstrPredicate ( MI , PredReg ) ; int Opcode = MI -> getOpcode ( ) ; DebugLoc dl = MI -> getDebugLoc ( ) ; bool isAM4 = ( Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) ; bool DoMerge = false ; <STR_LIT> Mode = <STR_LIT> ; unsigned Offset = <NUM_LIT> ; if ( isAM4 ) { for ( unsigned i = <NUM_LIT> , e = MI -> getNumOperands ( ) ; i != e ; ++ i ) { if ( MI -> getOperand ( i ) . getReg ( ) == Base ) return false ; } assert ( ! <STR_LIT> ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ) ; Mode = <STR_LIT> ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ; } else { assert ( ! <STR_LIT> ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ) ; Mode = <STR_LIT> ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ; Offset = <STR_LIT> ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ; } if ( MBBI != MBB . begin ( ) ) { MachineBasicBlock :: iterator PrevMBBI = prior ( MBBI ) ; if ( isAM4 ) { if ( Mode == <STR_LIT> && isMatchingDecrement ( PrevMBBI , Base , Bytes , <NUM_LIT> , Pred , PredReg ) ) { DoMerge = true ; Mode = <STR_LIT> ; } else if ( isAM4 && Mode == <STR_LIT> && isMatchingDecrement ( PrevMBBI , Base , Bytes , <NUM_LIT> , Pred , PredReg ) ) { DoMerge = true ; Mode = <STR_LIT> ; } } else { if ( Mode == <STR_LIT> && isMatchingDecrement ( PrevMBBI , Base , Bytes , <NUM_LIT> , Pred , PredReg ) ) { Mode = <STR_LIT> ; DoMerge = true ; } } if ( DoMerge ) MBB . erase ( PrevMBBI ) ; } if ( ! DoMerge && MBBI != MBB . end ( ) ) { MachineBasicBlock :: iterator NextMBBI = llvm :: next ( MBBI ) ; if ( isAM4 ) { if ( ( Mode == <STR_LIT> || Mode == <STR_LIT> ) && isMatchingIncrement ( NextMBBI , Base , Bytes , <NUM_LIT> , Pred , PredReg ) ) { DoMerge = true ; } else if ( ( Mode == <STR_LIT> || Mode == <STR_LIT> ) && isMatchingDecrement ( NextMBBI , Base , Bytes , <NUM_LIT> , Pred , PredReg ) ) {


<s> MachineBasicBlock & MBB = * MI -> getParent ( ) ; MachineBasicBlock :: iterator MBBI ( MI ) ; int Offset ; MachineBasicBlock :: iterator MergeInstr = findIncDecBefore ( MBBI , Base , Pred , PredReg , Offset ) ; <STR_LIT> Mode = getLoadStoreMultipleSubMode ( Opcode ) ; if ( Mode == <STR_LIT> && Offset == - Bytes ) { Mode = <STR_LIT> ; } else if ( Mode == <STR_LIT> && Offset == - Bytes ) { Mode = <STR_LIT> ; } else { MergeInstr = findIncDecAfter ( MBBI , Base , Pred , PredReg , Offset ) ; if ( ( ( Mode != <STR_LIT> && Mode != <STR_LIT> ) || Offset != Bytes ) && ( ( Mode != <STR_LIT> && Mode != <STR_LIT> ) || Offset != - Bytes ) ) { if ( ! STI -> optForMinSize ( ) || ! BaseKill ) return false ;


<s> NewOpc = Opcode == <STR_LIT> ? <STR_LIT> : <STR_LIT> ; } else { MergeInstr = findIncDecAfter ( MBBI , Base , Pred , PredReg , Offset ) ; if ( Offset == <NUM_LIT> || Offset == - <NUM_LIT> ) { NewOpc = Opcode == <STR_LIT> ? <STR_LIT> : <STR_LIT> ; } else return false ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MergeInstr ) ; MBB . erase ( MergeInstr ) ; DebugLoc DL = MI . getDebugLoc ( ) ; MachineInstrBuilder MIB = BuildMI ( MBB , MBBI , DL , TII -> get ( NewOpc ) ) ; if ( NewOpc == <STR_LIT> || NewOpc == <STR_LIT> ) { MIB . add ( Reg0Op ) . add ( Reg1Op ) . addReg ( BaseOp . getReg ( ) , RegState :: Define ) ; } else { assert ( NewOpc == <STR_LIT> || NewOpc == <STR_LIT> ) ;

MIB . addReg ( BaseOp . getReg ( ) , RegState :: Define ) . add ( Reg0Op ) . add ( Reg1Op ) ;


<s> bool <ISA_LIT>LoadStoreOpt :: MergeBaseUpdateLSMultiple ( MachineInstr * MI ) { if ( isThumb1 ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; const MachineOperand & BaseOP = MI -> getOperand ( <NUM_LIT> ) ; Register Base = BaseOP . getReg ( ) ; bool BaseKill = BaseOP . isKill ( ) ; Register PredReg ; <STR_LIT> Pred = getInstrPredicate ( * MI , PredReg ) ; unsigned Opcode = MI -> getOpcode ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; for ( unsigned i = <NUM_LIT> , e = MI -> getNumOperands ( ) ; i != e ; ++ i ) if ( MI -> getOperand ( i ) . getReg ( ) == Base ) return false ; int Bytes = getLSMultipleTransferSize ( MI ) ; MachineBasicBlock & MBB = * MI -> getParent ( ) ; MachineBasicBlock :: iterator MBBI ( MI ) ; int Offset ; MachineBasicBlock :: iterator MergeInstr = findIncDecBefore ( MBBI , Base , Pred , PredReg , Offset ) ; <STR_LIT> Mode = getLoadStoreMultipleSubMode ( Opcode ) ; if ( Mode == <STR_LIT> && Offset == - Bytes ) { Mode = <STR_LIT> ; } else if ( Mode == <STR_LIT> && Offset == - Bytes ) { Mode = <STR_LIT> ; } else { MergeInstr = findIncDecAfter ( MBBI , Base , Pred , PredReg , Offset ) ; if ( ( ( Mode != <STR_LIT> && Mode != <STR_LIT> ) || Offset != Bytes ) && ( ( Mode != <STR_LIT> && Mode != <STR_LIT> ) || Offset != - Bytes ) ) { if ( ! STI -> hasMinSize ( ) || ! BaseKill ) return false ; bool HighRegsUsed = false ; for ( unsigned i = <NUM_LIT> , e = MI -> getNumOperands ( ) ; i != e ; ++ i ) if ( MI -> getOperand ( i ) . getReg ( ) >= <STR_LIT> ) { HighRegsUsed = true ; break ; }

if ( ! HighRegsUsed ) MergeInstr = MBB . end ( ) ;


<s> DebugLoc DL = MI -> getDebugLoc ( ) ; for ( unsigned i = <NUM_LIT> , e = MI -> getNumOperands ( ) ; i != e ; ++ i ) if ( MI -> getOperand ( i ) . getReg ( ) == Base ) return false ; int Bytes = getLSMultipleTransferSize ( MI ) ; MachineBasicBlock & MBB = * MI -> getParent ( ) ; MachineBasicBlock :: iterator MBBI ( MI ) ; int Offset ; MachineBasicBlock :: iterator MergeInstr = findIncDecBefore ( MBBI , Base , Pred , PredReg , Offset ) ; <STR_LIT> Mode = getLoadStoreMultipleSubMode ( Opcode ) ; if ( Mode == <STR_LIT> && Offset == - Bytes ) { Mode = <STR_LIT> ; } else if ( Mode == <STR_LIT> && Offset == - Bytes ) { Mode = <STR_LIT> ; } else { MergeInstr = findIncDecAfter ( MBBI , Base , Pred , PredReg , Offset ) ; if ( ( ( Mode != <STR_LIT> && Mode != <STR_LIT> ) || Offset != Bytes ) && ( ( Mode != <STR_LIT> && Mode != <STR_LIT> ) || Offset != - Bytes ) ) { if ( ! STI -> hasMinSize ( ) || ! BaseKill ) return false ; bool HighRegsUsed = false ; for ( unsigned i = <NUM_LIT> , e = MI -> getNumOperands ( ) ; i != e ; ++ i ) if ( MI -> getOperand ( i ) . getReg ( ) >= <STR_LIT> ) { HighRegsUsed = true ; break ; } if ( ! HighRegsUsed ) MergeInstr = MBB . end ( ) ; else return false ; } } if ( MergeInstr != MBB . end ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MergeInstr ) ;


<s> if ( IsLoad || IsStore ) { MachineOperand & MO = MBBI -> getOperand ( MBBI -> getDesc ( ) . getNumOperands ( ) - <NUM_LIT> ) ; Offset = MO . getImm ( ) - WordOffset * getImmScale ( Opc ) ; unsigned InstrSrcReg = getLoadStoreRegOp ( * MBBI ) . getReg ( ) ; if ( Offset >= <NUM_LIT> && ! ( IsStore && InstrSrcReg == Base ) ) MO . setImm ( Offset ) ; else InsertSub = true ; } else if ( ( Opc == <STR_LIT> || Opc == <STR_LIT> ) && ! definesCPSR ( MBBI ) ) { MachineOperand & MO = MBBI -> getOperand ( MBBI -> getDesc ( ) . getNumOperands ( ) - <NUM_LIT> ) ; Offset = ( Opc == <STR_LIT> ) ? MO . getImm ( ) + WordOffset * <NUM_LIT> : MO . getImm ( ) - WordOffset * <NUM_LIT> ; if ( Offset >= <NUM_LIT> && TL -> isLegalAddImmediate ( Offset ) ) { MO . setImm ( Offset ) ; return ; } else { InsertSub = true ; } } else { InsertSub = true ; } } else if ( definesCPSR ( MBBI ) || MBBI -> isCall ( ) || MBBI -> isBranch ( ) ) { InsertSub = true ; } if ( InsertSub ) { AddDefaultT1CC ( BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , Base ) , true ) . addReg ( Base ) . addImm ( WordOffset * <NUM_LIT> ) . addImm ( Pred ) . addReg ( PredReg ) ; return ; } if ( MBBI -> killsRegister ( Base ) || MBBI -> definesRegister ( Base ) ) return ; } if ( MBB . succ_size ( ) > <NUM_LIT> ) { if ( MBBI != MBB . end ( ) ) -- MBBI ; AddDefaultT1CC ( BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , Base ) , true ) . addReg ( Base ) . addImm ( WordOffset * <NUM_LIT> ) . addImm ( Pred ) . addReg ( PredReg ) ;


<s> if ( Base != NewBase && Offset >= <NUM_LIT> ) { BuildMI ( MBB , MBBI , dl , TII -> get ( <STR_LIT> ) , NewBase ) . addReg ( Base , getKillRegState ( BaseKill ) ) . addImm ( Pred ) . addReg ( PredReg ) ; Base = NewBase ; BaseKill = false ; } AddDefaultT1CC ( BuildMI ( MBB , MBBI , dl , TII -> get ( BaseOpc ) , NewBase ) ) . addReg ( Base , getKillRegState ( BaseKill ) ) . addImm ( Offset ) . addImm ( Pred ) . addReg ( PredReg ) ; } else { BuildMI ( MBB , MBBI , dl , TII -> get ( BaseOpc ) , NewBase ) . addReg ( Base , getKillRegState ( BaseKill ) ) . addImm ( Offset ) . addImm ( Pred ) . addReg ( PredReg ) . addReg ( <NUM_LIT> ) ; } Base = NewBase ; BaseKill = true ; } bool isDef = ( isi32Load ( Opcode ) || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) ; Opcode = getLoadStoreMultipleOpcode ( Opcode , Mode ) ; if ( ! Opcode ) return false ; bool Writeback = isThumb1 ; if ( Opcode == <STR_LIT> && isThumb1 ) for ( unsigned I = <NUM_LIT> ; I < NumRegs ; ++ I ) if ( Base == Regs [ I ] . first ) { Writeback = false ; break ; } if ( isThumb1 && Writeback && ! BaseKill ) return false ; MachineInstrBuilder MIB ; if ( Writeback ) { if ( Opcode == <STR_LIT> ) Opcode = <STR_LIT> ; MIB = BuildMI ( MBB , MBBI , dl , TII -> get ( Opcode ) ) ; MIB . addReg ( Base , getDefRegState ( true ) ) . addReg ( Base , getKillRegState ( BaseKill ) ) ; } else { MIB = BuildMI ( MBB , MBBI , dl , TII -> get ( Opcode ) ) ; MIB . addReg ( Base , getKillRegState ( BaseKill ) ) ; } MIB . addImm ( Pred ) . addReg ( PredReg ) ; for ( unsigned i = <NUM_LIT> ; i != NumRegs ; ++ i ) MIB = MIB . addReg ( Regs [ i ] . first , getDefRegState ( isDef ) | getKillRegState ( Regs [ i ] . second ) ) ; for ( unsigned i = <NUM_LIT> , e = ImpDefs . size ( ) ; i != e ; ++ i ) MIB . addReg ( ImpDefs [ i ] , RegState :: ImplicitDefine ) ;


<s> bool isNotVFP = isi32Load ( Opcode ) || isi32Store ( Opcode ) ; bool haveIBAndDA = isNotVFP && ! isThumb2 && ! isThumb1 ; if ( Offset == <NUM_LIT> && haveIBAndDA ) { Mode = <STR_LIT> ; } else if ( Offset == - <NUM_LIT> * ( int ) NumRegs + <NUM_LIT> && haveIBAndDA ) { Mode = <STR_LIT> ; } else if ( Offset == - <NUM_LIT> * ( int ) NumRegs && isNotVFP && ! isThumb1 ) { Mode = <STR_LIT> ; } else if ( Offset != <NUM_LIT> ) { if ( ! getLoadStoreMultipleOpcode ( Opcode , Mode ) ) return false ; if ( NumRegs <= <NUM_LIT> ) return false ; unsigned NewBase ; if ( isi32Load ( Opcode ) ) { NewBase = Regs [ NumRegs - <NUM_LIT> ] . first ; } else { NewBase = Scratch ; if ( NewBase == <NUM_LIT> ) return false ; } int BaseOpc = isThumb2 ? <STR_LIT> : ( isThumb1 && Offset < <NUM_LIT> ) ? <STR_LIT> : isThumb1 ? <STR_LIT> : <STR_LIT> ; if ( Offset < <NUM_LIT> ) { Offset = - Offset ; BaseOpc = isThumb2 ? <STR_LIT> : ( isThumb1 && Offset < <NUM_LIT> ) ? <STR_LIT> : isThumb1 ? <STR_LIT> : <STR_LIT> ; } if ( ! TL -> isLegalAddImmediate ( Offset ) ) return false ; if ( isThumb1 ) { if ( Base != NewBase && Offset >= <NUM_LIT> ) { BuildMI ( MBB , MBBI , dl , TII -> get ( <STR_LIT> ) , NewBase ) . addReg ( Base , getKillRegState ( BaseKill ) ) . addImm ( Pred ) . addReg ( PredReg ) ; Base = NewBase ; BaseKill = false ; } AddDefaultT1CC ( BuildMI ( MBB , MBBI , dl , TII -> get ( BaseOpc ) , NewBase ) ) . addReg ( Base , getKillRegState ( BaseKill ) ) . addImm ( Offset ) . addImm ( Pred ) . addReg ( PredReg ) ; } else { BuildMI ( MBB , MBBI , dl , TII -> get ( BaseOpc ) , NewBase ) . addReg ( Base , getKillRegState ( BaseKill ) ) . addImm ( Offset ) . addImm ( Pred ) . addReg ( PredReg ) . addReg ( <NUM_LIT> ) ; } Base = NewBase ; BaseKill = true ; } bool isDef = ( isi32Load ( Opcode ) || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) ; Opcode = getLoadStoreMultipleOpcode ( Opcode , Mode ) ; if ( ! Opcode ) return false ; bool Writeback = isThumb1 ; if ( Opcode == <STR_LIT> && isThumb1 ) for ( unsigned I = <NUM_LIT> ; I < NumRegs ; ++ I ) if ( Base == Regs [ I ] . first ) { Writeback = false ; break ; } if ( isThumb1 && Writeback && ! BaseKill ) return false ; MachineInstrBuilder MIB ; if ( Writeback ) { if ( Opcode == <STR_LIT> ) Opcode = <STR_LIT> ;

MIB = BuildMI ( MBB , MBBI , dl , TII -> get ( Opcode ) ) ;


<s> if ( ! ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == Base && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == Base && ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) * Scale ) == Bytes && llvm :: getInstrPredicate ( MI , MyPredReg ) == Pred && MyPredReg == PredReg ) ) return false ;

return CheckCPSRDef ? ! definesCPSR ( MI ) : true ;


<s> default : return false ; case <STR_LIT> : case <STR_LIT> : CheckCPSRDef = true ; case <STR_LIT> : break ; } if ( Bytes == <NUM_LIT> || ( Limit && Bytes >= Limit ) ) return false ; unsigned Scale = ( MI -> getOpcode ( ) == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; if ( ! ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == Base && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == Base && ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) * Scale ) == Bytes && llvm :: getInstrPredicate ( MI , MyPredReg ) == Pred && MyPredReg == PredReg ) ) return false ;


<s> unsigned Scale = ( MI -> getOpcode ( ) == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; if ( ! ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == Base && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == Base && ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) * Scale ) == Bytes && llvm :: getInstrPredicate ( MI , MyPredReg ) == Pred && MyPredReg == PredReg ) ) return false ; return CheckCPSRDef ? ! definesCPSR ( MI ) : true ;


<s> STI = & static_cast < const <ISA_LIT>Subtarget & > ( Fn . getSubtarget ( ) ) ; TII = STI -> getInstrInfo ( ) ; TRI = STI -> getRegisterInfo ( ) ; MRI = & Fn . getRegInfo ( ) ; MF = & Fn ; bool Modified = false ; for ( MachineBasicBlock & MFI : Fn ) Modified |= RescheduleLoadStoreInstrs ( & MFI ) ;


<s> MF = & Fn ; bool Modified = false ; for ( MachineBasicBlock & MFI : Fn ) Modified |= RescheduleLoadStoreInstrs ( & MFI ) ; return Modified ;


<s> MemOps . push_back ( MemOpQueueEntry ( MBBI , Offset , Position ) ) ; continue ; } else { MemOpQueue :: iterator MI , ME ; for ( MI = MemOps . begin ( ) , ME = MemOps . end ( ) ; MI != ME ; ++ MI ) { if ( Offset < MI -> Offset ) { break ; } if ( Offset == MI -> Offset ) { MI = ME ; break ; } } if ( MI != MemOps . end ( ) ) { MemOps . insert ( MI , MemOpQueueEntry ( MBBI , Offset , Position ) ) ; continue ; } } } } MBBI = I ; -- Position ; } else if ( MBBI -> isDebugValue ( ) ) { continue ; } else if ( MBBI -> getOpcode ( ) == <STR_LIT> || MBBI -> getOpcode ( ) == <STR_LIT> ) { MergeBaseCandidates . push_back ( MBBI ) ; } if ( MemOps . size ( ) > <NUM_LIT> ) { FormCandidates ( MemOps ) ; CurrBase = <NUM_LIT> ; CurrOpc = ~ <NUM_LIT> ; CurrPred = <STR_LIT> ; MemOps . clear ( ) ; } } if ( MemOps . size ( ) > <NUM_LIT> ) FormCandidates ( MemOps ) ; auto LessThan = [ ] ( const MergeCandidate * M0 , const MergeCandidate * M1 ) { return M0 -> InsertPos < M1 -> InsertPos ; } ; std :: sort ( Candidates . begin ( ) , Candidates . end ( ) , LessThan ) ; bool Changed = false ; for ( const MergeCandidate * Candidate : Candidates ) {

if ( Candidate -> CanMergeToLSMulti || Candidate -> CanMergeToLSDouble ) {


<s> if ( MBBI != MBB . begin ( ) && ( MBBI -> getOpcode ( ) == <STR_LIT> || MBBI -> getOpcode ( ) == <STR_LIT> || MBBI -> getOpcode ( ) == <STR_LIT> ) ) { MachineInstr * PrevMI = std :: prev ( MBBI ) ; unsigned Opcode = PrevMI -> getOpcode ( ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { MachineOperand & MO = PrevMI -> getOperand ( PrevMI -> getNumOperands ( ) - <NUM_LIT> ) ;

if ( MO . getReg ( ) != <STR_LIT> ) return false ;


<s> DenseMap < unsigned , SmallVector < MachineInstr * , <NUM_LIT> > > :: iterator BI = Base2LdsMap . find ( Base ) ; if ( BI != Base2LdsMap . end ( ) ) { for ( unsigned i = <NUM_LIT> , e = BI -> second . size ( ) ; i != e ; ++ i ) { if ( Offset == getMemoryOpOffset ( BI -> second [ i ] ) ) { StopHere = true ; break ; } } if ( ! StopHere ) BI -> second . push_back ( MI ) ; } else { Base2LdsMap [ Base ] . push_back ( MI ) ; LdBases . push_back ( Base ) ; } } else { DenseMap < unsigned , SmallVector < MachineInstr * , <NUM_LIT> > > :: iterator BI = Base2StsMap . find ( Base ) ; if ( BI != Base2StsMap . end ( ) ) { for ( unsigned i = <NUM_LIT> , e = BI -> second . size ( ) ; i != e ; ++ i ) { if ( Offset == getMemoryOpOffset ( BI -> second [ i ] ) ) { StopHere = true ; break ; } } if ( ! StopHere ) BI -> second . push_back ( MI ) ; } else { Base2StsMap [ Base ] . push_back ( MI ) ; StBases . push_back ( Base ) ; } } if ( StopHere ) { -- Loc ; break ; } } for ( unsigned i = <NUM_LIT> , e = LdBases . size ( ) ; i != e ; ++ i ) { unsigned Base = LdBases [ i ] ; SmallVectorImpl < MachineInstr * > & Lds = Base2LdsMap [ Base ] ; if ( Lds . size ( ) > <NUM_LIT> ) RetVal |= RescheduleOps ( MBB , Lds , Base , true , MI2LocMap ) ; } for ( unsigned i = <NUM_LIT> , e = StBases . size ( ) ; i != e ; ++ i ) { unsigned Base = StBases [ i ] ; SmallVectorImpl < MachineInstr * > & Sts = Base2StsMap [ Base ] ; if ( Sts . size ( ) > <NUM_LIT> ) RetVal |= RescheduleOps ( MBB , Sts , Base , false , MI2LocMap ) ; } if ( MBBI != E ) {


<s> } } if ( ! StopHere ) BI -> second . push_back ( MI ) ; } else { Base2StsMap [ Base ] . push_back ( MI ) ; StBases . push_back ( Base ) ; } } if ( StopHere ) { -- Loc ; break ; } } for ( unsigned i = <NUM_LIT> , e = LdBases . size ( ) ; i != e ; ++ i ) { unsigned Base = LdBases [ i ] ; SmallVectorImpl < MachineInstr * > & Lds = Base2LdsMap [ Base ] ; if ( Lds . size ( ) > <NUM_LIT> ) RetVal |= RescheduleOps ( MBB , Lds , Base , true , MI2LocMap ) ; } for ( unsigned i = <NUM_LIT> , e = StBases . size ( ) ; i != e ; ++ i ) { unsigned Base = StBases [ i ] ; SmallVectorImpl < MachineInstr * > & Sts = Base2StsMap [ Base ] ; if ( Sts . size ( ) > <NUM_LIT> ) RetVal |= RescheduleOps ( MBB , Sts , Base , false , MI2LocMap ) ; } if ( MBBI != E ) { Base2LdsMap . clear ( ) ; Base2StsMap . clear ( ) ; LdBases . clear ( ) ; StBases . clear ( ) ; } } return RetVal ;


<s> bool <ISA_LIT>PreAllocLoadStoreOpt :: runOnMachineFunction ( MachineFunction & Fn ) { TD = & Fn . getDataLayout ( ) ; STI = & static_cast < const <ISA_LIT>Subtarget & > ( Fn . getSubtarget ( ) ) ; TII = STI -> getInstrInfo ( ) ; TRI = STI -> getRegisterInfo ( ) ; MRI = & Fn . getRegInfo ( ) ;


<s> void addInst ( MachineInstr * MI , SetVector < MachineInstr * > & Preds ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; if ( ! Divergent && ! set_difference ( Preds , PredicateThen -> Predicates ) . empty ( ) ) { Divergent = & Insts . back ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Divergent -> MI ) ; }


<s> for ( auto I = ++ MachineBasicBlock :: iterator ( Block . getPredicateThen ( ) ) , E = ++ MachineBasicBlock :: iterator ( Divergent -> MI ) ; I != E ; ++ I ) RemovePredicate ( & * I ) ; unsigned Size = <NUM_LIT> ; auto E = MachineBasicBlock :: reverse_iterator ( Divergent -> MI ) ; auto I = MachineBasicBlock :: reverse_iterator ( Insts . back ( ) . MI ) ; MachineInstr * InsertAt = nullptr ; while ( I != E ) { InsertAt = & * I ; ++ Size ; ++ I ; } MachineInstrBuilder MIB = BuildMI ( * InsertAt -> getParent ( ) , InsertAt , InsertAt -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . addImm ( <NUM_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Block . getPredicateThen ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; LoLoop . ToRemove . insert ( Block . getPredicateThen ( ) ) ; LoLoop . BlockMasksToRecompute . insert ( MIB . getInstr ( ) ) ; } else if ( Block . isVPT ( ) ) { bool RemovedVCTP = false ; for ( PredicatedMI & Elt : Block . getInsts ( ) ) { MachineInstr * MI = Elt . MI ; if ( isVCTP ( MI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; LoLoop . ToRemove . insert ( MI ) ; RemovedVCTP = true ; continue ; } } if ( RemovedVCTP ) LoLoop . BlockMasksToRecompute . insert ( Block . getPredicateThen ( ) ) ; } } else if ( Block . IsOnlyPredicatedOn ( LoLoop . VCTP ) && Block . isVPST ( ) ) {


<s> if ( ! FoundAllComponents ( ) ) dbgs ( ) << <STR_LIT> ; else if ( ! ( Start && Dec && End ) ) dbgs ( ) << <STR_LIT> ;


<s> if ( Dec ) dbgs ( ) << <STR_LIT> << * Dec ; if ( End ) dbgs ( ) << <STR_LIT> << * End ; if ( VCTP ) dbgs ( ) << <STR_LIT> << * VCTP ;

if ( ! FoundAllComponents ( ) ) dbgs ( ) << <STR_LIT> ;


<s> LoLoop . ToRemove . insert ( End ) ; return & * MIB ; } ; auto RemoveDeadBranch = [ ] ( MachineInstr * I ) { MachineBasicBlock * BB = I -> getParent ( ) ; MachineInstr * Terminator = & BB -> instr_back ( ) ; if ( Terminator -> isUnconditionalBranch ( ) && I != Terminator ) { MachineBasicBlock * Succ = Terminator -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( BB -> isLayoutSuccessor ( Succ ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Terminator ) ; Terminator -> eraseFromParent ( ) ; } } } ; if ( LoLoop . Revert ) { if ( LoLoop . Start -> getOpcode ( ) == <STR_LIT> ) RevertWhile ( LoLoop . Start ) ; else LoLoop . Start -> eraseFromParent ( ) ;

bool FlagsAlreadySet = RevertLoopDec ( LoLoop . Dec ) ;


<s> MachineInstr * Start = LoLoop . Start ; MachineBasicBlock * MBB = InsertPt -> getParent ( ) ; bool IsDo = Start -> getOpcode ( ) == <STR_LIT> ; unsigned Opc = LoLoop . getStartOpcode ( ) ; MachineOperand & Count = LoLoop . getCount ( ) ; MachineInstrBuilder MIB = BuildMI ( * MBB , InsertPt , InsertPt -> getDebugLoc ( ) , TII -> get ( Opc ) ) ; MIB . addDef ( <STR_LIT> ) ;


<s> MachineInstr * <ISA_LIT>LowOverheadLoops :: ExpandLoopStart ( LowOverheadLoop & LoLoop ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; IterationCountDCE ( LoLoop ) ; MachineInstr * InsertPt = LoLoop . InsertPt ; MachineInstr * Start = LoLoop . Start ; MachineBasicBlock * MBB = InsertPt -> getParent ( ) ; bool IsDo = Start -> getOpcode ( ) == <STR_LIT> ; unsigned Opc = LoLoop . getStartOpcode ( ) ; MachineOperand & Count = LoLoop . getCount ( ) ; MachineInstrBuilder MIB = BuildMI ( * MBB , InsertPt , InsertPt -> getDebugLoc ( ) , TII -> get ( Opc ) ) ; MIB . addDef ( <STR_LIT> ) ; MIB . add ( Count ) ;

if ( ! IsDo ) MIB . add ( Start -> getOperand ( <NUM_LIT> ) ) ;


<s> } Init -> getOperand ( <NUM_LIT> ) . setReg ( DestReg ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Init ) ; } else BuildMov ( LoLoop . Preheader -> instr_back ( ) , DestReg , Copy . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Reduce . getOperand ( <NUM_LIT> ) . setReg ( DestReg ) ; Reduce . getOperand ( <NUM_LIT> ) . setReg ( DestReg ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Reduce ) ; MachineInstr & VPSEL = Reduction -> VPSEL ; if ( VPSEL . getOperand ( <NUM_LIT> ) . getReg ( ) != DestReg ) BuildMov ( VPSEL , VPSEL . getOperand ( <NUM_LIT> ) . getReg ( ) , DestReg ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << Copy << <STR_LIT> << VPSEL << <STR_LIT> ) ; Copy . eraseFromParent ( ) ;


<s> MachineInstr & Reduce = Reduction -> Reduce ; Register DestReg = Copy . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( Reduction -> Init ) { MachineInstr * Init = Reduction -> Init ; for ( unsigned i = <NUM_LIT> ; i < Init -> getNumOperands ( ) ; ++ i ) { MachineOperand & MO = Init -> getOperand ( i ) ; if ( MO . isReg ( ) && MO . isUse ( ) && MO . isTied ( ) && Init -> findTiedOperandIdx ( i ) == <NUM_LIT> ) Init -> getOperand ( i ) . setReg ( DestReg ) ; } Init -> getOperand ( <NUM_LIT> ) . setReg ( DestReg ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Init ) ; } else BuildMov ( LoLoop . Preheader -> instr_back ( ) , DestReg , Copy . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Reduce . getOperand ( <NUM_LIT> ) . setReg ( DestReg ) ; Reduce . getOperand ( <NUM_LIT> ) . setReg ( DestReg ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Reduce ) ; MachineInstr & VPSEL = Reduction -> VPSEL ; if ( VPSEL . getOperand ( <NUM_LIT> ) . getReg ( ) != DestReg ) BuildMov ( VPSEL , VPSEL . getOperand ( <NUM_LIT> ) . getReg ( ) , DestReg ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << Copy << <STR_LIT> << VPSEL << <STR_LIT> ) ; Copy . eraseFromParent ( ) ;


<s> return IsTailPredicationLegal ( ) ? VCTP -> getOperand ( <NUM_LIT> ) : Start -> getOperand ( <NUM_LIT> ) ;


<s> MachineOperand & getCount ( ) { return IsTailPredicationLegal ( ) ? VCTP -> getOperand ( <NUM_LIT> ) : Start -> getOperand ( <NUM_LIT> ) ;


<s> SmallVectorImpl < PredicatedMI > & getInsts ( ) { return Insts ;


<s> SmallVectorImpl < PredicatedMI > & getInsts ( ) {


<s> bool IsDo = Start -> getOpcode ( ) == <STR_LIT> ; if ( ! IsTailPredicationLegal ( ) ) return IsDo ? <STR_LIT> : <STR_LIT> ; return VCTPOpcodeToLSTP ( VCTP -> getOpcode ( ) , IsDo ) ;


<s> if ( ! IsTailPredicationLegal ( ) ) return IsDo ? <STR_LIT> : <STR_LIT> ;

return VCTPOpcodeToLSTP ( VCTP -> getOpcode ( ) , IsDo ) ;


<s> SmallVectorImpl < VPTBlock > & getVPTBlocks ( ) { return VPTBlocks ;


<s> bool HasNonUniformPredicate ( ) const { return Divergent != nullptr ;


<s> return IsPredicatedOn ( MI ) && PredicateThen -> Predicates . size ( ) == <NUM_LIT> ;


<s> unsigned CountReg = Start -> getOperand ( <NUM_LIT> ) . getReg ( ) ; auto IsMoveLR = [ & CountReg ] ( MachineInstr * MI ) { return MI -> getOpcode ( ) == <STR_LIT> && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == CountReg && MI -> getOperand ( <NUM_LIT> ) . getImm ( ) == <STR_LIT> ; } ; MachineBasicBlock * MBB = Start -> getParent ( ) ; if ( auto * LRDef = RDA . getUniqueReachingMIDef ( Start , <STR_LIT> ) ) if ( IsMoveLR ( LRDef ) && RDA . hasSameReachingDef ( Start , LRDef , CountReg ) ) return LRDef ; if ( auto * LRDef = RDA . getLocalLiveOutMIDef ( MBB , <STR_LIT> ) ) if ( IsMoveLR ( LRDef ) && RDA . hasSameReachingDef ( Start , LRDef , CountReg ) ) return LRDef ; return RDA . isSafeToDefRegAt ( Start , <STR_LIT> ) ? Start : nullptr ;


<s> bool IsTailPredicationLegal ( ) const {

return ! Revert && FoundAllComponents ( ) && VCTP && ! CannotTailPredicate && ML . getNumBlocks ( ) == <NUM_LIT> ;


return PredicateThen -> MI -> getOpcode ( ) == <STR_LIT> ;


<s> return PredicateThen -> MI -> getOpcode ( ) == <STR_LIT> ;


<s> bool isVPT ( ) const { return ! isVPST ( ) ;


<s> return ; } RDA -> collectKilledOperands ( Def , Killed ) ; SmallPtrSet < MachineBasicBlock * , <NUM_LIT> > BasicBlocks ; for ( auto * MI : Killed ) BasicBlocks . insert ( MI -> getParent ( ) ) ; std :: map < MachineInstr * , SmallPtrSet < MachineInstr * , <NUM_LIT> >> ITBlocks ; for ( auto * MBB : BasicBlocks ) { for ( auto & MI : * MBB ) { if ( MI . getOpcode ( ) != <STR_LIT> ) continue ; RDA -> getReachingLocalUses ( & MI , <STR_LIT> , ITBlocks [ & MI ] ) ; } } SmallPtrSet < MachineInstr * , <NUM_LIT> > ModifiedITs ; for ( auto * MI : Killed ) { if ( MachineOperand * MO = MI -> findRegisterUseOperand ( <STR_LIT> ) ) { MachineInstr * IT = RDA -> getMIOperand ( MI , * MO ) ; auto & CurrentBlock = ITBlocks [ IT ] ; CurrentBlock . erase ( MI ) ;

if ( CurrentBlock . empty ( ) ) ModifiedITs . erase ( IT ) ;


<s> for ( auto * MI : Killed ) BasicBlocks . insert ( MI -> getParent ( ) ) ; std :: map < MachineInstr * , SmallPtrSet < MachineInstr * , <NUM_LIT> >> ITBlocks ; for ( auto * MBB : BasicBlocks ) { for ( auto & MI : * MBB ) { if ( MI . getOpcode ( ) != <STR_LIT> ) continue ; RDA -> getReachingLocalUses ( & MI , <STR_LIT> , ITBlocks [ & MI ] ) ; } } SmallPtrSet < MachineInstr * , <NUM_LIT> > ModifiedITs ; for ( auto * MI : Killed ) { if ( MachineOperand * MO = MI -> findRegisterUseOperand ( <STR_LIT> ) ) { MachineInstr * IT = RDA -> getMIOperand ( MI , * MO ) ; auto & CurrentBlock = ITBlocks [ IT ] ; CurrentBlock . erase ( MI ) ; if ( CurrentBlock . empty ( ) ) ModifiedITs . erase ( IT ) ; else ModifiedITs . insert ( IT ) ; } } if ( ModifiedITs . empty ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; for ( auto * MI : Killed ) dbgs ( ) << <STR_LIT> << * MI ) ;

LoLoop . ToRemove . insert ( Killed . begin ( ) , Killed . end ( ) ) ;


<s> LowOverheadLoop ( MachineLoop & ML , MachineLoopInfo & MLI , ReachingDefAnalysis & RDA , const TargetRegisterInfo & TRI , const <ISA_LIT>BaseInstrInfo & TII ) : ML ( ML ) , MLI ( MLI ) , RDA ( RDA ) , TRI ( TRI ) , TII ( TII ) { MF = ML . getHeader ( ) -> getParent ( ) ; if ( auto * MBB = ML . getLoopPreheader ( ) ) Preheader = MBB ; else if ( auto * MBB = MLI . findLoopPreheader ( & ML , true ) ) Preheader = MBB ;


<s> if ( LoLoop . Preheader ) LoLoop . Start = SearchForStart ( LoLoop . Preheader ) ; else return false ; for ( auto * MBB : reverse ( ML -> getBlocks ( ) ) ) { for ( auto & MI : * MBB ) { if ( MI . isDebugValue ( ) ) continue ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . Dec = & MI ; else if ( MI . getOpcode ( ) == <STR_LIT> ) LoLoop . End = & MI ; else if ( isLoopStart ( MI ) ) LoLoop . Start = & MI ; else if ( MI . getDesc ( ) . isCall ( ) ) { LoLoop . Revert = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } else { LoLoop . AnalyseMVEInst ( & MI ) ; } } } LLVM_DEBUG ( LoLoop . dump ( ) ) ;

if ( ! LoLoop . FoundAllComponents ( ) ) {


<s> if ( ! isRegInClass ( MO , QPRs ) && AllowScalars ) continue ; if ( auto * OpDef = RDA . getMIOperand ( & MI , MO ) ) if ( FalseLanesZero . count ( OpDef ) ) continue ;


<s> Reduction ( MachineInstr * Init , MachineInstr * Mov , MachineInstr * Add , MachineInstr * Sel ) : Init ( Init ) , Copy ( * Mov ) , Reduce ( * Add ) , VPSEL ( * Sel ) {


<s> SmallPtrSet < MachineInstr * , <NUM_LIT> > Ignore ; for ( auto I = MachineBasicBlock :: iterator ( MI ) , E = MBB -> end ( ) ; I != E ; ++ I ) { if ( I -> getOpcode ( ) == <STR_LIT> ) { Ignore . insert ( & * I ) ; break ; } } bool SetFlags = RDA -> isSafeToDefRegAt ( MI , <STR_LIT> , Ignore ) ;

MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ;


<s> MIB . addImm ( <NUM_LIT> ) ; MIB . addImm ( <STR_LIT> ) ; MIB . addReg ( <STR_LIT> ) ; } MachineBasicBlock * DestBB = MI -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; unsigned BrOpc = BBUtils -> isBBInRange ( MI , DestBB , <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ;

MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( BrOpc ) ) ;


<s> SmallVector < MachineInstr * , <NUM_LIT> > Decs ; SmallVector < MachineInstr * , <NUM_LIT> > Ends ; for ( auto & I : MBB ) { if ( isLoopStart ( I ) ) Starts . push_back ( & I ) ; else if ( I . getOpcode ( ) == <STR_LIT> ) Decs . push_back ( & I ) ; else if ( I . getOpcode ( ) == <STR_LIT> ) Ends . push_back ( & I ) ; } if ( Starts . empty ( ) && Decs . empty ( ) && Ends . empty ( ) ) continue ; Changed = true ; for ( auto * Start : Starts ) { if ( Start -> getOpcode ( ) == <STR_LIT> ) RevertWhile ( Start ) ; else Start -> eraseFromParent ( ) ; } for ( auto * Dec : Decs ) RevertLoopDec ( Dec ) ;

for ( auto * End : Ends ) RevertLoopEnd ( End ) ;


<s> void <ISA_LIT>LowOverheadLoops :: RevertWhile ( MachineInstr * MI ) const { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; MachineBasicBlock * MBB = MI -> getParent ( ) ; MachineInstrBuilder MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MIB . addImm ( <NUM_LIT> ) ; MIB . addImm ( <STR_LIT> ) ;


<s> MLI = & getAnalysis < MachineLoopInfo > ( ) ; RDA = & getAnalysis < ReachingDefAnalysis > ( ) ; MF -> getProperties ( ) . set ( MachineFunctionProperties :: Property :: TracksLiveness ) ; MRI = & MF -> getRegInfo ( ) ; TII = static_cast < const <ISA_LIT>BaseInstrInfo * > ( ST . getInstrInfo ( ) ) ; TRI = ST . getRegisterInfo ( ) ; BBUtils = std :: unique_ptr < <ISA_LIT>BasicBlockUtils > ( new <ISA_LIT>BasicBlockUtils ( * MF ) ) ; BBUtils -> computeAllBlockSizes ( ) ; BBUtils -> adjustBBOffsetsAfter ( & MF -> front ( ) ) ; bool Changed = false ;


<s> TRI = ST . getRegisterInfo ( ) ; BBUtils = std :: unique_ptr < <ISA_LIT>BasicBlockUtils > ( new <ISA_LIT>BasicBlockUtils ( * MF ) ) ; BBUtils -> computeAllBlockSizes ( ) ; BBUtils -> adjustBBOffsetsAfter ( & MF -> front ( ) ) ; bool Changed = false ; for ( auto ML : * MLI ) {

if ( ! ML -> getParentLoop ( ) ) Changed |= ProcessLoop ( ML ) ;


<s> uint64_t Flags = MCID . TSFlags ; if ( ( Flags & <STR_LIT> ) != <STR_LIT> ) continue ; if ( isVCTP ( & MI ) || isVPTOpcode ( MI . getOpcode ( ) ) ) continue ; if ( isVectorPredicated ( & MI ) ) { if ( MI . mayLoad ( ) ) FalseLanesZero . insert ( & MI ) ; Predicated . insert ( & MI ) ; continue ; } if ( MI . getNumDefs ( ) == <NUM_LIT> ) continue ; if ( ! producesFalseLanesZero ( MI , QPRs , RDA , FalseLanesZero ) ) { if ( retainsPreviousHalfElement ( MI ) || isHorizontalReduction ( MI ) ) return false ; FalseLanesUnknown . insert ( & MI ) ; } else if ( ! isHorizontalReduction ( MI ) ) FalseLanesZero . insert ( & MI ) ; } auto HasPredicatedUsers = [ this ] ( MachineInstr * MI , const MachineOperand & MO , SmallPtrSetImpl < MachineInstr * > & Predicated ) { SmallPtrSet < MachineInstr * , <NUM_LIT> > Uses ; RDA . getGlobalUses ( MI , MO . getReg ( ) , Uses ) ; for ( auto * Use : Uses ) { if ( Use != MI && ! Predicated . count ( Use ) ) return false ; } return true ; } ; SmallPtrSet < MachineInstr * , <NUM_LIT> > NonPredicated ; for ( auto * MI : reverse ( FalseLanesUnknown ) ) { for ( auto & MO : MI -> operands ( ) ) { if ( ! isRegInClass ( MO , QPRs ) || ! MO . isDef ( ) ) continue ; if ( ! HasPredicatedUsers ( MI , MO , Predicated ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << TRI . getRegAsmName ( MO . getReg ( ) ) << <STR_LIT> << * MI ) ; NonPredicated . insert ( MI ) ; continue ; } } Predicated . insert ( MI ) ; } SmallPtrSet < MachineInstr * , <NUM_LIT> > LiveOutMIs ; SmallPtrSet < MachineInstr * , <NUM_LIT> > LiveOutUsers ;

SmallVector < MachineBasicBlock * , <NUM_LIT> > ExitBlocks ;


<s> } else if ( isVPTOpcode ( MI -> getOpcode ( ) ) ) { if ( MI -> getOpcode ( ) != <STR_LIT> ) { assert ( MI -> findRegisterDefOperandIdx ( <STR_LIT> ) != - <NUM_LIT> && <STR_LIT> ) ; CurrentPredicate . insert ( MI ) ; } VPTBlocks . emplace_back ( MI , CurrentPredicate ) ; CurrentBlock = & VPTBlocks . back ( ) ; return true ; } else if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { return false ; } bool IsUse = false ; bool IsDef = false ; const MCInstrDesc & MCID = MI -> getDesc ( ) ; for ( int i = MI -> getNumOperands ( ) - <NUM_LIT> ; i >= <NUM_LIT> ; -- i ) { const MachineOperand & MO = MI -> getOperand ( i ) ; if ( ! MO . isReg ( ) || MO . getReg ( ) != <STR_LIT> ) continue ; if ( MO . isDef ( ) ) { CurrentPredicate . insert ( MI ) ; IsDef = true ; } else if ( <STR_LIT> ( MCID . OpInfo [ i ] . OperandType ) ) { CurrentBlock -> addInst ( MI , CurrentPredicate ) ; IsUse = true ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; return false ; } }

if ( IsDef && ! IsUse && VCTP && ! isVCTP ( MI ) ) {


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; SecondaryVCTPs . insert ( MI ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; VCTP = MI ; } } else if ( isVPTOpcode ( MI -> getOpcode ( ) ) ) { if ( MI -> getOpcode ( ) != <STR_LIT> ) { assert ( MI -> findRegisterDefOperandIdx ( <STR_LIT> ) != - <NUM_LIT> && <STR_LIT> ) ; CurrentPredicate . insert ( MI ) ; } VPTBlocks . emplace_back ( MI , CurrentPredicate ) ; CurrentBlock = & VPTBlocks . back ( ) ; return true ; } else if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { return false ; } bool IsUse = false ; bool IsDef = false ; const MCInstrDesc & MCID = MI -> getDesc ( ) ; for ( int i = MI -> getNumOperands ( ) - <NUM_LIT> ; i >= <NUM_LIT> ; -- i ) { const MachineOperand & MO = MI -> getOperand ( i ) ; if ( ! MO . isReg ( ) || MO . getReg ( ) != <STR_LIT> ) continue ; if ( MO . isDef ( ) ) { CurrentPredicate . insert ( MI ) ; IsDef = true ; } else if ( <STR_LIT> ( MCID . OpInfo [ i ] . OperandType ) ) {

CurrentBlock -> addInst ( MI , CurrentPredicate ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MachineBasicBlock * InsertBB = StartInsertPt -> getParent ( ) ; if ( ! RDA . isReachingDefLiveOut ( StartInsertPt , NumElements ) ) { if ( auto * ElemDef = RDA . getLocalLiveOutMIDef ( InsertBB , NumElements ) ) { if ( RDA . isSafeToMoveForwards ( ElemDef , StartInsertPt ) ) { ElemDef -> removeFromParent ( ) ; InsertBB -> insert ( MachineBasicBlock :: iterator ( StartInsertPt ) , ElemDef ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ElemDef ) ; } else if ( RDA . isSafeToMoveBackwards ( StartInsertPt , ElemDef ) ) { StartInsertPt -> removeFromParent ( ) ; InsertBB -> insertAfter ( MachineBasicBlock :: iterator ( ElemDef ) , StartInsertPt ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ElemDef ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return false ; } } } auto CannotProvideElements = [ this ] ( MachineBasicBlock * MBB , Register NumElements ) { if ( RDA . hasLocalDefBefore ( & MBB -> back ( ) , NumElements ) ) return true ; if ( MBB -> pred_size ( ) > <NUM_LIT> ) return true ; return false ; } ; MachineBasicBlock * MBB = Preheader ;


<s> SmallVectorImpl < PredicatedMI > & Insts = Block . getInsts ( ) ; for ( auto & PredMI : Insts ) { if ( PredMI . Predicates . count ( VCTP ) || isVCTP ( PredMI . MI ) || VCMPOpcodeToVPT ( PredMI . MI -> getOpcode ( ) ) != <NUM_LIT> ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * PredMI . MI << <STR_LIT> ; for ( auto * MI : PredMI . Predicates ) dbgs ( ) << <STR_LIT> << * MI ) ; return false ; } } if ( ! ValidateLiveOuts ( ) ) return false ; Register NumElements = VCTP -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( RDA . hasLocalDefBefore ( VCTP , NumElements ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MachineBasicBlock * InsertBB = StartInsertPt -> getParent ( ) ; if ( ! RDA . isReachingDefLiveOut ( StartInsertPt , NumElements ) ) { if ( auto * ElemDef = RDA . getLocalLiveOutMIDef ( InsertBB , NumElements ) ) { if ( RDA . isSafeToMoveForwards ( ElemDef , StartInsertPt ) ) { ElemDef -> removeFromParent ( ) ; InsertBB -> insert ( MachineBasicBlock :: iterator ( StartInsertPt ) , ElemDef ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ElemDef ) ; } else if ( RDA . isSafeToMoveBackwards ( StartInsertPt , ElemDef ) ) { StartInsertPt -> removeFromParent ( ) ; InsertBB -> insertAfter ( MachineBasicBlock :: iterator ( ElemDef ) , StartInsertPt ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * ElemDef ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return false ; } } } auto CannotProvideElements = [ this ] ( MachineBasicBlock * MBB , Register NumElements ) { if ( RDA . hasLocalDefBefore ( & MBB -> back ( ) , NumElements ) ) return true ; if ( MBB -> pred_size ( ) > <NUM_LIT> ) return true ; return false ; } ; MachineBasicBlock * MBB = Preheader ; if ( ! MBB ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } while ( MBB != InsertBB ) { if ( CannotProvideElements ( MBB , NumElements ) ) {


<s> PredicateThen = std :: make_unique < PredicatedMI > ( MI , Preds ) ;


return llvm :: any_of ( Insts , isVCTP ) ;


<s> MIB . addImm ( <NUM_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; LoLoop . BlockMasksToRecompute . insert ( MIB . getInstr ( ) ) ; } else { ReplaceVCMPWithVPT ( VCMP , VCMP ) ; } } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * VPST ) ; LoLoop . ToRemove . insert ( VPST ) ; } else if ( Block . containsVCTP ( ) ) { MachineInstr * VPST = Insts . front ( ) ; if ( Block . size ( ) == <NUM_LIT> ) { assert ( VPST -> getOpcode ( ) == <STR_LIT> && <STR_LIT> ) ; LoLoop . ToRemove . insert ( VPST ) ; } else LoLoop . BlockMasksToRecompute . insert ( VPST ) ; } else if ( Insts . front ( ) -> getOpcode ( ) == <STR_LIT> ) { MachineInstr * VPST = Insts . front ( ) ; auto Next = ++ MachineBasicBlock :: iterator ( VPST ) ; assert ( getVPTInstrPredicate ( * Next ) != <STR_LIT> && <STR_LIT> ) ; ( void ) Next ; MachineInstr * VprDef = RDA -> getUniqueReachingMIDef ( VPST , <STR_LIT> ) ; if ( VprDef && VCMPOpcodeToVPT ( VprDef -> getOpcode ( ) ) && ! LoLoop . ToRemove . contains ( VprDef ) ) { MachineInstr * VCMP = VprDef ; if ( std :: none_of ( ++ MachineBasicBlock :: iterator ( VCMP ) , MachineBasicBlock :: iterator ( VPST ) , hasVPRUse ) && RDA -> hasSameReachingDef ( VCMP , VPST , VCMP -> getOperand ( <NUM_LIT> ) . getReg ( ) ) && RDA -> hasSameReachingDef ( VCMP , VPST , VCMP -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { ReplaceVCMPWithVPT ( VCMP , VPST ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * VPST ) ;


<s> } ; for ( auto & Block : LoLoop . getVPTBlocks ( ) ) { SmallVectorImpl < MachineInstr * > & Insts = Block . getInsts ( ) ; auto ReplaceVCMPWithVPT = [ & ] ( MachineInstr * & TheVCMP , MachineInstr * At ) { assert ( TheVCMP && <STR_LIT> ) ; MachineInstrBuilder MIB = BuildMI ( * At -> getParent ( ) , At , At -> getDebugLoc ( ) , TII -> get ( VCMPOpcodeToVPT ( TheVCMP -> getOpcode ( ) ) ) ) ; MIB . addImm ( <STR_LIT> ) ; MIB . add ( TheVCMP -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( TheVCMP -> getOperand ( <NUM_LIT> ) ) ; MIB . add ( TheVCMP -> getOperand ( <NUM_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; LoLoop . BlockMasksToRecompute . insert ( MIB . getInstr ( ) ) ; LoLoop . ToRemove . insert ( TheVCMP ) ; TheVCMP = nullptr ; } ; if ( VPTState :: isEntryPredicatedOnVCTP ( Block , true ) ) { MachineInstr * VPST = Insts . front ( ) ; if ( VPTState :: hasUniformPredicate ( Block ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * VPST ) ; for ( unsigned i = <NUM_LIT> ; i < Insts . size ( ) ; ++ i ) RemovePredicate ( Insts [ i ] ) ; } else { MachineInstr * Divergent = VPTState :: getDivergent ( Block ) ; MachineBasicBlock * MBB = Divergent -> getParent ( ) ; auto DivergentNext = ++ MachineBasicBlock :: iterator ( Divergent ) ; while ( DivergentNext != MBB -> end ( ) && DivergentNext -> isDebugInstr ( ) ) ++ DivergentNext ; bool DivergentNextIsPredicated = DivergentNext != MBB -> end ( ) && getVPTInstrPredicate ( * DivergentNext ) != <STR_LIT> ; for ( auto I = ++ MachineBasicBlock :: iterator ( VPST ) , E = DivergentNext ; I != E ; ++ I ) RemovePredicate ( & * I ) ; MachineInstr * VCMP = VCMPOpcodeToVPT ( Divergent -> getOpcode ( ) ) != <NUM_LIT> ? Divergent : nullptr ; if ( DivergentNextIsPredicated ) { if ( ! VCMP ) { MachineInstrBuilder MIB = BuildMI ( * Divergent -> getParent ( ) , Divergent , Divergent -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . addImm ( <NUM_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; LoLoop . BlockMasksToRecompute . insert ( MIB . getInstr ( ) ) ; } else {


<s> SetVector < MachineInstr * > & Predicates = PredicatedInsts [ MI ] -> Predicates ; if ( Exclusive && Predicates . size ( ) != <NUM_LIT> ) return false ; return llvm :: any_of ( Predicates , isVCTP ) ;