<s> assert ( BBOffsets . size ( ) == BBSizes . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = BBOffsets . size ( ) ; i != e ; ++ i ) assert ( BBOffsets [ i - <NUM_LIT> ] + BBSizes [ i - <NUM_LIT> ] == BBOffsets [ i ] ) ; if ( isThumb ) { for ( MachineFunction :: iterator MBBI = Fn . begin ( ) , E = Fn . end ( ) ;


<s> unsigned MaxDisp = U . MaxDisp ; MachineFunction :: iterator I = next ( MachineFunction :: iterator ( Water ) ) ; unsigned CPEOffset = BBOffsets [ Water -> getNumber ( ) ] + BBSizes [ Water -> getNumber ( ) ] ; if ( CPEOffset < UserOffset ) UserOffset += U . CPEMI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; return OffsetIsInRange ( UserOffset , CPEOffset , MaxDisp , ! isThumb ) ;


<s> MachineFunction :: iterator I = next ( MachineFunction :: iterator ( Water ) ) ; unsigned CPEOffset = BBOffsets [ Water -> getNumber ( ) ] + BBSizes [ Water -> getNumber ( ) ] ; if ( CPEOffset < UserOffset ) UserOffset += U . CPEMI -> getOperand ( <NUM_LIT> ) . getImm ( ) ;

return OffsetIsInRange ( UserOffset , CPEOffset , MaxDisp , ! isThumb ) ;


<s> bool B = TII -> AnalyzeBranch ( * BB , TBB , FBB , Cond ) ; if ( ! B && Cond . empty ( ) && BB != MF -> begin ( ) && ! TII -> AnalyzeBranch ( * OldPrior , TBB , FBB , CondPrior ) ) { BB -> moveAfter ( JTBB ) ; OldPrior -> updateTerminator ( ) ; BB -> updateTerminator ( ) ; MF -> RenumberBlocks ( ) ; ++ NumJTMoved ; return NULL ; } MachineBasicBlock * NewBB = MF -> CreateMachineBasicBlock ( JTBB -> getBasicBlock ( ) ) ; MachineFunction :: iterator MBBI = JTBB ; ++ MBBI ; MF -> insert ( MBBI , NewBB ) ;


<s> ++ NumJTMoved ; return NULL ; } MachineBasicBlock * NewBB = MF -> CreateMachineBasicBlock ( JTBB -> getBasicBlock ( ) ) ; MachineFunction :: iterator MBBI = JTBB ; ++ MBBI ; MF -> insert ( MBBI , NewBB ) ; assert ( isThumb2 && <STR_LIT> ) ; BuildMI ( NewBB , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( BB ) . addImm ( <STR_LIT> ) . addReg ( <NUM_LIT> ) ; MF -> RenumberBlocks ( NewBB ) ; NewBB -> addSuccessor ( BB ) ; JTBB -> removeSuccessor ( BB ) ; JTBB -> addSuccessor ( NewBB ) ; ++ NumJTInserted ; return NewBB ;


<s> MachineFunction :: iterator MBBI = MBB ;

if ( std :: next ( MBBI ) == MBB -> getParent ( ) -> end ( ) ) return false ;


<s> MachineBasicBlock * WaterBB = * IP ; unsigned Growth ; if ( isWaterInRange ( UserOffset , WaterBB , U , Growth ) && ( WaterBB -> getNumber ( ) < U . HighWaterMark -> getNumber ( ) || NewWaterList . count ( WaterBB ) ) && Growth < BestGrowth ) { BestGrowth = Growth ; WaterIter = IP ; DEBUG ( dbgs ( ) << <STR_LIT> << WaterBB -> getNumber ( ) << <STR_LIT> << Growth << '\n' ) ; if ( BestGrowth == <NUM_LIT> ) return true ; } if ( IP == B ) break ; }


<s> MachineInstr * MI = Br . MI ; MachineBasicBlock * DestBB = MI -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; <STR_LIT> CC = ( <STR_LIT> ) MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; CC = <STR_LIT> ( CC ) ; unsigned CCReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineBasicBlock * MBB = MI -> getParent ( ) ; MachineInstr * BMI = & MBB -> back ( ) ; bool NeedSplit = ( BMI != MI ) || ! BBHasFallthrough ( MBB ) ; ++ NumCBrFixed ; if ( BMI != MI ) { if ( std :: next ( MachineBasicBlock :: iterator ( MI ) ) == std :: prev ( MBB -> end ( ) ) && BMI -> getOpcode ( ) == Br . UncondBr ) { MachineBasicBlock * NewDest = BMI -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( isBBInRange ( MI , NewDest , Br . MaxDisp ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << * BMI ) ; BMI -> getOperand ( <NUM_LIT> ) . setMBB ( DestBB ) ; MI -> getOperand ( <NUM_LIT> ) . setMBB ( NewDest ) ; MI -> getOperand ( <NUM_LIT> ) . setImm ( CC ) ; return true ; } } } if ( NeedSplit ) { splitBlockBeforeInstr ( MI ) ; int delta = TII -> GetInstSizeInBytes ( & MBB -> back ( ) ) ; BBInfo [ MBB -> getNumber ( ) ] . Size -= delta ; MBB -> back ( ) . eraseFromParent ( ) ; } MachineBasicBlock * NextBB = std :: next ( MachineFunction :: iterator ( MBB ) ) ;

DEBUG ( dbgs ( ) << <STR_LIT> << DestBB -> getNumber ( ) << <STR_LIT> << NextBB -> getNumber ( ) << <STR_LIT> ) ;


<s> NewWaterList . insert ( NewIsland ) ; } if ( IP != WaterList . end ( ) ) WaterList . erase ( IP ) ; MF -> insert ( NewMBB , NewIsland ) ; updateForInsertedWaterBlock ( NewIsland ) ; decrementCPEReferenceCount ( CPI , CPEMI ) ; U . HighWaterMark = NewIsland ; U . CPEMI = BuildMI ( NewIsland , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( ID ) . addConstantPoolIndex ( CPI ) . addImm ( Size ) ; CPEntries [ CPI ] . push_back ( CPEntry ( U . CPEMI , ID , <NUM_LIT> ) ) ; ++ NumCPEs ; NewIsland -> setAlignment ( getCPELogAlign ( U . CPEMI ) ) ; BBInfo [ NewIsland -> getNumber ( ) ] . Size += Size ; adjustBBOffsetsAfter ( std :: prev ( MachineFunction :: iterator ( NewIsland ) ) ) ; for ( unsigned i = <NUM_LIT> , e = UserMI -> getNumOperands ( ) ; i != e ; ++ i ) if ( UserMI -> getOperand ( i ) . isCPI ( ) ) {

UserMI -> getOperand ( i ) . setIndex ( ID ) ;


<s> MachineBasicBlock * OrigBB = MI -> getParent ( ) ; MachineBasicBlock * NewBB = MF -> CreateMachineBasicBlock ( OrigBB -> getBasicBlock ( ) ) ; MachineFunction :: iterator MBBI = OrigBB ; ++ MBBI ; MF -> insert ( MBBI , NewBB ) ; NewBB -> splice ( NewBB -> end ( ) , OrigBB , MI , OrigBB -> end ( ) ) ; unsigned Opc = isThumb ? ( isThumb2 ? <STR_LIT> : <STR_LIT> ) : <STR_LIT> ; if ( ! isThumb ) BuildMI ( OrigBB , DebugLoc ( ) , TII -> get ( Opc ) ) . addMBB ( NewBB ) ; else BuildMI ( OrigBB , DebugLoc ( ) , TII -> get ( Opc ) ) . addMBB ( NewBB ) . addImm ( <STR_LIT> ) . addReg ( <NUM_LIT> ) ; ++ NumSplit ; NewBB -> transferSuccessors ( OrigBB ) ; OrigBB -> addSuccessor ( NewBB ) ; MF -> RenumberBlocks ( NewBB ) ; BBInfo . insert ( BBInfo . begin ( ) + NewBB -> getNumber ( ) , BasicBlockInfo ( ) ) ; water_iterator IP = std :: lower_bound ( WaterList . begin ( ) , WaterList . end ( ) , OrigBB , CompareMBBNumbers ) ; MachineBasicBlock * WaterBB = * IP ; if ( WaterBB == OrigBB ) WaterList . insert ( std :: next ( IP ) , NewBB ) ; else WaterList . insert ( IP , OrigBB ) ; NewWaterList . insert ( OrigBB ) ; computeBlockSize ( OrigBB ) ; computeBlockSize ( NewBB ) ;


<s> unsigned Opc = isThumb ? ( isThumb2 ? <STR_LIT> : <STR_LIT> ) : <STR_LIT> ; if ( ! isThumb ) BuildMI ( OrigBB , DebugLoc ( ) , TII -> get ( Opc ) ) . addMBB ( NewBB ) ; else BuildMI ( OrigBB , DebugLoc ( ) , TII -> get ( Opc ) ) . addMBB ( NewBB ) . addImm ( <STR_LIT> ) . addReg ( <NUM_LIT> ) ; ++ NumSplit ; NewBB -> transferSuccessors ( OrigBB ) ; OrigBB -> addSuccessor ( NewBB ) ; MF -> RenumberBlocks ( NewBB ) ; BBInfo . insert ( BBInfo . begin ( ) + NewBB -> getNumber ( ) , BasicBlockInfo ( ) ) ;

water_iterator IP = std :: lower_bound ( WaterList . begin ( ) , WaterList . end ( ) , OrigBB , CompareMBBNumbers ) ;


<s> else BuildMI ( UserMBB , DebugLoc ( ) , TII -> get ( UncondBr ) ) . addMBB ( NewMBB ) . add ( predOps ( <STR_LIT> ) ) ; unsigned MaxDisp = getUnconditionalBrDisp ( UncondBr ) ; ImmBranches . push_back ( ImmBranch ( & UserMBB -> back ( ) , MaxDisp , false , UncondBr ) ) ; computeBlockSize ( MF , UserMBB , BBInfo [ UserMBB -> getNumber ( ) ] ) ; adjustBBOffsetsAfter ( UserMBB ) ; return ; } } unsigned LogAlign = MF -> getAlignment ( ) ; assert ( LogAlign >= CPELogAlign && <STR_LIT> ) ; unsigned KnownBits = UserBBI . internalKnownBits ( ) ; unsigned UPad = UnknownPadding ( LogAlign , KnownBits ) ; unsigned BaseInsertOffset = UserOffset + U . getMaxDisp ( ) - UPad ; DEBUG ( dbgs ( ) << format ( <STR_LIT> , BaseInsertOffset ) ) ; BaseInsertOffset -= <NUM_LIT> ; DEBUG ( dbgs ( ) << format ( <STR_LIT> , BaseInsertOffset ) << <STR_LIT> << LogAlign << <STR_LIT> << KnownBits << <STR_LIT> << UPad << '\n' ) ; if ( BaseInsertOffset + <NUM_LIT> >= UserBBI . postOffset ( ) ) { BaseInsertOffset = std :: max ( UserBBI . postOffset ( ) - UPad - <NUM_LIT> , UserOffset + TII -> getInstSizeInBytes ( * UserMI ) + <NUM_LIT> ) ;


<s> assert ( isPowerOf2_32 ( Align ) && <STR_LIT> ) ; assert ( ( Size % Align ) == <NUM_LIT> && <STR_LIT> ) ; unsigned LogAlign = Log2_32 ( Align ) ; MachineBasicBlock :: iterator InsAt = InsPoint [ LogAlign ] ; MachineInstr * CPEMI = BuildMI ( * BB , InsAt , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( i ) . addConstantPoolIndex ( i ) . addImm ( Size ) ; CPEMIs . push_back ( CPEMI ) ; for ( unsigned a = LogAlign + <NUM_LIT> ; a <= MaxAlign ; ++ a ) if ( InsPoint [ a ] == InsAt ) InsPoint [ a ] = CPEMI ; CPEntries . emplace_back ( <NUM_LIT> , CPEntry ( CPEMI , i ) ) ; ++ NumCPEs ;

DEBUG ( dbgs ( ) << <STR_LIT> << i << <STR_LIT> << Size << <STR_LIT> << Align << '\n' ) ;


<s> const BasicBlockInfo & BBI = BBInfo [ J ] ; dbgs ( ) << format ( <STR_LIT> , BBI . Offset , J ) << <STR_LIT> << unsigned ( BBI . KnownBits ) << <STR_LIT> << unsigned ( BBI . Unalign ) << <STR_LIT> << unsigned ( BBI . PostAlign ) << format ( <STR_LIT> , BBInfo [ J ] . Size ) ;


<s> const BasicBlockInfo & BBI = BBInfo [ J ] ;

dbgs ( ) << format ( <STR_LIT> , BBI . Offset , J ) << <STR_LIT> << unsigned ( BBI . KnownBits ) << <STR_LIT> << unsigned ( BBI . Unalign ) << <STR_LIT> << unsigned ( BBI . PostAlign ) << format ( <STR_LIT> , BBInfo [ J ] . Size ) ;


<s> -- IP ) { MachineBasicBlock * WaterBB = * IP ; unsigned Growth ; if ( isWaterInRange ( UserOffset , WaterBB , U , Growth ) && ( WaterBB -> getNumber ( ) < U . HighWaterMark -> getNumber ( ) || NewWaterList . count ( WaterBB ) || WaterBB == U . MI -> getParent ( ) ) && Growth < BestGrowth ) { BestGrowth = Growth ; WaterIter = IP ; DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * WaterBB ) << <STR_LIT> << Growth << '\n' ) ; if ( CloserWater && WaterBB == U . MI -> getParent ( ) ) return true ; if ( ! CloserWater && BestGrowth == <NUM_LIT> ) return true ; } if ( IP == B ) break ; } return BestGrowth != ~ <NUM_LIT> ;


<s> MachineBasicBlock * UserBB = U . MI -> getParent ( ) ; unsigned MinNoSplitDisp = BBInfo [ UserBB -> getNumber ( ) ] . postOffset ( getCPELogAlign ( U . CPEMI ) ) ; if ( CloserWater && MinNoSplitDisp > U . getMaxDisp ( ) / <NUM_LIT> ) return false ; for ( water_iterator IP = std :: prev ( WaterList . end ( ) ) , B = WaterList . begin ( ) ; ; -- IP ) { MachineBasicBlock * WaterBB = * IP ; unsigned Growth ; if ( isWaterInRange ( UserOffset , WaterBB , U , Growth ) && ( WaterBB -> getNumber ( ) < U . HighWaterMark -> getNumber ( ) || NewWaterList . count ( WaterBB ) || WaterBB == U . MI -> getParent ( ) ) && Growth < BestGrowth ) { BestGrowth = Growth ; WaterIter = IP ; DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * WaterBB ) << <STR_LIT> << Growth << '\n' ) ; if ( CloserWater && WaterBB == U . MI -> getParent ( ) ) return true ; if ( ! CloserWater && BestGrowth == <NUM_LIT> ) return true ; }


<s> BBInfo [ MBB -> getNumber ( ) ] . Size -= delta ; MBB -> back ( ) . eraseFromParent ( ) ; MBB -> addSuccessor ( DestBB ) ; std :: next ( MBB -> getIterator ( ) ) -> removeSuccessor ( DestBB ) ; } MachineBasicBlock * NextBB = & * ++ MBB -> getIterator ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * DestBB ) << <STR_LIT> << printMBBReference ( * NextBB ) << <STR_LIT> ) ; BuildMI ( MBB , DebugLoc ( ) , TII -> get ( MI -> getOpcode ( ) ) ) . addMBB ( NextBB ) . addImm ( CC ) . addReg ( CCReg ) ; Br . MI = & MBB -> back ( ) ; BBInfo [ MBB -> getNumber ( ) ] . Size += TII -> getInstSizeInBytes ( MBB -> back ( ) ) ; if ( isThumb ) BuildMI ( MBB , DebugLoc ( ) , TII -> get ( Br . UncondBr ) ) . addMBB ( DestBB ) . add ( predOps ( <STR_LIT> ) ) ; else BuildMI ( MBB , DebugLoc ( ) , TII -> get ( Br . UncondBr ) ) . addMBB ( DestBB ) ; BBInfo [ MBB -> getNumber ( ) ] . Size += TII -> getInstSizeInBytes ( MBB -> back ( ) ) ; unsigned MaxDisp = getUnconditionalBrDisp ( Br . UncondBr ) ; ImmBranches . push_back ( ImmBranch ( & MBB -> back ( ) , MaxDisp , false , Br . UncondBr ) ) ; BBInfo [ MI -> getParent ( ) -> getNumber ( ) ] . Size -= TII -> getInstSizeInBytes ( * MI ) ; MI -> eraseFromParent ( ) ; adjustBBOffsetsAfter ( MBB ) ;


<s> DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * DestBB ) << <STR_LIT> << printMBBReference ( * MI -> getParent ( ) ) << <STR_LIT> << MaxDisp << <STR_LIT> << getOffsetOf ( MI ) << <STR_LIT> << DestOffset << <STR_LIT> << int ( DestOffset - BrOffset ) << <STR_LIT> << * MI ) ; if ( BrOffset <= DestOffset ) { if ( DestOffset - BrOffset <= MaxDisp ) return true ; } else { if ( BrOffset - DestOffset <= MaxDisp ) return true ; } return false ;


<s> dbgs ( ) << <STR_LIT> << CPEMI -> getOperand ( <NUM_LIT> ) . getImm ( ) << <STR_LIT> << MaxDisp << format ( <STR_LIT> , UserOffset ) << <STR_LIT> << printMBBReference ( * MI -> getParent ( ) ) << <STR_LIT> << format ( <STR_LIT> , BBI . Offset , BBI . postOffset ( ) ) << * MI << format ( <STR_LIT> , CPEOffset , int ( CPEOffset - UserOffset ) ) ; } ) ;


<s> int UOpc = Opc ; switch ( Opc ) { default : continue ; case <STR_LIT> : case <STR_LIT> : T2JumpTables . push_back ( & I ) ; continue ; case <STR_LIT> : isCond = true ; UOpc = <STR_LIT> ; LLVM_FALLTHROUGH ; case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : isCond = true ; UOpc = <STR_LIT> ; Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : isCond = true ; UOpc = <STR_LIT> ; Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; } unsigned MaxOffs = ( ( <NUM_LIT> << ( Bits - <NUM_LIT> ) ) - <NUM_LIT> ) * Scale ; ImmBranches . push_back ( ImmBranch ( & I , MaxOffs , isCond , UOpc ) ) ; } if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) PushPopMIs . push_back ( & I ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) continue ; for ( unsigned op = <NUM_LIT> , e = I . getNumOperands ( ) ; op != e ; ++ op ) if ( I . getOperand ( op ) . isCPI ( ) || I . getOperand ( op ) . isJTI ( ) ) { unsigned Bits = <NUM_LIT> ; unsigned Scale = <NUM_LIT> ; bool NegOk = false ; bool IsSoImm = false ; switch ( Opc ) { default : llvm_unreachable ( <STR_LIT> ) ; case <STR_LIT> : case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; NegOk = true ; IsSoImm = true ;


<s> for ( unsigned i = <NUM_LIT> , e = PushPopMIs . size ( ) ; i != e ; ++ i ) { MachineInstr * MI = PushPopMIs [ i ] ; if ( MI -> getOpcode ( ) == <STR_LIT> && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && MI -> getNumExplicitOperands ( ) == <NUM_LIT> ) { BuildMI ( MI -> getParent ( ) , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . add ( MI -> getOperand ( <NUM_LIT> ) ) . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MI -> eraseFromParent ( ) ; MadeChange = true ; } } return MadeChange ;


<s> bool MadeChange = false ; for ( unsigned i = <NUM_LIT> , e = PushPopMIs . size ( ) ; i != e ; ++ i ) { MachineInstr * MI = PushPopMIs [ i ] ; if ( MI -> getOpcode ( ) == <STR_LIT> && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && MI -> getNumExplicitOperands ( ) == <NUM_LIT> ) {

BuildMI ( MI -> getParent ( ) , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . add ( MI -> getOperand ( <NUM_LIT> ) ) . add ( MI -> getOperand ( <NUM_LIT> ) ) ;


<s> MachineFunction :: iterator OldPrior = std :: prev ( BBi ) ; bool B = TII -> analyzeBranch ( * BB , TBB , FBB , Cond ) ; if ( ! B && Cond . empty ( ) && BB != & MF -> front ( ) && ! TII -> analyzeBranch ( * OldPrior , TBB , FBB , CondPrior ) ) { BB -> moveAfter ( JTBB ) ; OldPrior -> updateTerminator ( ) ; BB -> updateTerminator ( ) ; MF -> RenumberBlocks ( ) ; ++ NumJTMoved ; return nullptr ; } MachineBasicBlock * NewBB = MF -> CreateMachineBasicBlock ( JTBB -> getBasicBlock ( ) ) ;

MachineFunction :: iterator MBBI = ++ JTBB -> getIterator ( ) ;


<s> MF -> insert ( MBBI , NewBB ) ; for ( const MachineBasicBlock :: RegisterMaskPair & RegMaskPair : BB -> liveins ( ) ) NewBB -> addLiveIn ( RegMaskPair ) ; if ( isThumb2 ) BuildMI ( NewBB , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( BB ) . add ( predOps ( <STR_LIT> ) ) ; else BuildMI ( NewBB , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( BB ) . add ( predOps ( <STR_LIT> ) ) ; MF -> RenumberBlocks ( NewBB ) ;


<s> MF -> push_back ( BB ) ; const Align MaxAlign = MCP -> getConstantPoolAlign ( ) ; const unsigned MaxLogAlign = Log2 ( MaxAlign ) ; BB -> setAlignment ( MaxAlign ) ; MF -> ensureAlignment ( BB -> getAlignment ( ) ) ; SmallVector < MachineBasicBlock :: iterator , <NUM_LIT> > InsPoint ( MaxLogAlign + <NUM_LIT> , BB -> end ( ) ) ; const std :: vector < MachineConstantPoolEntry > & CPs = MCP -> getConstants ( ) ; const DataLayout & TD = MF -> getDataLayout ( ) ; for ( unsigned i = <NUM_LIT> , e = CPs . size ( ) ; i != e ; ++ i ) { unsigned Size = TD . getTypeAllocSize ( CPs [ i ] . getType ( ) ) ; Align Alignment = CPs [ i ] . getAlign ( ) ; assert ( isAligned ( Alignment , Size ) && <STR_LIT> ) ; unsigned LogAlign = Log2 ( Alignment ) ; MachineBasicBlock :: iterator InsAt = InsPoint [ LogAlign ] ; MachineInstr * CPEMI = BuildMI ( * BB , InsAt , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( i ) . addConstantPoolIndex ( i ) . addImm ( Size ) ; CPEMIs . push_back ( CPEMI ) ; for ( unsigned a = LogAlign + <NUM_LIT> ; a <= MaxLogAlign ; ++ a ) if ( InsPoint [ a ] == InsAt ) InsPoint [ a ] = CPEMI ;

CPEntries . emplace_back ( <NUM_LIT> , CPEntry ( CPEMI , i ) ) ;


<s> unsigned Size = CPEMI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; unsigned UserOffset = getUserOffset ( U ) ; int result = findInRangeCPEntry ( U , UserOffset ) ; if ( result == <NUM_LIT> ) return false ; else if ( result == <NUM_LIT> ) return true ; unsigned ID = AFI -> createPICLabelUId ( ) ; MachineBasicBlock * NewIsland = MF -> CreateMachineBasicBlock ( ) ; MachineBasicBlock * NewMBB ; water_iterator IP ; if ( findAvailableWater ( U , UserOffset , IP , CloserWater ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; MachineBasicBlock * WaterBB = * IP ; if ( NewWaterList . erase ( WaterBB ) ) NewWaterList . insert ( NewIsland ) ; NewMBB = & * ++ WaterBB -> getIterator ( ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; createNewWater ( CPUserIndex , UserOffset , NewMBB ) ; MachineBasicBlock * WaterBB = & * -- NewMBB -> getIterator ( ) ; IP = find ( WaterList , WaterBB ) ; if ( IP != WaterList . end ( ) ) NewWaterList . erase ( WaterBB ) ; NewWaterList . insert ( NewIsland ) ; } const Align Alignment = isThumb ? Align ( <NUM_LIT> ) : Align ( <NUM_LIT> ) ; if ( NewMBB -> getAlignment ( ) < Alignment ) NewMBB -> setAlignment ( Alignment ) ; if ( IP != WaterList . end ( ) ) WaterList . erase ( IP ) ; MF -> insert ( NewMBB -> getIterator ( ) , NewIsland ) ; updateForInsertedWaterBlock ( NewIsland ) ; U . HighWaterMark = NewIsland ; U . CPEMI = BuildMI ( NewIsland , DebugLoc ( ) , CPEMI -> getDesc ( ) ) . addImm ( ID ) . add ( CPEMI -> getOperand ( <NUM_LIT> ) ) . addImm ( Size ) ; CPEntries [ CPI ] . push_back ( CPEntry ( U . CPEMI , ID , <NUM_LIT> ) ) ; ++ NumCPEs ; decrementCPEReferenceCount ( CPI , CPEMI ) ; NewIsland -> setAlignment ( getCPEAlign ( U . CPEMI ) ) ; BBUtils -> adjustBBSize ( NewIsland , Size ) ; BBUtils -> adjustBBOffsetsAfter ( & * -- NewIsland -> getIterator ( ) ) ; for ( unsigned i = <NUM_LIT> , e = UserMI -> getNumOperands ( ) ; i != e ; ++ i ) if ( UserMI -> getOperand ( i ) . isCPI ( ) ) {


<s> createNewWater ( CPUserIndex , UserOffset , NewMBB ) ; MachineBasicBlock * WaterBB = & * -- NewMBB -> getIterator ( ) ; IP = find ( WaterList , WaterBB ) ; if ( IP != WaterList . end ( ) ) NewWaterList . erase ( WaterBB ) ; NewWaterList . insert ( NewIsland ) ; } const Align Alignment = isThumb ? Align ( <NUM_LIT> ) : Align ( <NUM_LIT> ) ; if ( NewMBB -> getAlignment ( ) < Alignment ) NewMBB -> setAlignment ( Alignment ) ; if ( IP != WaterList . end ( ) ) WaterList . erase ( IP ) ; MF -> insert ( NewMBB -> getIterator ( ) , NewIsland ) ; updateForInsertedWaterBlock ( NewIsland ) ; U . HighWaterMark = NewIsland ; U . CPEMI = BuildMI ( NewIsland , DebugLoc ( ) , CPEMI -> getDesc ( ) ) . addImm ( ID ) . add ( CPEMI -> getOperand ( <NUM_LIT> ) ) . addImm ( Size ) ; CPEntries [ CPI ] . push_back ( CPEntry ( U . CPEMI , ID , <NUM_LIT> ) ) ; ++ NumCPEs ; decrementCPEReferenceCount ( CPI , CPEMI ) ;

NewIsland -> setAlignment ( getCPEAlign ( U . CPEMI ) ) ;


<s> unsigned Opc = I . getOpcode ( ) ; if ( I . isBranch ( ) ) { bool isCond = false ; unsigned Bits = <NUM_LIT> ; unsigned Scale = <NUM_LIT> ; int UOpc = Opc ; switch ( Opc ) { default : continue ; case <STR_LIT> : case <STR_LIT> : T2JumpTables . push_back ( & I ) ; continue ; case <STR_LIT> : isCond = true ; UOpc = <STR_LIT> ; LLVM_FALLTHROUGH ; case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : isCond = true ; UOpc = <STR_LIT> ; Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : isCond = true ; UOpc = <STR_LIT> ; Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; } unsigned MaxOffs = ( ( <NUM_LIT> << ( Bits - <NUM_LIT> ) ) - <NUM_LIT> ) * Scale ; ImmBranches . push_back ( ImmBranch ( & I , MaxOffs , isCond , UOpc ) ) ; } if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) PushPopMIs . push_back ( & I ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) continue ; for ( unsigned op = <NUM_LIT> , e = I . getNumOperands ( ) ; op != e ; ++ op ) if ( I . getOperand ( op ) . isCPI ( ) || I . getOperand ( op ) . isJTI ( ) ) { unsigned Bits = <NUM_LIT> ; unsigned Scale = <NUM_LIT> ; bool NegOk = false ; bool IsSoImm = false ; switch ( Opc ) { default : llvm_unreachable ( <STR_LIT> ) ; case <STR_LIT> : case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; NegOk = true ; IsSoImm = true ; break ; case <STR_LIT> : case <STR_LIT> : Bits = <NUM_LIT> ; NegOk = true ; break ; case <STR_LIT> : case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> for ( MachineBasicBlock & MBB : * MF ) { if ( ! BBHasFallthrough ( & MBB ) ) WaterList . push_back ( & MBB ) ; for ( MachineInstr & I : MBB ) { if ( I . isDebugInstr ( ) ) continue ; unsigned Opc = I . getOpcode ( ) ; if ( I . isBranch ( ) ) { bool isCond = false ; unsigned Bits = <NUM_LIT> ; unsigned Scale = <NUM_LIT> ; int UOpc = Opc ; switch ( Opc ) { default : continue ; case <STR_LIT> : case <STR_LIT> : T2JumpTables . push_back ( & I ) ; continue ; case <STR_LIT> : isCond = true ; UOpc = <STR_LIT> ; LLVM_FALLTHROUGH ; case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : isCond = true ; UOpc = <STR_LIT> ; Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : isCond = true ; UOpc = <STR_LIT> ; Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; } unsigned MaxOffs = ( ( <NUM_LIT> << ( Bits - <NUM_LIT> ) ) - <NUM_LIT> ) * Scale ; ImmBranches . push_back ( ImmBranch ( & I , MaxOffs , isCond , UOpc ) ) ; } if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) PushPopMIs . push_back ( & I ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) continue ; for ( unsigned op = <NUM_LIT> , e = I . getNumOperands ( ) ; op != e ; ++ op ) if ( I . getOperand ( op ) . isCPI ( ) || I . getOperand ( op ) . isJTI ( ) ) {


<s> auto FindCmpForCBZ = [ this ] ( ImmBranch & Br , ImmCompare & ImmCmp , MachineBasicBlock * DestBB ) { ImmCmp . MI = nullptr ; ImmCmp . NewOpc = <NUM_LIT> ; if ( ! Br . MI -> killsRegister ( <STR_LIT> ) ) return false ; Register PredReg ; unsigned NewOpc = <NUM_LIT> ; <STR_LIT> Pred = getInstrPredicate ( * Br . MI , PredReg ) ; if ( Pred == <STR_LIT> ) NewOpc = <STR_LIT> ; else if ( Pred == <STR_LIT> ) NewOpc = <STR_LIT> ; else return false ; unsigned BrOffset = BBUtils -> getOffsetOf ( Br . MI ) + <NUM_LIT> - <NUM_LIT> ; BBInfoVector & BBInfo = BBUtils -> getBBInfo ( ) ; unsigned DestOffset = BBInfo [ DestBB -> getNumber ( ) ] . Offset ; if ( BrOffset >= DestOffset || ( DestOffset - BrOffset ) > <NUM_LIT> ) return false ; auto * TRI = STI -> getRegisterInfo ( ) ; MachineInstr * CmpMI = findCMPToFoldIntoCBZ ( Br . MI , TRI ) ; if ( ! CmpMI || CmpMI -> getOpcode ( ) != <STR_LIT> ) return false ; ImmCmp . MI = CmpMI ; ImmCmp . NewOpc = NewOpc ; return true ; } ; auto TryConvertToLE = [ this ] ( ImmBranch & Br , ImmCompare & Cmp ) { if ( Br . MI -> getOpcode ( ) != <STR_LIT> || ! STI -> hasLOB ( ) || STI -> hasMinSize ( ) ) return false ; MachineBasicBlock * MBB = Br . MI -> getParent ( ) ; MachineBasicBlock * DestBB = Br . MI -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( BBUtils -> getOffsetOf ( MBB ) < BBUtils -> getOffsetOf ( DestBB ) || ! BBUtils -> isBBInRange ( Br . MI , DestBB , <NUM_LIT> ) ) return false ; if ( ! DT -> dominates ( DestBB , MBB ) ) return false ; Cmp . NewOpc = Cmp . NewOpc == <STR_LIT> ? <STR_LIT> : <STR_LIT> ; MachineInstrBuilder MIB = BuildMI ( * MBB , Br . MI , Br . MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; MIB . add ( Br . MI -> getOperand ( <NUM_LIT> ) ) ; Br . MI -> eraseFromParent ( ) ; Br . MI = MIB ; ++ NumLEInserted ; return true ;


<s> Br . MI = MIB ; ++ NumLEInserted ; return true ; } ; bool MadeChange = false ; for ( ImmBranch & Br : reverse ( ImmBranches ) ) { MachineBasicBlock * DestBB = Br . MI -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock * MBB = Br . MI -> getParent ( ) ; MachineBasicBlock * ExitBB = & MBB -> back ( ) == Br . MI ? MBB -> getFallThrough ( ) : MBB -> back ( ) . getOperand ( <NUM_LIT> ) . getMBB ( ) ; ImmCompare Cmp ; if ( FindCmpForCBZ ( Br , Cmp , ExitBB ) && TryConvertToLE ( Br , Cmp ) ) { DestBB = ExitBB ; MadeChange = true ; } else { FindCmpForCBZ ( Br , Cmp , DestBB ) ; MadeChange |= TryShrinkBranch ( Br ) ; } unsigned Opcode = Br . MI -> getOpcode ( ) ; if ( ( Opcode != <STR_LIT> && Opcode != <STR_LIT> ) || ! Cmp . NewOpc ) continue ; Register Reg = Cmp . MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; auto * TRI = STI -> getRegisterInfo ( ) ; MachineBasicBlock :: iterator KillMI = Br . MI ; bool RegKilled = false ; do { -- KillMI ; if ( KillMI -> killsRegister ( Reg , TRI ) ) { KillMI -> clearRegisterKills ( Reg , TRI ) ; RegKilled = true ; break ; } } while ( KillMI != Cmp . MI ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Cmp . MI << <STR_LIT> << * Br . MI ) ; MachineInstr * NewBR = BuildMI ( * MBB , Br . MI , Br . MI -> getDebugLoc ( ) , TII -> get ( Cmp . NewOpc ) ) . addReg ( Reg , getKillRegState ( RegKilled ) ) . addMBB ( DestBB , Br . MI -> getOperand ( <NUM_LIT> ) . getTargetFlags ( ) ) ; Cmp . MI -> eraseFromParent ( ) ; if ( Br . MI -> getOpcode ( ) == <STR_LIT> ) { Br . MI -> eraseFromParent ( ) ; Br . MI = NewBR ; BBUtils -> adjustBBSize ( MBB , - <NUM_LIT> ) ; } else if ( MBB -> back ( ) . getOpcode ( ) != <STR_LIT> ) { MachineInstr * LastMI = & MBB -> back ( ) ; BBUtils -> adjustBBSize ( MBB , - LastMI -> getDesc ( ) . getSize ( ) ) ; LastMI -> eraseFromParent ( ) ; }

BBUtils -> adjustBBOffsetsAfter ( MBB ) ;


<s> if ( registerDefinedBetween ( IdxReg , Add -> getNextNode ( ) , MI , TRI ) ) continue ; Add -> eraseFromParent ( ) ; DeadSize += <NUM_LIT> ; } else { if ( Load -> getOperand ( <NUM_LIT> ) . getReg ( ) != MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) continue ; if ( registerDefinedBetween ( IdxReg , Load -> getNextNode ( ) , MI , TRI ) ) continue ; } CanDeleteLEA = true ; Shift -> eraseFromParent ( ) ; Load -> eraseFromParent ( ) ; DeadSize += <NUM_LIT> ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; MachineInstr * CPEMI = User . CPEMI ; unsigned Opc = ByteOk ? <STR_LIT> : <STR_LIT> ; if ( ! isThumb2 ) Opc = ByteOk ? <STR_LIT> : <STR_LIT> ; MachineBasicBlock :: iterator MI_JT = MI ; MachineInstr * NewJTMI = BuildMI ( * MBB , MI_JT , MI -> getDebugLoc ( ) , TII -> get ( Opc ) ) . addReg ( User . MI -> getOperand ( <NUM_LIT> ) . getReg ( ) , getKillRegState ( BaseRegKill ) ) . addReg ( IdxReg , getKillRegState ( IdxRegKill ) ) . addJumpTableIndex ( JTI , JTOP . getTargetFlags ( ) ) . addImm ( CPEMI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ; LLVM_DEBUG ( dbgs ( ) << printMBBReference ( * MBB ) << <STR_LIT> << * NewJTMI ) ; unsigned JTOpc = ByteOk ? <STR_LIT> : <STR_LIT> ; CPEMI -> setDesc ( TII -> get ( JTOpc ) ) ; if ( jumpTableFollowsTB ( MI , User . CPEMI ) ) { NewJTMI -> getOperand ( <NUM_LIT> ) . setReg ( <STR_LIT> ) ; NewJTMI -> getOperand ( <NUM_LIT> ) . setIsKill ( false ) ; if ( CanDeleteLEA ) { if ( isThumb2 ) RemoveDeadAddBetweenLEAAndJT ( User . MI , MI , DeadSize ) ; User . MI -> eraseFromParent ( ) ; DeadSize += isThumb2 ? <NUM_LIT> : <NUM_LIT> ; User . MI = NewJTMI ; User . MaxDisp = <NUM_LIT> ; User . NegOk = false ; User . IsSoImm = false ; User . KnownAlignment = false ; } else { int CPEntryIdx = JumpTableEntryIndices [ JTI ] ; auto & CPEs = CPEntries [ CPEntryIdx ] ; auto Entry = find_if ( CPEs , [ & ] ( CPEntry & E ) { return E . CPEMI == User . CPEMI ; } ) ; ++ Entry -> RefCount ; CPUsers . emplace_back ( CPUser ( NewJTMI , User . CPEMI , <NUM_LIT> , false , false ) ) ; } } unsigned NewSize = TII -> getInstSizeInBytes ( * NewJTMI ) ; unsigned OrigSize = TII -> getInstSizeInBytes ( * MI ) ; MI -> eraseFromParent ( ) ; int Delta = OrigSize - NewSize + DeadSize ; BBInfo [ MBB -> getNumber ( ) ] . Size -= Delta ; BBUtils -> adjustBBOffsetsAfter ( MBB ) ; ++ NumTBs ; MadeChange = true ; }


<s> for ( unsigned i = <NUM_LIT> , e = CPUsers . size ( ) ; i != e ; ++ i ) CPChange |= handleConstantPoolUser ( i , NoCPIters >= CPMaxIteration / <NUM_LIT> ) ; if ( CPChange && ++ NoCPIters > CPMaxIteration ) report_fatal_error ( <STR_LIT> ) ; LLVM_DEBUG ( dumpBBs ( ) ) ; NewWaterList . clear ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << NoBRIters << '\n' ) ; bool BRChange = false ; for ( unsigned i = <NUM_LIT> , e = ImmBranches . size ( ) ; i != e ; ++ i ) BRChange |= fixupImmediateBr ( ImmBranches [ i ] ) ; if ( BRChange && ++ NoBRIters > <NUM_LIT> ) report_fatal_error ( <STR_LIT> ) ; LLVM_DEBUG ( dumpBBs ( ) ) ; if ( ! CPChange && ! BRChange ) break ; MadeChange = true ; } if ( isThumb2 && ! STI -> prefers32BitThumb ( ) ) MadeChange |= optimizeThumb2Instructions ( ) ; if ( isThumb && STI -> hasV8MBaselineOps ( ) ) MadeChange |= optimizeThumb2Branches ( ) ; if ( GenerateTBB && ! STI -> genExecuteOnly ( ) ) MadeChange |= optimizeThumb2JumpTables ( ) ; verify ( ) ; for ( unsigned i = <NUM_LIT> , e = CPEntries . size ( ) ; i != e ; ++ i ) { for ( unsigned j = <NUM_LIT> , je = CPEntries [ i ] . size ( ) ; j != je ; ++ j ) { const CPEntry & CPE = CPEntries [ i ] [ j ] ; if ( CPE . CPEMI && CPE . CPEMI -> getOperand ( <NUM_LIT> ) . isCPI ( ) ) AFI -> recordCPEClone ( i , CPE . CPI ) ; } } LLVM_DEBUG ( dbgs ( ) << '\n' ; dumpBBs ( ) ) ; BBUtils -> clear ( ) ; WaterList . clear ( ) ; CPUsers . clear ( ) ; CPEntries . clear ( ) ; JumpTableEntryIndices . clear ( ) ;


<s> assert ( std :: is_sorted ( MF -> begin ( ) , MF -> end ( ) , [ & BBInfo ] ( const MachineBasicBlock & LHS , const MachineBasicBlock & RHS ) { return BBInfo [ LHS . getNumber ( ) ] . postOffset ( ) < BBInfo [ RHS . getNumber ( ) ] . postOffset ( ) ; } ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << CPUsers . size ( ) << <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = CPUsers . size ( ) ; i != e ; ++ i ) { CPUser & U = CPUsers [ i ] ; unsigned UserOffset = getUserOffset ( U ) ; if ( isCPEntryInRange ( U . MI , UserOffset , U . CPEMI , U . getMaxDisp ( ) + <NUM_LIT> , U . NegOk , true ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ;


<s> BBInfoVector & BBInfo = BBUtils -> getBBInfo ( ) ; assert ( std :: is_sorted ( MF -> begin ( ) , MF -> end ( ) , [ & BBInfo ] ( const MachineBasicBlock & LHS , const MachineBasicBlock & RHS ) { return BBInfo [ LHS . getNumber ( ) ] . postOffset ( ) < BBInfo [ RHS . getNumber ( ) ] . postOffset ( ) ; } ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << CPUsers . size ( ) << <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = CPUsers . size ( ) ; i != e ; ++ i ) { CPUser & U = CPUsers [ i ] ; unsigned UserOffset = getUserOffset ( U ) ;

if ( isCPEntryInRange ( U . MI , UserOffset , U . CPEMI , U . getMaxDisp ( ) + <NUM_LIT> , U . NegOk , true ) ) {


<s> assert ( Size >= <NUM_LIT> && <STR_LIT> ) ; unsigned Align = CPs [ i ] . getAlignment ( ) ; assert ( isPowerOf2_32 ( Align ) && <STR_LIT> ) ; assert ( ( Size % Align ) == <NUM_LIT> && <STR_LIT> ) ; unsigned LogAlign = Log2_32 ( Align ) ; MachineBasicBlock :: iterator InsAt = InsPoint [ LogAlign ] ; MachineInstr * CPEMI = BuildMI ( * BB , InsAt , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( i ) . addConstantPoolIndex ( i ) . addImm ( Size ) ; CPEMIs . push_back ( CPEMI ) ; for ( unsigned a = LogAlign + <NUM_LIT> ; a <= MaxAlign ; ++ a ) if ( InsPoint [ a ] == InsAt ) InsPoint [ a ] = CPEMI ;


<s> MachineBasicBlock * BB = MF -> CreateMachineBasicBlock ( ) ; MF -> push_back ( BB ) ; unsigned MaxAlign = Log2_32 ( MCP -> getConstantPoolAlignment ( ) ) ; BB -> setAlignment ( AlignConstantIslands ? MaxAlign : <NUM_LIT> ) ; MF -> ensureAlignment ( BB -> getAlignment ( ) ) ; SmallVector < MachineBasicBlock :: iterator , <NUM_LIT> > InsPoint ( MaxAlign + <NUM_LIT> , BB -> end ( ) ) ; const std :: vector < MachineConstantPoolEntry > & CPs = MCP -> getConstants ( ) ; const TargetData & TD = * MF -> getTarget ( ) . getTargetData ( ) ; for ( unsigned i = <NUM_LIT> , e = CPs . size ( ) ; i != e ; ++ i ) { unsigned Size = TD . getTypeAllocSize ( CPs [ i ] . getType ( ) ) ; assert ( Size >= <NUM_LIT> && <STR_LIT> ) ; unsigned Align = CPs [ i ] . getAlignment ( ) ; assert ( isPowerOf2_32 ( Align ) && <STR_LIT> ) ; assert ( ( Size % Align ) == <NUM_LIT> && <STR_LIT> ) ; unsigned LogAlign = Log2_32 ( Align ) ; MachineBasicBlock :: iterator InsAt = InsPoint [ LogAlign ] ; MachineInstr * CPEMI = BuildMI ( * BB , InsAt , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( i ) . addConstantPoolIndex ( i ) . addImm ( Size ) ;


<s> for ( unsigned op = <NUM_LIT> , e = I -> getNumOperands ( ) ; op != e ; ++ op ) if ( I -> getOperand ( op ) . isCPI ( ) ) { unsigned Bits = <NUM_LIT> ; unsigned Scale = <NUM_LIT> ; bool NegOk = false ; bool IsSoImm = false ; switch ( Opc ) { default : llvm_unreachable ( <STR_LIT> ) ; break ; case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; NegOk = true ; IsSoImm = true ; break ; case <STR_LIT> : Bits = <NUM_LIT> ; NegOk = true ; break ; case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : Bits = <NUM_LIT> ; NegOk = true ; break ; case <STR_LIT> : case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; NegOk = true ; break ; } unsigned CPI = I -> getOperand ( op ) . getIndex ( ) ; MachineInstr * CPEMI = CPEMIs [ CPI ] ; unsigned MaxOffs = ( ( <NUM_LIT> << Bits ) - <NUM_LIT> ) * Scale ; CPUsers . push_back ( CPUser ( I , CPEMI , MaxOffs , NegOk , IsSoImm ) ) ; CPEntry * CPE = findConstPoolEntry ( CPI , CPEMI ) ; assert ( CPE && <STR_LIT> ) ; CPE -> RefCount ++ ; break ; } }

if ( isThumb && ! MBB . empty ( ) && MBB . begin ( ) -> getOpcode ( ) == <STR_LIT> && ( ( Offset % <NUM_LIT> ) != <NUM_LIT> || HasInlineAsm ) ) MBBSize += <NUM_LIT> ;


<s> MachineBasicBlock * TBB = nullptr , * FBB = nullptr ; SmallVector < MachineOperand , <NUM_LIT> > Cond ; SmallVector < MachineOperand , <NUM_LIT> > CondPrior ; MachineFunction :: iterator BBi = BB ; MachineFunction :: iterator OldPrior = std :: prev ( BBi ) ; bool B = TII -> AnalyzeBranch ( * BB , TBB , FBB , Cond ) ; if ( ! B && Cond . empty ( ) && BB != MF -> begin ( ) && ! TII -> AnalyzeBranch ( * OldPrior , TBB , FBB , CondPrior ) ) { BB -> moveAfter ( JTBB ) ; OldPrior -> updateTerminator ( ) ; BB -> updateTerminator ( ) ; MF -> RenumberBlocks ( ) ; ++ NumJTMoved ; return nullptr ; } MachineBasicBlock * NewBB = MF -> CreateMachineBasicBlock ( JTBB -> getBasicBlock ( ) ) ; MachineFunction :: iterator MBBI = JTBB ; ++ MBBI ; MF -> insert ( MBBI , NewBB ) ; assert ( isThumb2 && <STR_LIT> ) ; BuildMI ( NewBB , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( BB ) . addImm ( <STR_LIT> ) . addReg ( <NUM_LIT> ) ; MF -> RenumberBlocks ( NewBB ) ; NewBB -> addSuccessor ( BB ) ;


<s> MachineFunction :: iterator MBBI = JTBB ; ++ MBBI ; MF -> insert ( MBBI , NewBB ) ; assert ( isThumb2 && <STR_LIT> ) ; BuildMI ( NewBB , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( BB ) . addImm ( <STR_LIT> ) . addReg ( <NUM_LIT> ) ; MF -> RenumberBlocks ( NewBB ) ; NewBB -> addSuccessor ( BB ) ; JTBB -> removeSuccessor ( BB ) ; JTBB -> addSuccessor ( NewBB ) ; ++ NumJTInserted ; return NewBB ;


<s> DEBUG ( dbgs ( ) << <STR_LIT> << UserMBB -> getNumber ( ) << format ( <STR_LIT> , CPEOffset ) ) ; NewMBB = std :: next ( MachineFunction :: iterator ( UserMBB ) ) ; int UncondBr = isThumb ? ( ( isThumb2 ) ? <STR_LIT> : <STR_LIT> ) : <STR_LIT> ; if ( ! isThumb ) BuildMI ( UserMBB , DebugLoc ( ) , TII -> get ( UncondBr ) ) . addMBB ( NewMBB ) ; else BuildMI ( UserMBB , DebugLoc ( ) , TII -> get ( UncondBr ) ) . addMBB ( NewMBB ) . addImm ( <STR_LIT> ) . addReg ( <NUM_LIT> ) ; unsigned MaxDisp = getUnconditionalBrDisp ( UncondBr ) ; ImmBranches . push_back ( ImmBranch ( & UserMBB -> back ( ) , MaxDisp , false , UncondBr ) ) ; BBInfo [ UserMBB -> getNumber ( ) ] . Size += Delta ; adjustBBOffsetsAfter ( UserMBB ) ; return ; } } unsigned LogAlign = MF -> getAlignment ( ) ; assert ( LogAlign >= CPELogAlign && <STR_LIT> ) ; unsigned KnownBits = UserBBI . internalKnownBits ( ) ; unsigned UPad = UnknownPadding ( LogAlign , KnownBits ) ; unsigned BaseInsertOffset = UserOffset + U . getMaxDisp ( ) - UPad ; DEBUG ( dbgs ( ) << format ( <STR_LIT> , BaseInsertOffset ) ) ; BaseInsertOffset -= <NUM_LIT> ; DEBUG ( dbgs ( ) << format ( <STR_LIT> , BaseInsertOffset ) << <STR_LIT> << LogAlign << <STR_LIT> << KnownBits << <STR_LIT> << UPad << '\n' ) ; if ( BaseInsertOffset + <NUM_LIT> >= UserBBI . postOffset ( ) ) { BaseInsertOffset = UserBBI . postOffset ( ) - UPad - <NUM_LIT> ; DEBUG ( dbgs ( ) << format ( <STR_LIT> , BaseInsertOffset ) ) ; }

unsigned EndInsertOffset = BaseInsertOffset + <NUM_LIT> + UPad + CPEMI -> getOperand ( <NUM_LIT> ) . getImm ( ) ;


<s> MF -> ensureAlignment ( BB -> getAlignment ( ) ) ; SmallVector < MachineBasicBlock :: iterator , <NUM_LIT> > InsPoint ( MaxAlign + <NUM_LIT> , BB -> end ( ) ) ; const std :: vector < MachineConstantPoolEntry > & CPs = MCP -> getConstants ( ) ; const DataLayout & TD = * MF -> getSubtarget ( ) . getDataLayout ( ) ; for ( unsigned i = <NUM_LIT> , e = CPs . size ( ) ; i != e ; ++ i ) { unsigned Size = TD . getTypeAllocSize ( CPs [ i ] . getType ( ) ) ; assert ( Size >= <NUM_LIT> && <STR_LIT> ) ; unsigned Align = CPs [ i ] . getAlignment ( ) ; assert ( isPowerOf2_32 ( Align ) && <STR_LIT> ) ; assert ( ( Size % Align ) == <NUM_LIT> && <STR_LIT> ) ; unsigned LogAlign = Log2_32 ( Align ) ; MachineBasicBlock :: iterator InsAt = InsPoint [ LogAlign ] ; MachineInstr * CPEMI = BuildMI ( * BB , InsAt , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( i ) . addConstantPoolIndex ( i ) . addImm ( Size ) ; CPEMIs . push_back ( CPEMI ) ; for ( unsigned a = LogAlign + <NUM_LIT> ; a <= MaxAlign ; ++ a ) if ( InsPoint [ a ] == InsAt ) InsPoint [ a ] = CPEMI ; CPEntries . emplace_back ( <NUM_LIT> , CPEntry ( CPEMI , i ) ) ;


<s> int <ISA_LIT>ConstantIslands :: findInRangeCPEntry ( CPUser & U , unsigned UserOffset ) { MachineInstr * UserMI = U . MI ; MachineInstr * CPEMI = U . CPEMI ; if ( isCPEntryInRange ( UserMI , UserOffset , CPEMI , U . getMaxDisp ( ) , U . NegOk , true ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; return <NUM_LIT> ; } unsigned CPI = CPEMI -> getOperand ( <NUM_LIT> ) . getIndex ( ) ;

std :: vector < CPEntry > & CPEs = CPEntries [ CPI ] ;


<s> for ( unsigned k = <NUM_LIT> , eee = AddrMI -> getNumOperands ( ) ; k != eee ; ++ k ) { const MachineOperand & MO = AddrMI -> getOperand ( k ) ; if ( ! MO . isReg ( ) || ! MO . getReg ( ) ) continue ; if ( MO . isDef ( ) && MO . getReg ( ) != BaseReg ) { OptOk = false ; break ; } if ( MO . isUse ( ) && ! MO . isKill ( ) && MO . getReg ( ) != IdxReg ) { OptOk = false ; break ; } } if ( ! OptOk ) continue ; for ( -- PrevI ; PrevI != B && ! PrevI -> definesRegister ( BaseReg ) ; -- PrevI ) ; MachineInstr * LeaMI = PrevI ; if ( ( LeaMI -> getOpcode ( ) != <STR_LIT> && LeaMI -> getOpcode ( ) != <STR_LIT> ) || LeaMI -> getOperand ( <NUM_LIT> ) . getReg ( ) != BaseReg ) OptOk = false ; if ( ! OptOk ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << * MI << <STR_LIT> << * AddrMI << <STR_LIT> << * LeaMI ) ; unsigned Opc = ByteOk ? <STR_LIT> : <STR_LIT> ; MachineInstr * NewJTMI = BuildMI ( MBB , MI -> getDebugLoc ( ) , TII -> get ( Opc ) ) . addReg ( IdxReg , getKillRegState ( IdxRegKill ) ) . addJumpTableIndex ( JTI , JTOP . getTargetFlags ( ) ) . addImm ( MI -> getOperand ( JTOpIdx + <NUM_LIT> ) . getImm ( ) ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> << * NewJTMI ) ; unsigned NewSize = TII -> GetInstSizeInBytes ( NewJTMI ) ; unsigned OrigSize = TII -> GetInstSizeInBytes ( AddrMI ) ; OrigSize += TII -> GetInstSizeInBytes ( LeaMI ) ; OrigSize += TII -> GetInstSizeInBytes ( MI ) ; AddrMI -> eraseFromParent ( ) ; LeaMI -> eraseFromParent ( ) ; MI -> eraseFromParent ( ) ; int delta = OrigSize - NewSize ; BBInfo [ MBB -> getNumber ( ) ] . Size -= delta ; adjustBBOffsetsAfter ( MBB ) ; ++ NumTBs ; MadeChange = true ; } } return MadeChange ;


<s> assert ( JTI < JT . size ( ) ) ; bool ByteOk = true ; bool HalfWordOk = true ; unsigned JTOffset = getOffsetOf ( MI ) + <NUM_LIT> ; const std :: vector < MachineBasicBlock * > & JTBBs = JT [ JTI ] . MBBs ; for ( unsigned j = <NUM_LIT> , ee = JTBBs . size ( ) ; j != ee ; ++ j ) { MachineBasicBlock * MBB = JTBBs [ j ] ; unsigned DstOffset = BBInfo [ MBB -> getNumber ( ) ] . Offset ; if ( ByteOk && ( DstOffset - JTOffset ) > ( ( <NUM_LIT> << <NUM_LIT> ) - <NUM_LIT> ) * <NUM_LIT> ) ByteOk = false ; unsigned TBHLimit = ( ( <NUM_LIT> << <NUM_LIT> ) - <NUM_LIT> ) * <NUM_LIT> ; if ( HalfWordOk && ( DstOffset - JTOffset ) > TBHLimit ) HalfWordOk = false ; if ( ! ByteOk && ! HalfWordOk ) break ; } if ( ByteOk || HalfWordOk ) { MachineBasicBlock * MBB = MI -> getParent ( ) ; unsigned BaseReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool BaseRegKill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; if ( ! BaseRegKill ) continue ; unsigned IdxReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool IdxRegKill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; MachineBasicBlock :: iterator PrevI = MI ; MachineBasicBlock :: iterator B = MBB -> begin ( ) ; while ( PrevI != B && ! PrevI -> definesRegister ( BaseReg ) ) -- PrevI ; if ( ! PrevI -> definesRegister ( BaseReg ) ) continue ; MachineInstr * AddrMI = PrevI ;


<s> unsigned NumOps = MCID . getNumOperands ( ) ; unsigned JTOpIdx = NumOps - ( MI -> isPredicable ( ) ? <NUM_LIT> : <NUM_LIT> ) ; MachineOperand JTOP = MI -> getOperand ( JTOpIdx ) ; unsigned JTI = JTOP . getIndex ( ) ; assert ( JTI < JT . size ( ) ) ; int JTNumber = MI -> getParent ( ) -> getNumber ( ) ; const std :: vector < MachineBasicBlock * > & JTBBs = JT [ JTI ] . MBBs ; for ( unsigned j = <NUM_LIT> , ee = JTBBs . size ( ) ; j != ee ; ++ j ) { MachineBasicBlock * MBB = JTBBs [ j ] ; int DTNumber = MBB -> getNumber ( ) ; if ( DTNumber < JTNumber ) {

MachineBasicBlock * NewBB = adjustJTTargetBlockForward ( MBB , MI -> getParent ( ) ) ;


<s> int JTNumber = MI -> getParent ( ) -> getNumber ( ) ; const std :: vector < MachineBasicBlock * > & JTBBs = JT [ JTI ] . MBBs ; for ( unsigned j = <NUM_LIT> , ee = JTBBs . size ( ) ; j != ee ; ++ j ) { MachineBasicBlock * MBB = JTBBs [ j ] ; int DTNumber = MBB -> getNumber ( ) ; if ( DTNumber < JTNumber ) { MachineBasicBlock * NewBB = adjustJTTargetBlockForward ( MBB , MI -> getParent ( ) ) ; if ( NewBB ) MJTI -> ReplaceMBBInJumpTable ( JTI , JTBBs [ j ] , NewBB ) ;


<s> unsigned NoCPIters = <NUM_LIT> , NoBRIters = <NUM_LIT> ; while ( true ) { DEBUG ( dbgs ( ) << <STR_LIT> << NoCPIters << '\n' ) ; bool CPChange = false ; for ( unsigned i = <NUM_LIT> , e = CPUsers . size ( ) ; i != e ; ++ i ) CPChange |= handleConstantPoolUser ( i ) ; if ( CPChange && ++ NoCPIters > <NUM_LIT> ) report_fatal_error ( <STR_LIT> ) ; DEBUG ( dumpBBs ( ) ) ; NewWaterList . clear ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> << NoBRIters << '\n' ) ; bool BRChange = false ; for ( unsigned i = <NUM_LIT> , e = ImmBranches . size ( ) ; i != e ; ++ i ) BRChange |= fixupImmediateBr ( ImmBranches [ i ] ) ; if ( BRChange && ++ NoBRIters > <NUM_LIT> ) report_fatal_error ( <STR_LIT> ) ; DEBUG ( dumpBBs ( ) ) ; if ( ! CPChange && ! BRChange ) break ; MadeChange = true ; } if ( isThumb2 && ! STI -> prefers32BitThumb ( ) ) MadeChange |= optimizeThumb2Instructions ( ) ; verify ( ) ; if ( isThumb && ! HasFarJump && AFI -> isLRSpilledForFarJump ( ) ) MadeChange |= undoLRSpillRestore ( ) ; for ( unsigned i = <NUM_LIT> , e = CPEntries . size ( ) ; i != e ; ++ i ) { for ( unsigned j = <NUM_LIT> , je = CPEntries [ i ] . size ( ) ; j != je ; ++ j ) { const CPEntry & CPE = CPEntries [ i ] [ j ] ; AFI -> recordCPEClone ( i , CPE . CPI ) ; } } DEBUG ( dbgs ( ) << '\n' ; dumpBBs ( ) ) ; BBInfo . clear ( ) ; WaterList . clear ( ) ; CPUsers . clear ( ) ; CPEntries . clear ( ) ;


<s> bool IdxRegKill = true ; if ( isThumb2 ) { IdxReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; IdxRegKill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; bool PreservedBaseReg = preserveBaseRegister ( MI , User . MI , DeadSize , CanDeleteLEA , BaseRegKill ) ; if ( ! jumpTableFollowsTB ( MI , User . CPEMI ) && ! PreservedBaseReg ) continue ; } else { unsigned BaseReg = User . MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( User . MI -> getIterator ( ) == User . MI -> getParent ( ) -> begin ( ) ) continue ; MachineInstr * Shift = User . MI -> getPrevNode ( ) ; if ( Shift -> getOpcode ( ) != <STR_LIT> || Shift -> getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> || ! Shift -> getOperand ( <NUM_LIT> ) . isKill ( ) ) continue ; IdxReg = Shift -> getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned ShiftedIdxReg = Shift -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( BaseReg == IdxReg && ! jumpTableFollowsTB ( MI , User . CPEMI ) ) continue ; MachineInstr * Load = User . MI -> getNextNode ( ) ; if ( Load -> getOpcode ( ) != <STR_LIT> ) continue ; if ( Load -> getOperand ( <NUM_LIT> ) . getReg ( ) != BaseReg || Load -> getOperand ( <NUM_LIT> ) . getReg ( ) != ShiftedIdxReg || ! Load -> getOperand ( <NUM_LIT> ) . isKill ( ) ) continue ; auto * TRI = STI -> getRegisterInfo ( ) ; if ( registerDefinedBetween ( BaseReg , Load -> getNextNode ( ) , MBB -> end ( ) , TRI ) ) continue ; if ( isPositionIndependent_ROPI_PIP ) { MachineInstr * Add = Load -> getNextNode ( ) ; if ( Add -> getOpcode ( ) != <STR_LIT> || Add -> getOperand ( <NUM_LIT> ) . getReg ( ) != BaseReg || Add -> getOperand ( <NUM_LIT> ) . getReg ( ) != Load -> getOperand ( <NUM_LIT> ) . getReg ( ) || ! Add -> getOperand ( <NUM_LIT> ) . isKill ( ) ) continue ; if ( Add -> getOperand ( <NUM_LIT> ) . getReg ( ) != MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) continue ; if ( registerDefinedBetween ( IdxReg , Add -> getNextNode ( ) , MI , TRI ) ) continue ; Add -> eraseFromParent ( ) ; DeadSize += <NUM_LIT> ; } else {

if ( Load -> getOperand ( <NUM_LIT> ) . getReg ( ) != MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) continue ;


<s> MachineBasicBlock * <ISA_LIT>ConstantIslands :: adjustJTTargetBlockForward ( unsigned JTI , MachineBasicBlock * BB , MachineBasicBlock * JTBB ) { MachineBasicBlock * TBB = nullptr , * FBB = nullptr ; SmallVector < MachineOperand , <NUM_LIT> > Cond ; SmallVector < MachineOperand , <NUM_LIT> > CondPrior ; MachineFunction :: iterator BBi = BB -> getIterator ( ) ; MachineFunction :: iterator OldPrior = std :: prev ( BBi ) ; MachineFunction :: iterator OldNext = std :: next ( BBi ) ; bool B = TII -> analyzeBranch ( * BB , TBB , FBB , Cond ) ; if ( ! B && Cond . empty ( ) && BB != & MF -> front ( ) && ! TII -> analyzeBranch ( * OldPrior , TBB , FBB , CondPrior ) ) { BB -> moveAfter ( JTBB ) ; OldPrior -> updateTerminator ( BB ) ; BB -> updateTerminator ( OldNext != MF -> end ( ) ? & * OldNext : nullptr ) ; MF -> RenumberBlocks ( ) ; ++ NumJTMoved ; return nullptr ; } MachineBasicBlock * NewBB = MF -> CreateMachineBasicBlock ( JTBB -> getBasicBlock ( ) ) ; MachineFunction :: iterator MBBI = ++ JTBB -> getIterator ( ) ; MF -> insert ( MBBI , NewBB ) ; for ( const MachineBasicBlock :: RegisterMaskPair & RegMaskPair : BB -> liveins ( ) ) NewBB -> addLiveIn ( RegMaskPair ) ; if ( isThumb2 ) BuildMI ( NewBB , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( BB ) . add ( predOps ( <STR_LIT> ) ) ; else BuildMI ( NewBB , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( BB ) . add ( predOps ( <STR_LIT> ) ) ; MF -> RenumberBlocks ( NewBB ) ; NewBB -> addSuccessor ( BB ) ; JTBB -> replaceSuccessor ( BB , NewBB ) ; if ( MF -> getInfo < <ISA_LIT>FunctionInfo > ( ) -> branchTargetEnforcement ( ) ) fixupBTI ( JTI , * BB , * NewBB ) ; ++ NumJTInserted ; return NewBB ;


<s> bool B = TII -> analyzeBranch ( * BB , TBB , FBB , Cond ) ; if ( ! B && Cond . empty ( ) && BB != & MF -> front ( ) && ! TII -> analyzeBranch ( * OldPrior , TBB , FBB , CondPrior ) ) { BB -> moveAfter ( JTBB ) ; OldPrior -> updateTerminator ( BB ) ; BB -> updateTerminator ( OldNext != MF -> end ( ) ? & * OldNext : nullptr ) ; MF -> RenumberBlocks ( ) ; ++ NumJTMoved ; return nullptr ; } MachineBasicBlock * NewBB = MF -> CreateMachineBasicBlock ( JTBB -> getBasicBlock ( ) ) ; MachineFunction :: iterator MBBI = ++ JTBB -> getIterator ( ) ; MF -> insert ( MBBI , NewBB ) ; for ( const MachineBasicBlock :: RegisterMaskPair & RegMaskPair : BB -> liveins ( ) ) NewBB -> addLiveIn ( RegMaskPair ) ; if ( isThumb2 ) BuildMI ( NewBB , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( BB ) . add ( predOps ( <STR_LIT> ) ) ; else BuildMI ( NewBB , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( BB ) . add ( predOps ( <STR_LIT> ) ) ; MF -> RenumberBlocks ( NewBB ) ; NewBB -> addSuccessor ( BB ) ; JTBB -> replaceSuccessor ( BB , NewBB ) ; if ( MF -> getInfo < <ISA_LIT>FunctionInfo > ( ) -> branchTargetEnforcement ( ) ) fixupBTI ( JTI , * BB , * NewBB ) ; ++ NumJTInserted ; return NewBB ;


<s> std :: vector < CPEntry > & CPEs = CPEntries [ CPI ] ; for ( CPEntry & CPE : CPEs ) if ( CPE . CPEMI == CPEMI ) return & CPE ;


<s> if ( isCPEntryInRange ( UserMI , UserOffset , CPE . CPEMI , U . getMaxDisp ( ) , U . NegOk ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << CPI << <STR_LIT> << CPE . CPI << <STR_LIT> ) ; U . CPEMI = CPE . CPEMI ; for ( MachineOperand & MO : UserMI -> operands ( ) ) if ( MO . isCPI ( ) ) { MO . setIndex ( CPE . CPI ) ; break ; }


<s> assert ( isThumb2 && <STR_LIT> ) ; BuildMI ( NewBB , NewBB . begin ( ) , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; const MachineJumpTableInfo & MJTI = * MF -> getJumpTableInfo ( ) ; const MachineJumpTableEntry & JTE = MJTI . getJumpTables ( ) [ JTI ] ; for ( const MachineBasicBlock * MBB : JTE . MBBs ) { if ( MBB != & OldBB ) continue ; -- BlockJumpTableRefCount [ MBB ] ;


<s> void <ISA_LIT>ConstantIslands :: fixupBTI ( unsigned JTI , MachineBasicBlock & OldBB , MachineBasicBlock & NewBB ) { assert ( isThumb2 && <STR_LIT> ) ; BuildMI ( NewBB , NewBB . begin ( ) , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; const MachineJumpTableInfo & MJTI = * MF -> getJumpTableInfo ( ) ; const MachineJumpTableEntry & JTE = MJTI . getJumpTables ( ) [ JTI ] ; for ( const MachineBasicBlock * MBB : JTE . MBBs ) { if ( MBB != & OldBB ) continue ;


<s> unsigned Opcode = U . MI -> getOpcode ( ) ; unsigned NewOpc = <NUM_LIT> ; unsigned Scale = <NUM_LIT> ; unsigned Bits = <NUM_LIT> ; switch ( Opcode ) { default : break ; case <STR_LIT> : if ( is<ISA_LIT>LowRegister ( U . MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { NewOpc = <STR_LIT> ; Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; } break ; case <STR_LIT> : if ( is<ISA_LIT>LowRegister ( U . MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { NewOpc = <STR_LIT> ; Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; }


<s> unsigned Opcode = U . MI -> getOpcode ( ) ; unsigned NewOpc = <NUM_LIT> ; unsigned Scale = <NUM_LIT> ; unsigned Bits = <NUM_LIT> ; switch ( Opcode ) { default : break ; case <STR_LIT> : if ( is<ISA_LIT>LowRegister ( U . MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { NewOpc = <STR_LIT> ; Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; } break ; case <STR_LIT> : if ( is<ISA_LIT>LowRegister ( U . MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { NewOpc = <STR_LIT> ; Bits = <NUM_LIT> ; Scale = <NUM_LIT> ;


<s> bool <ISA_LIT>ConstantIslands :: removeUnusedCPEntries ( ) { unsigned MadeChange = false ; for ( std :: vector < CPEntry > & CPEs : CPEntries ) { for ( CPEntry & CPE : CPEs ) { if ( CPE . RefCount == <NUM_LIT> && CPE . CPEMI ) { removeDeadCPEMI ( CPE . CPEMI ) ; CPE . CPEMI = nullptr ; MadeChange = true ; }


if ( CPE . RefCount == <NUM_LIT> && CPE . CPEMI ) {


<s> const std :: vector < MachineBasicBlock * > & JTBBs = JT [ JTI ] . MBBs ; for ( MachineBasicBlock * MBB : JTBBs ) { int DTNumber = MBB -> getNumber ( ) ; if ( DTNumber < JTNumber ) { MachineBasicBlock * NewBB = adjustJTTargetBlockForward ( JTI , MBB , MI -> getParent ( ) ) ; if ( NewBB ) MJTI -> ReplaceMBBInJumpTable ( JTI , MBB , NewBB ) ; MadeChange = true ; } } } return MadeChange ;


<s> unsigned JTOpIdx = NumOps - ( MI -> isPredicable ( ) ? <NUM_LIT> : <NUM_LIT> ) ; MachineOperand JTOP = MI -> getOperand ( JTOpIdx ) ; unsigned JTI = JTOP . getIndex ( ) ; assert ( JTI < JT . size ( ) ) ; int JTNumber = MI -> getParent ( ) -> getNumber ( ) ; const std :: vector < MachineBasicBlock * > & JTBBs = JT [ JTI ] . MBBs ;

for ( MachineBasicBlock * MBB : JTBBs ) {


<s> bool GenerateTBB = isThumb2 || ( isThumb1 && SynthesizeThumb1TBB ) ; if ( STI -> hardenSlsRetBr ( ) ) GenerateTBB = false ; MF -> RenumberBlocks ( ) ; bool MadeChange = false ; if ( GenerateTBB && AdjustJumpTableBlocks ) { scanFunctionJumpTables ( ) ; MadeChange |= reorderThumb2JumpTables ( ) ; T2JumpTables . clear ( ) ; MF -> RenumberBlocks ( ) ; } MadeChange |= AlignBlocks ( MF , STI ) ; std :: vector < MachineInstr * > CPEMIs ; if ( ! MCP -> isEmpty ( ) ) doInitialConstPlacement ( CPEMIs ) ; if ( MF -> getJumpTableInfo ( ) ) doInitialJumpTablePlacement ( CPEMIs ) ; AFI -> initPICLabelUId ( CPEMIs . size ( ) ) ; initializeFunctionInfo ( CPEMIs ) ; CPEMIs . clear ( ) ; LLVM_DEBUG ( dumpBBs ( ) ) ; if ( ! T2JumpTables . empty ( ) ) MF -> ensureAlignment ( Align ( <NUM_LIT> ) ) ; MadeChange |= removeUnusedCPEntries ( ) ; unsigned NoCPIters = <NUM_LIT> , NoBRIters = <NUM_LIT> ; while ( true ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << NoCPIters << '\n' ) ; bool CPChange = false ; for ( unsigned i = <NUM_LIT> , e = CPUsers . size ( ) ; i != e ; ++ i ) CPChange |= handleConstantPoolUser ( i , NoCPIters >= CPMaxIteration / <NUM_LIT> ) ; if ( CPChange && ++ NoCPIters > CPMaxIteration ) report_fatal_error ( <STR_LIT> ) ; LLVM_DEBUG ( dumpBBs ( ) ) ; NewWaterList . clear ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << NoBRIters << '\n' ) ; bool BRChange = false ; for ( unsigned i = <NUM_LIT> , e = ImmBranches . size ( ) ; i != e ; ++ i ) BRChange |= fixupImmediateBr ( ImmBranches [ i ] ) ; if ( BRChange && ++ NoBRIters > <NUM_LIT> ) report_fatal_error ( <STR_LIT> ) ; LLVM_DEBUG ( dumpBBs ( ) ) ; if ( ! CPChange && ! BRChange ) break ; MadeChange = true ; } if ( isThumb2 && ! STI -> prefers32BitThumb ( ) ) MadeChange |= optimizeThumb2Instructions ( ) ; if ( isThumb && STI -> hasV8MBaselineOps ( ) ) MadeChange |= optimizeThumb2Branches ( ) ; if ( GenerateTBB && ! STI -> genExecuteOnly ( ) ) MadeChange |= optimizeThumb2JumpTables ( ) ; verify ( ) ; for ( unsigned i = <NUM_LIT> , e = CPEntries . size ( ) ; i != e ; ++ i ) { for ( unsigned j = <NUM_LIT> , je = CPEntries [ i ] . size ( ) ; j != je ; ++ j ) { const CPEntry & CPE = CPEntries [ i ] [ j ] ; if ( CPE . CPEMI && CPE . CPEMI -> getOperand ( <NUM_LIT> ) . isCPI ( ) ) AFI -> recordCPEClone ( i , CPE . CPI ) ;


<s> if ( ! MF -> getInfo < <ISA_LIT>FunctionInfo > ( ) -> branchTargetEnforcement ( ) ) return ; if ( const MachineJumpTableInfo * JTI = MF -> getJumpTableInfo ( ) ) for ( const MachineJumpTableEntry & JTE : JTI -> getJumpTables ( ) ) for ( const MachineBasicBlock * MBB : JTE . MBBs ) { if ( isAlwaysIndirectTarget ( * MBB ) ) BlockJumpTableRefCount [ MBB ] = std :: numeric_limits < int > :: max ( ) ;

else ++ BlockJumpTableRefCount [ MBB ] ;


<s> for ( CPUser & U : CPUsers ) { unsigned UserOffset = getUserOffset ( U ) ; if ( isCPEntryInRange ( U . MI , UserOffset , U . CPEMI , U . getMaxDisp ( ) + <NUM_LIT> , U . NegOk , true ) ) {


<s> LLVM_DEBUG ( dbgs ( ) << format ( <STR_LIT> , BaseInsertOffset ) ) ; } unsigned EndInsertOffset = BaseInsertOffset + <NUM_LIT> + UPad + CPEMI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; MachineBasicBlock :: iterator MI = UserMI ; ++ MI ; unsigned CPUIndex = CPUserIndex + <NUM_LIT> ; unsigned NumCPUsers = CPUsers . size ( ) ; MachineInstr * LastIT = nullptr ; for ( unsigned Offset = UserOffset + TII -> getInstSizeInBytes ( * UserMI ) ; Offset < BaseInsertOffset ; Offset += TII -> getInstSizeInBytes ( * MI ) , MI = std :: next ( MI ) ) { assert ( MI != UserMBB -> end ( ) && <STR_LIT> ) ; if ( CPUIndex < NumCPUsers && CPUsers [ CPUIndex ] . MI == & * MI ) { CPUser & U = CPUsers [ CPUIndex ] ; if ( ! isOffsetInRange ( Offset , EndInsertOffset , U ) ) { BaseInsertOffset -= <NUM_LIT> << LogAlign ; EndInsertOffset -= <NUM_LIT> << LogAlign ; } EndInsertOffset += U . CPEMI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; CPUIndex ++ ; } if ( MI -> getOpcode ( ) == <STR_LIT> ) LastIT = & * MI ; } -- MI ; if ( LastIT ) {


<s> unsigned UPad = UnknownPadding ( LogAlign , KnownBits ) ; unsigned BaseInsertOffset = UserOffset + U . getMaxDisp ( ) - UPad ; LLVM_DEBUG ( dbgs ( ) << format ( <STR_LIT> , BaseInsertOffset ) ) ; BaseInsertOffset -= <NUM_LIT> ; LLVM_DEBUG ( dbgs ( ) << format ( <STR_LIT> , BaseInsertOffset ) << <STR_LIT> << LogAlign << <STR_LIT> << KnownBits << <STR_LIT> << UPad << '\n' ) ; if ( BaseInsertOffset + <NUM_LIT> >= UserBBI . postOffset ( ) ) { BaseInsertOffset = std :: max ( UserBBI . postOffset ( ) - UPad - <NUM_LIT> , UserOffset + TII -> getInstSizeInBytes ( * UserMI ) + <NUM_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << format ( <STR_LIT> , BaseInsertOffset ) ) ; } unsigned EndInsertOffset = BaseInsertOffset + <NUM_LIT> + UPad + CPEMI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; MachineBasicBlock :: iterator MI = UserMI ; ++ MI ; unsigned CPUIndex = CPUserIndex + <NUM_LIT> ; unsigned NumCPUsers = CPUsers . size ( ) ; MachineInstr * LastIT = nullptr ; for ( unsigned Offset = UserOffset + TII -> getInstSizeInBytes ( * UserMI ) ; Offset < BaseInsertOffset ; Offset += TII -> getInstSizeInBytes ( * MI ) , MI = std :: next ( MI ) ) { assert ( MI != UserMBB -> end ( ) && <STR_LIT> ) ; if ( CPUIndex < NumCPUsers && CPUsers [ CPUIndex ] . MI == & * MI ) { CPUser & U = CPUsers [ CPUIndex ] ; if ( ! isOffsetInRange ( Offset , EndInsertOffset , U ) ) { BaseInsertOffset -= <NUM_LIT> << LogAlign ;

EndInsertOffset -= <NUM_LIT> << LogAlign ;


<s> BuildMI ( MI -> getParent ( ) , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . add ( MI -> getOperand ( <NUM_LIT> ) ) . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MI -> eraseFromParent ( ) ; MadeChange = true ; } if ( MI -> getOpcode ( ) == <STR_LIT> && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && MI -> getNumExplicitOperands ( ) == <NUM_LIT> ) { MI -> eraseFromParent ( ) ;


<s> MachineInstr * MI = PushPopMIs [ i ] ; if ( MI -> getOpcode ( ) == <STR_LIT> && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && MI -> getNumExplicitOperands ( ) == <NUM_LIT> ) { BuildMI ( MI -> getParent ( ) , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . add ( MI -> getOperand ( <NUM_LIT> ) ) . add ( MI -> getOperand ( <NUM_LIT> ) ) ; MI -> eraseFromParent ( ) ; MadeChange = true ; } if ( MI -> getOpcode ( ) == <STR_LIT> && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && MI -> getNumExplicitOperands ( ) == <NUM_LIT> ) { MI -> eraseFromParent ( ) ; MadeChange = true ; } } return MadeChange ;


<s> if ( std :: find ( MBB -> succ_begin ( ) , MBB -> succ_end ( ) , NextBB ) == MBB -> succ_end ( ) ) return false ; MachineBasicBlock * TBB , * FBB ; SmallVector < MachineOperand , <NUM_LIT> > Cond ; bool TooDifficult = TII -> AnalyzeBranch ( * MBB , TBB , FBB , Cond ) ; return TooDifficult || FBB == nullptr ;


<s> } unsigned LogAlign = MF -> getAlignment ( ) ; assert ( LogAlign >= CPELogAlign && <STR_LIT> ) ; unsigned KnownBits = UserBBI . internalKnownBits ( ) ; unsigned UPad = UnknownPadding ( LogAlign , KnownBits ) ; unsigned BaseInsertOffset = UserOffset + U . getMaxDisp ( ) - UPad ; DEBUG ( dbgs ( ) << format ( <STR_LIT> , BaseInsertOffset ) ) ; BaseInsertOffset -= <NUM_LIT> ; DEBUG ( dbgs ( ) << format ( <STR_LIT> , BaseInsertOffset ) << <STR_LIT> << LogAlign << <STR_LIT> << KnownBits << <STR_LIT> << UPad << '\n' ) ; if ( BaseInsertOffset + <NUM_LIT> >= UserBBI . postOffset ( ) ) { BaseInsertOffset = std :: max ( UserBBI . postOffset ( ) - UPad - <NUM_LIT> , UserOffset + TII -> GetInstSizeInBytes ( UserMI ) + <NUM_LIT> ) ; DEBUG ( dbgs ( ) << format ( <STR_LIT> , BaseInsertOffset ) ) ; } unsigned EndInsertOffset = BaseInsertOffset + <NUM_LIT> + UPad + CPEMI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; MachineBasicBlock :: iterator MI = UserMI ; ++ MI ; unsigned CPUIndex = CPUserIndex + <NUM_LIT> ; unsigned NumCPUsers = CPUsers . size ( ) ; MachineInstr * LastIT = nullptr ; for ( unsigned Offset = UserOffset + TII -> GetInstSizeInBytes ( UserMI ) ; Offset < BaseInsertOffset ; Offset += TII -> GetInstSizeInBytes ( MI ) , MI = std :: next ( MI ) ) { assert ( MI != UserMBB -> end ( ) && <STR_LIT> ) ; if ( CPUIndex < NumCPUsers && CPUsers [ CPUIndex ] . MI == MI ) { CPUser & U = CPUsers [ CPUIndex ] ; if ( ! isOffsetInRange ( Offset , EndInsertOffset , U ) ) { BaseInsertOffset -= <NUM_LIT> << LogAlign ; EndInsertOffset -= <NUM_LIT> << LogAlign ; } EndInsertOffset += U . CPEMI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; CPUIndex ++ ; } if ( MI -> getOpcode ( ) == <STR_LIT> ) LastIT = MI ; } -- MI ; if ( LastIT ) { unsigned PredReg = <NUM_LIT> ;

<STR_LIT> CC = getITInstrPredicate ( MI , PredReg ) ;


<s> unsigned Size = TD . getTypeAllocSize ( CPs [ i ] . getType ( ) ) ; assert ( Size >= <NUM_LIT> && <STR_LIT> ) ; unsigned Align = CPs [ i ] . getAlignment ( ) ; assert ( isPowerOf2_32 ( Align ) && <STR_LIT> ) ; assert ( ( Size % Align ) == <NUM_LIT> && <STR_LIT> ) ; unsigned LogAlign = Log2_32 ( Align ) ; MachineBasicBlock :: iterator InsAt = InsPoint [ LogAlign ] ; MachineInstr * CPEMI = BuildMI ( * BB , InsAt , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( i ) . addConstantPoolIndex ( i ) . addImm ( Size ) ; CPEMIs . push_back ( CPEMI ) ; for ( unsigned a = LogAlign + <NUM_LIT> ; a <= MaxAlign ; ++ a ) if ( InsPoint [ a ] == InsAt ) InsPoint [ a ] = CPEMI ; CPEntries . emplace_back ( <NUM_LIT> , CPEntry ( CPEMI , i ) ) ; ++ NumCPEs ; DEBUG ( dbgs ( ) << <STR_LIT> << i << <STR_LIT> << Size << <STR_LIT> << Align << '\n' ) ;


<s> assert ( Size >= <NUM_LIT> && <STR_LIT> ) ; unsigned Align = CPs [ i ] . getAlignment ( ) ; assert ( isPowerOf2_32 ( Align ) && <STR_LIT> ) ; assert ( ( Size % Align ) == <NUM_LIT> && <STR_LIT> ) ; unsigned LogAlign = Log2_32 ( Align ) ; MachineBasicBlock :: iterator InsAt = InsPoint [ LogAlign ] ; MachineInstr * CPEMI = BuildMI ( * BB , InsAt , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( i ) . addConstantPoolIndex ( i ) . addImm ( Size ) ; CPEMIs . push_back ( CPEMI ) ; for ( unsigned a = LogAlign + <NUM_LIT> ; a <= MaxAlign ; ++ a ) if ( InsPoint [ a ] == InsAt ) InsPoint [ a ] = CPEMI ; CPEntries . emplace_back ( <NUM_LIT> , CPEntry ( CPEMI , i ) ) ; ++ NumCPEs ; DEBUG ( dbgs ( ) << <STR_LIT> << i << <STR_LIT> << Size << <STR_LIT> << Align << '\n' ) ;


<s> MachineBasicBlock :: iterator B = MBB -> begin ( ) ; while ( PrevI != B && ! PrevI -> definesRegister ( BaseReg ) ) -- PrevI ; if ( ! PrevI -> definesRegister ( BaseReg ) ) continue ; MachineInstr * AddrMI = PrevI ; bool OptOk = true ; for ( unsigned k = <NUM_LIT> , eee = AddrMI -> getNumOperands ( ) ; k != eee ; ++ k ) { const MachineOperand & MO = AddrMI -> getOperand ( k ) ; if ( ! MO . isReg ( ) || ! MO . getReg ( ) ) continue ; if ( MO . isDef ( ) && MO . getReg ( ) != BaseReg ) { OptOk = false ; break ; } if ( MO . isUse ( ) && ! MO . isKill ( ) && MO . getReg ( ) != IdxReg ) { OptOk = false ; break ; } } if ( ! OptOk ) continue ; for ( -- PrevI ; PrevI != B && ! PrevI -> definesRegister ( BaseReg ) ; -- PrevI ) ; MachineInstr * LeaMI = PrevI ; if ( ( LeaMI -> getOpcode ( ) != <STR_LIT> && LeaMI -> getOpcode ( ) != <STR_LIT> ) || LeaMI -> getOperand ( <NUM_LIT> ) . getReg ( ) != BaseReg ) OptOk = false ; if ( ! OptOk ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << * MI << <STR_LIT> << * AddrMI << <STR_LIT> << * LeaMI ) ; unsigned Opc = ByteOk ? <STR_LIT> : <STR_LIT> ; MachineBasicBlock :: iterator MI_JT = MI ; MachineInstr * NewJTMI = BuildMI ( * MBB , MI_JT , MI -> getDebugLoc ( ) , TII -> get ( Opc ) ) . addReg ( IdxReg , getKillRegState ( IdxRegKill ) ) . addJumpTableIndex ( JTI , JTOP . getTargetFlags ( ) ) . addImm ( MI -> getOperand ( JTOpIdx + <NUM_LIT> ) . getImm ( ) ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> << * NewJTMI ) ; unsigned NewSize = TII -> GetInstSizeInBytes ( NewJTMI ) ; unsigned OrigSize = TII -> GetInstSizeInBytes ( AddrMI ) ; OrigSize += TII -> GetInstSizeInBytes ( LeaMI ) ; OrigSize += TII -> GetInstSizeInBytes ( MI ) ;


<s> bool MadeChange = false ; if ( isThumb2 && AdjustJumpTableBlocks ) { scanFunctionJumpTables ( ) ; MadeChange |= reorderThumb2JumpTables ( ) ; T2JumpTables . clear ( ) ; MF -> RenumberBlocks ( ) ; } if ( ! isThumb1 ) MF -> ensureAlignment ( <NUM_LIT> ) ; std :: vector < MachineInstr * > CPEMIs ; if ( ! MCP -> isEmpty ( ) ) doInitialPlacement ( CPEMIs ) ; AFI -> initPICLabelUId ( CPEMIs . size ( ) ) ; initializeFunctionInfo ( CPEMIs ) ; CPEMIs . clear ( ) ; DEBUG ( dumpBBs ( ) ) ; MadeChange |= removeUnusedCPEntries ( ) ; unsigned NoCPIters = <NUM_LIT> , NoBRIters = <NUM_LIT> ; while ( true ) { DEBUG ( dbgs ( ) << <STR_LIT> << NoCPIters << '\n' ) ; bool CPChange = false ; for ( unsigned i = <NUM_LIT> , e = CPUsers . size ( ) ; i != e ; ++ i ) CPChange |= handleConstantPoolUser ( i ) ; if ( CPChange && ++ NoCPIters > <NUM_LIT> ) report_fatal_error ( <STR_LIT> ) ; DEBUG ( dumpBBs ( ) ) ; NewWaterList . clear ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> << NoBRIters << '\n' ) ; bool BRChange = false ; for ( unsigned i = <NUM_LIT> , e = ImmBranches . size ( ) ; i != e ; ++ i ) BRChange |= fixupImmediateBr ( ImmBranches [ i ] ) ; if ( BRChange && ++ NoBRIters > <NUM_LIT> ) report_fatal_error ( <STR_LIT> ) ; DEBUG ( dumpBBs ( ) ) ; if ( ! CPChange && ! BRChange ) break ; MadeChange = true ; } if ( isThumb2 && ! STI -> prefers32BitThumb ( ) ) MadeChange |= optimizeThumb2Instructions ( ) ; verify ( ) ; if ( isThumb && ! HasFarJump && AFI -> isLRSpilledForFarJump ( ) ) MadeChange |= undoLRSpillRestore ( ) ; for ( unsigned i = <NUM_LIT> , e = CPEntries . size ( ) ; i != e ; ++ i ) { for ( unsigned j = <NUM_LIT> , je = CPEntries [ i ] . size ( ) ; j != je ; ++ j ) { const CPEntry & CPE = CPEntries [ i ] [ j ] ; AFI -> recordCPEClone ( i , CPE . CPI ) ; } } DEBUG ( dbgs ( ) << '\n' ; dumpBBs ( ) ) ; BBInfo . clear ( ) ; WaterList . clear ( ) ;


<s> for ( unsigned i = <NUM_LIT> , e = ImmBranches . size ( ) ; i != e ; ++ i ) BRChange |= fixupImmediateBr ( ImmBranches [ i ] ) ; if ( BRChange && ++ NoBRIters > <NUM_LIT> ) report_fatal_error ( <STR_LIT> ) ; DEBUG ( dumpBBs ( ) ) ; if ( ! CPChange && ! BRChange ) break ; MadeChange = true ; } if ( isThumb2 && ! STI -> prefers32BitThumb ( ) ) MadeChange |= optimizeThumb2Instructions ( ) ; verify ( ) ; if ( isThumb && ! HasFarJump && AFI -> isLRSpilledForFarJump ( ) ) MadeChange |= undoLRSpillRestore ( ) ; for ( unsigned i = <NUM_LIT> , e = CPEntries . size ( ) ; i != e ; ++ i ) { for ( unsigned j = <NUM_LIT> , je = CPEntries [ i ] . size ( ) ; j != je ; ++ j ) { const CPEntry & CPE = CPEntries [ i ] [ j ] ; AFI -> recordCPEClone ( i , CPE . CPI ) ; } } DEBUG ( dbgs ( ) << '\n' ; dumpBBs ( ) ) ; BBInfo . clear ( ) ; WaterList . clear ( ) ; CPUsers . clear ( ) ; CPEntries . clear ( ) ;


<s> STI = & MF -> getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = STI -> getInstrInfo ( ) ; isPositionIndependentOrROPI = STI -> getTargetLowering ( ) -> isPositionIndependent ( ) || STI -> isROPI ( ) ; AFI = MF -> getInfo < <ISA_LIT>FunctionInfo > ( ) ; DT = & getAnalysis < MachineDominatorTree > ( ) ; isThumb = AFI -> isThumbFunction ( ) ; isThumb1 = AFI -> isThumb1OnlyFunction ( ) ; isThumb2 = AFI -> isThumb2Function ( ) ; bool GenerateTBB = isThumb2 || ( isThumb1 && SynthesizeThumb1TBB ) ; if ( STI -> hardenSlsRetBr ( ) ) GenerateTBB = false ; MF -> RenumberBlocks ( ) ; bool MadeChange = false ; if ( GenerateTBB && AdjustJumpTableBlocks ) { scanFunctionJumpTables ( ) ; MadeChange |= reorderThumb2JumpTables ( ) ; T2JumpTables . clear ( ) ; MF -> RenumberBlocks ( ) ; } MadeChange |= AlignBlocks ( MF , STI ) ; std :: vector < MachineInstr * > CPEMIs ; if ( ! MCP -> isEmpty ( ) ) doInitialConstPlacement ( CPEMIs ) ; if ( MF -> getJumpTableInfo ( ) ) doInitialJumpTablePlacement ( CPEMIs ) ; AFI -> initPICLabelUId ( CPEMIs . size ( ) ) ; initializeFunctionInfo ( CPEMIs ) ; CPEMIs . clear ( ) ; LLVM_DEBUG ( dumpBBs ( ) ) ; if ( ! T2JumpTables . empty ( ) ) MF -> ensureAlignment ( Align ( <NUM_LIT> ) ) ; MadeChange |= removeUnusedCPEntries ( ) ; unsigned NoCPIters = <NUM_LIT> , NoBRIters = <NUM_LIT> ; while ( true ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << NoCPIters << '\n' ) ; bool CPChange = false ; for ( unsigned i = <NUM_LIT> , e = CPUsers . size ( ) ; i != e ; ++ i ) CPChange |= handleConstantPoolUser ( i , NoCPIters >= CPMaxIteration / <NUM_LIT> ) ; if ( CPChange && ++ NoCPIters > CPMaxIteration ) report_fatal_error ( <STR_LIT> ) ; LLVM_DEBUG ( dumpBBs ( ) ) ;


<s> bool BRChange = false ; for ( unsigned i = <NUM_LIT> , e = ImmBranches . size ( ) ; i != e ; ++ i ) BRChange |= fixupImmediateBr ( ImmBranches [ i ] ) ; if ( BRChange && ++ NoBRIters > <NUM_LIT> ) report_fatal_error ( <STR_LIT> ) ; LLVM_DEBUG ( dumpBBs ( ) ) ; if ( ! CPChange && ! BRChange ) break ; MadeChange = true ; } if ( isThumb2 && ! STI -> prefers32BitThumb ( ) ) MadeChange |= optimizeThumb2Instructions ( ) ; if ( isThumb && STI -> hasV8MBaselineOps ( ) ) MadeChange |= optimizeThumb2Branches ( ) ; if ( GenerateTBB && ! STI -> genExecuteOnly ( ) ) MadeChange |= optimizeThumb2JumpTables ( ) ; verify ( ) ; for ( unsigned i = <NUM_LIT> , e = CPEntries . size ( ) ; i != e ; ++ i ) { for ( unsigned j = <NUM_LIT> , je = CPEntries [ i ] . size ( ) ; j != je ; ++ j ) { const CPEntry & CPE = CPEntries [ i ] [ j ] ; if ( CPE . CPEMI && CPE . CPEMI -> getOperand ( <NUM_LIT> ) . isCPI ( ) ) AFI -> recordCPEClone ( i , CPE . CPI ) ; } } LLVM_DEBUG ( dbgs ( ) << '\n' ; dumpBBs ( ) ) ; BBUtils -> clear ( ) ;


<s> } MachineBasicBlock * NewBB = MF -> CreateMachineBasicBlock ( JTBB -> getBasicBlock ( ) ) ; MachineFunction :: iterator MBBI = ++ JTBB -> getIterator ( ) ; MF -> insert ( MBBI , NewBB ) ; assert ( isThumb2 && <STR_LIT> ) ; BuildMI ( NewBB , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( BB ) . addImm ( <STR_LIT> ) . addReg ( <NUM_LIT> ) ; MF -> RenumberBlocks ( NewBB ) ; NewBB -> addSuccessor ( BB ) ;

JTBB -> replaceSuccessor ( BB , NewBB ) ;


<s> MachineBasicBlock * NextBB = & * std :: next ( MBBI ) ;

if ( std :: find ( MBB -> succ_begin ( ) , MBB -> succ_end ( ) , NextBB ) == MBB -> succ_end ( ) ) return false ;


<s> DEBUG ( dbgs ( ) << <STR_LIT> << DestBB -> getNumber ( ) << <STR_LIT> << NextBB -> getNumber ( ) << <STR_LIT> ) ; BuildMI ( MBB , DebugLoc ( ) , TII -> get ( MI -> getOpcode ( ) ) ) . addMBB ( NextBB ) . addImm ( CC ) . addReg ( CCReg ) ; Br . MI = & MBB -> back ( ) ; BBInfo [ MBB -> getNumber ( ) ] . Size += TII -> getInstSizeInBytes ( MBB -> back ( ) ) ; if ( isThumb ) BuildMI ( MBB , DebugLoc ( ) , TII -> get ( Br . UncondBr ) ) . addMBB ( DestBB ) . addImm ( <STR_LIT> ) . addReg ( <NUM_LIT> ) ; else BuildMI ( MBB , DebugLoc ( ) , TII -> get ( Br . UncondBr ) ) . addMBB ( DestBB ) ; BBInfo [ MBB -> getNumber ( ) ] . Size += TII -> getInstSizeInBytes ( MBB -> back ( ) ) ; unsigned MaxDisp = getUnconditionalBrDisp ( Br . UncondBr ) ; ImmBranches . push_back ( ImmBranch ( & MBB -> back ( ) , MaxDisp , false , Br . UncondBr ) ) ; BBInfo [ MI -> getParent ( ) -> getNumber ( ) ] . Size -= TII -> getInstSizeInBytes ( * MI ) ;


<s> return MachineFunctionProperties ( ) . set ( MachineFunctionProperties :: Property :: AllVRegsAllocated ) ;


<s> MachineFunctionProperties getRequiredProperties ( ) const override { return MachineFunctionProperties ( ) . set ( MachineFunctionProperties :: Property :: AllVRegsAllocated ) ;


<s> water_iterator IP ; if ( findAvailableWater ( U , UserOffset , IP , CloserWater ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; MachineBasicBlock * WaterBB = * IP ; if ( NewWaterList . erase ( WaterBB ) ) NewWaterList . insert ( NewIsland ) ; NewMBB = & * ++ WaterBB -> getIterator ( ) ; } else { DEBUG ( dbgs ( ) << <STR_LIT> ) ; createNewWater ( CPUserIndex , UserOffset , NewMBB ) ; MachineBasicBlock * WaterBB = & * -- NewMBB -> getIterator ( ) ; IP = std :: find ( WaterList . begin ( ) , WaterList . end ( ) , WaterBB ) ; if ( IP != WaterList . end ( ) ) NewWaterList . erase ( WaterBB ) ; NewWaterList . insert ( NewIsland ) ; } if ( IP != WaterList . end ( ) ) WaterList . erase ( IP ) ; MF -> insert ( NewMBB -> getIterator ( ) , NewIsland ) ; updateForInsertedWaterBlock ( NewIsland ) ; U . HighWaterMark = NewIsland ;

U . CPEMI = BuildMI ( NewIsland , DebugLoc ( ) , CPEMI -> getDesc ( ) ) . addImm ( ID ) . addOperand ( CPEMI -> getOperand ( <NUM_LIT> ) ) . addImm ( Size ) ;


<s> default : continue ; case <STR_LIT> : T2JumpTables . push_back ( & I ) ; continue ; case <STR_LIT> : isCond = true ; UOpc = <STR_LIT> ; case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : isCond = true ; UOpc = <STR_LIT> ; Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : isCond = true ; UOpc = <STR_LIT> ; Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; } unsigned MaxOffs = ( ( <NUM_LIT> << ( Bits - <NUM_LIT> ) ) - <NUM_LIT> ) * Scale ; ImmBranches . push_back ( ImmBranch ( & I , MaxOffs , isCond , UOpc ) ) ; } if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) PushPopMIs . push_back ( & I ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) continue ; for ( unsigned op = <NUM_LIT> , e = I . getNumOperands ( ) ; op != e ; ++ op ) if ( I . getOperand ( op ) . isCPI ( ) || I . getOperand ( op ) . isJTI ( ) ) { unsigned Bits = <NUM_LIT> ; unsigned Scale = <NUM_LIT> ; bool NegOk = false ; bool IsSoImm = false ; switch ( Opc ) { default : llvm_unreachable ( <STR_LIT> ) ; case <STR_LIT> : case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; NegOk = true ; IsSoImm = true ; break ; case <STR_LIT> :


<s> unsigned DestOffset = BBInfo [ DestBB -> getNumber ( ) ] . Offset ; if ( BrOffset < DestOffset && ( DestOffset - BrOffset ) <= <NUM_LIT> ) { MachineBasicBlock :: iterator CmpMI = Br . MI ; if ( CmpMI != Br . MI -> getParent ( ) -> begin ( ) ) { -- CmpMI ; if ( CmpMI -> getOpcode ( ) == <STR_LIT> ) { unsigned Reg = CmpMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; Pred = getInstrPredicate ( * CmpMI , PredReg ) ; if ( Pred == <STR_LIT> && CmpMI -> getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> && is<ISA_LIT>LowRegister ( Reg ) ) { MachineBasicBlock * MBB = Br . MI -> getParent ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * CmpMI << <STR_LIT> << * Br . MI ) ; MachineInstr * NewBR = BuildMI ( * MBB , CmpMI , Br . MI -> getDebugLoc ( ) , TII -> get ( NewOpc ) ) . addReg ( Reg ) . addMBB ( DestBB , Br . MI -> getOperand ( <NUM_LIT> ) . getTargetFlags ( ) ) ; CmpMI -> eraseFromParent ( ) ; Br . MI -> eraseFromParent ( ) ;


<s> unsigned DestOffset = BBInfo [ DestBB -> getNumber ( ) ] . Offset ; if ( BrOffset < DestOffset && ( DestOffset - BrOffset ) <= <NUM_LIT> ) { MachineBasicBlock :: iterator CmpMI = Br . MI ; if ( CmpMI != Br . MI -> getParent ( ) -> begin ( ) ) { -- CmpMI ; if ( CmpMI -> getOpcode ( ) == <STR_LIT> ) { unsigned Reg = CmpMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; Pred = getInstrPredicate ( * CmpMI , PredReg ) ; if ( Pred == <STR_LIT> && CmpMI -> getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> && is<ISA_LIT>LowRegister ( Reg ) ) { MachineBasicBlock * MBB = Br . MI -> getParent ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * CmpMI << <STR_LIT> << * Br . MI ) ;

MachineInstr * NewBR = BuildMI ( * MBB , CmpMI , Br . MI -> getDebugLoc ( ) , TII -> get ( NewOpc ) ) . addReg ( Reg ) . addMBB ( DestBB , Br . MI -> getOperand ( <NUM_LIT> ) . getTargetFlags ( ) ) ;


<s> if ( ByteOk && ( DstOffset - JTOffset ) > ( ( <NUM_LIT> << <NUM_LIT> ) - <NUM_LIT> ) * <NUM_LIT> ) ByteOk = false ; unsigned TBHLimit = ( ( <NUM_LIT> << <NUM_LIT> ) - <NUM_LIT> ) * <NUM_LIT> ; if ( HalfWordOk && ( DstOffset - JTOffset ) > TBHLimit ) HalfWordOk = false ; if ( ! ByteOk && ! HalfWordOk ) break ; } if ( ! ByteOk && ! HalfWordOk ) continue ; MachineBasicBlock * MBB = MI -> getParent ( ) ; if ( ! MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ) continue ; unsigned IdxReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool IdxRegKill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; CPUser & User = CPUsers [ JumpTableUserIndices [ JTI ] ] ; unsigned DeadSize = <NUM_LIT> ; bool CanDeleteLEA = false ; bool BaseRegKill = false ; bool PreservedBaseReg = preserveBaseRegister ( MI , User . MI , DeadSize , CanDeleteLEA , BaseRegKill ) ; if ( ! jumpTableFollowsTB ( MI , User . CPEMI ) && ! PreservedBaseReg ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; MachineInstr * CPEMI = User . CPEMI ; unsigned Opc = ByteOk ? <STR_LIT> : <STR_LIT> ; MachineBasicBlock :: iterator MI_JT = MI ; MachineInstr * NewJTMI = BuildMI ( * MBB , MI_JT , MI -> getDebugLoc ( ) , TII -> get ( Opc ) ) . addReg ( User . MI -> getOperand ( <NUM_LIT> ) . getReg ( ) , getKillRegState ( BaseRegKill ) ) . addReg ( IdxReg , getKillRegState ( IdxRegKill ) ) . addJumpTableIndex ( JTI , JTOP . getTargetFlags ( ) ) . addImm ( CPEMI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> << * NewJTMI ) ; unsigned JTOpc = ByteOk ? <STR_LIT> : <STR_LIT> ; CPEMI -> setDesc ( TII -> get ( JTOpc ) ) ; if ( jumpTableFollowsTB ( MI , User . CPEMI ) ) {

NewJTMI -> getOperand ( <NUM_LIT> ) . setReg ( <STR_LIT> ) ;


<s> MF -> RenumberBlocks ( ) ; } std :: vector < MachineInstr * > CPEMIs ; if ( ! MCP -> isEmpty ( ) ) doInitialConstPlacement ( CPEMIs ) ; if ( MF -> getJumpTableInfo ( ) ) doInitialJumpTablePlacement ( CPEMIs ) ; AFI -> initPICLabelUId ( CPEMIs . size ( ) ) ; initializeFunctionInfo ( CPEMIs ) ; CPEMIs . clear ( ) ; DEBUG ( dumpBBs ( ) ) ; if ( ! T2JumpTables . empty ( ) ) MF -> ensureAlignment ( <NUM_LIT> ) ; MadeChange |= removeUnusedCPEntries ( ) ; unsigned NoCPIters = <NUM_LIT> , NoBRIters = <NUM_LIT> ; while ( true ) { DEBUG ( dbgs ( ) << <STR_LIT> << NoCPIters << '\n' ) ; bool CPChange = false ; for ( unsigned i = <NUM_LIT> , e = CPUsers . size ( ) ; i != e ; ++ i ) CPChange |= handleConstantPoolUser ( i , NoCPIters >= CPMaxIteration / <NUM_LIT> ) ; if ( CPChange && ++ NoCPIters > CPMaxIteration ) report_fatal_error ( <STR_LIT> ) ; DEBUG ( dumpBBs ( ) ) ; NewWaterList . clear ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> << NoBRIters << '\n' ) ;


<s> for ( MachineInstr & I : MBB ) if ( I . isBranch ( ) && I . getOpcode ( ) == <STR_LIT> ) T2JumpTables . push_back ( & I ) ;


<s> BBInfo . insert ( BBInfo . begin ( ) + NewBB -> getNumber ( ) , BasicBlockInfo ( ) ) ; water_iterator IP = std :: lower_bound ( WaterList . begin ( ) , WaterList . end ( ) , OrigBB , CompareMBBNumbers ) ; MachineBasicBlock * WaterBB = * IP ; if ( WaterBB == OrigBB ) WaterList . insert ( std :: next ( IP ) , NewBB ) ; else WaterList . insert ( IP , OrigBB ) ; NewWaterList . insert ( OrigBB ) ; computeBlockSize ( MF , OrigBB , BBInfo [ OrigBB -> getNumber ( ) ] ) ; computeBlockSize ( MF , NewBB , BBInfo [ NewBB -> getNumber ( ) ] ) ; adjustBBOffsetsAfter ( OrigBB ) ; return NewBB ;


<s> Align Alignment = CPs [ i ] . getAlign ( ) ; assert ( isAligned ( Alignment , Size ) && <STR_LIT> ) ; unsigned LogAlign = Log2 ( Alignment ) ; MachineBasicBlock :: iterator InsAt = InsPoint [ LogAlign ] ; MachineInstr * CPEMI = BuildMI ( * BB , InsAt , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( i ) . addConstantPoolIndex ( i ) . addImm ( Size ) ; CPEMIs . push_back ( CPEMI ) ; for ( unsigned a = LogAlign + <NUM_LIT> ; a <= MaxLogAlign ; ++ a ) if ( InsPoint [ a ] == InsAt ) InsPoint [ a ] = CPEMI ; CPEntries . emplace_back ( <NUM_LIT> , CPEntry ( CPEMI , i ) ) ; ++ NumCPEs ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << i << <STR_LIT> << Size << <STR_LIT> << Alignment . value ( ) << '\n' ) ; } LLVM_DEBUG ( BB -> dump ( ) ) ;


<s> unsigned CPEOffset = UserBBI . postOffset ( CPEAlign ) + Delta ; if ( isOffsetInRange ( UserOffset , CPEOffset , U ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * UserMBB ) << format ( <STR_LIT> , CPEOffset ) ) ; NewMBB = & * ++ UserMBB -> getIterator ( ) ; int UncondBr = isThumb ? ( ( isThumb2 ) ? <STR_LIT> : <STR_LIT> ) : <STR_LIT> ; if ( ! isThumb ) BuildMI ( UserMBB , DebugLoc ( ) , TII -> get ( UncondBr ) ) . addMBB ( NewMBB ) ; else BuildMI ( UserMBB , DebugLoc ( ) , TII -> get ( UncondBr ) ) . addMBB ( NewMBB ) . add ( predOps ( <STR_LIT> ) ) ; unsigned MaxDisp = getUnconditionalBrDisp ( UncondBr ) ; ImmBranches . push_back ( ImmBranch ( & UserMBB -> back ( ) , MaxDisp , false , UncondBr ) ) ; BBUtils -> computeBlockSize ( UserMBB ) ; BBUtils -> adjustBBOffsetsAfter ( UserMBB ) ; return ; } } const Align Align = MF -> getAlignment ( ) ; assert ( Align >= CPEAlign && <STR_LIT> ) ; unsigned KnownBits = UserBBI . internalKnownBits ( ) ; unsigned UPad = UnknownPadding ( Align , KnownBits ) ; unsigned BaseInsertOffset = UserOffset + U . getMaxDisp ( ) - UPad ; LLVM_DEBUG ( dbgs ( ) << format ( <STR_LIT> , BaseInsertOffset ) ) ; BaseInsertOffset -= <NUM_LIT> ; LLVM_DEBUG ( dbgs ( ) << format ( <STR_LIT> , BaseInsertOffset ) << <STR_LIT> << Log2 ( Align ) << <STR_LIT> << KnownBits << <STR_LIT> << UPad << '\n' ) ; if ( BaseInsertOffset + <NUM_LIT> >= UserBBI . postOffset ( ) ) { BaseInsertOffset = std :: max ( UserBBI . postOffset ( ) - UPad - <NUM_LIT> , UserOffset + TII -> getInstSizeInBytes ( * UserMI ) + <NUM_LIT> ) ; MachineBasicBlock :: iterator I = UserMI ; ++ I ; for ( unsigned Offset = UserOffset + TII -> getInstSizeInBytes ( * UserMI ) , PredReg = <NUM_LIT> ; I -> getOpcode ( ) != <STR_LIT> && getITInstrPredicate ( * I , PredReg ) != <STR_LIT> ;

Offset += TII -> getInstSizeInBytes ( * I ) , I = std :: next ( I ) ) {


<s> assert ( isPowerOf2_32 ( Align ) && <STR_LIT> ) ; assert ( ( Size % Align ) == <NUM_LIT> && <STR_LIT> ) ; unsigned LogAlign = Log2_32 ( Align ) ; MachineBasicBlock :: iterator InsAt = InsPoint [ LogAlign ] ; MachineInstr * CPEMI = BuildMI ( * BB , InsAt , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( i ) . addConstantPoolIndex ( i ) . addImm ( Size ) ; CPEMIs . push_back ( CPEMI ) ;

for ( unsigned a = LogAlign + <NUM_LIT> ; a <= MaxLogAlign ; ++ a ) if ( InsPoint [ a ] == InsAt ) InsPoint [ a ] = CPEMI ;


<s> } unsigned CPI = getCombinedIndex ( CPEMI ) ; assert ( CPI < MCP -> getConstants ( ) . size ( ) && <STR_LIT> ) ; return Align ( MCP -> getConstants ( ) [ CPI ] . getAlignment ( ) ) ;


<s> initializeFunctionInfo ( CPEMIs ) ; CPEMIs . clear ( ) ; LLVM_DEBUG ( dumpBBs ( ) ) ; if ( ! T2JumpTables . empty ( ) ) MF -> ensureAlignment ( Align ( <NUM_LIT> ) ) ; MadeChange |= removeUnusedCPEntries ( ) ; unsigned NoCPIters = <NUM_LIT> , NoBRIters = <NUM_LIT> ; while ( true ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << NoCPIters << '\n' ) ; bool CPChange = false ; for ( unsigned i = <NUM_LIT> , e = CPUsers . size ( ) ; i != e ; ++ i ) CPChange |= handleConstantPoolUser ( i , NoCPIters >= CPMaxIteration / <NUM_LIT> ) ; if ( CPChange && ++ NoCPIters > CPMaxIteration ) report_fatal_error ( <STR_LIT> ) ; LLVM_DEBUG ( dumpBBs ( ) ) ; NewWaterList . clear ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << NoBRIters << '\n' ) ; bool BRChange = false ; for ( unsigned i = <NUM_LIT> , e = ImmBranches . size ( ) ; i != e ; ++ i ) BRChange |= fixupImmediateBr ( ImmBranches [ i ] ) ; if ( BRChange && ++ NoBRIters > <NUM_LIT> ) report_fatal_error ( <STR_LIT> ) ; LLVM_DEBUG ( dumpBBs ( ) ) ; if ( ! CPChange && ! BRChange ) break ; MadeChange = true ; } if ( isThumb2 && ! STI -> prefers32BitThumb ( ) ) MadeChange |= optimizeThumb2Instructions ( ) ; if ( isThumb && STI -> hasV8MBaselineOps ( ) ) MadeChange |= optimizeThumb2Branches ( ) ; if ( GenerateTBB && ! STI -> genExecuteOnly ( ) ) MadeChange |= optimizeThumb2JumpTables ( ) ; verify ( ) ; for ( unsigned i = <NUM_LIT> , e = CPEntries . size ( ) ; i != e ; ++ i ) { for ( unsigned j = <NUM_LIT> , je = CPEntries [ i ] . size ( ) ; j != je ; ++ j ) { const CPEntry & CPE = CPEntries [ i ] [ j ] ; if ( CPE . CPEMI && CPE . CPEMI -> getOperand ( <NUM_LIT> ) . isCPI ( ) ) AFI -> recordCPEClone ( i , CPE . CPI ) ; } }

LLVM_DEBUG ( dbgs ( ) << '\n' ; dumpBBs ( ) ) ;


<s> AFI -> initPICLabelUId ( CPEMIs . size ( ) ) ; initializeFunctionInfo ( CPEMIs ) ; CPEMIs . clear ( ) ; LLVM_DEBUG ( dumpBBs ( ) ) ; if ( ! T2JumpTables . empty ( ) ) MF -> ensureAlignment ( Align ( <NUM_LIT> ) ) ; MadeChange |= removeUnusedCPEntries ( ) ; unsigned NoCPIters = <NUM_LIT> , NoBRIters = <NUM_LIT> ; while ( true ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << NoCPIters << '\n' ) ; bool CPChange = false ; for ( unsigned i = <NUM_LIT> , e = CPUsers . size ( ) ; i != e ; ++ i ) CPChange |= handleConstantPoolUser ( i , NoCPIters >= CPMaxIteration / <NUM_LIT> ) ; if ( CPChange && ++ NoCPIters > CPMaxIteration ) report_fatal_error ( <STR_LIT> ) ; LLVM_DEBUG ( dumpBBs ( ) ) ; NewWaterList . clear ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << NoBRIters << '\n' ) ; bool BRChange = false ; for ( unsigned i = <NUM_LIT> , e = ImmBranches . size ( ) ; i != e ; ++ i ) BRChange |= fixupImmediateBr ( ImmBranches [ i ] ) ; if ( BRChange && ++ NoBRIters > <NUM_LIT> ) report_fatal_error ( <STR_LIT> ) ; LLVM_DEBUG ( dumpBBs ( ) ) ; if ( ! CPChange && ! BRChange ) break ; MadeChange = true ; } if ( isThumb2 && ! STI -> prefers32BitThumb ( ) ) MadeChange |= optimizeThumb2Instructions ( ) ; if ( isThumb && STI -> hasV8MBaselineOps ( ) ) MadeChange |= optimizeThumb2Branches ( ) ; if ( GenerateTBB && ! STI -> genExecuteOnly ( ) ) MadeChange |= optimizeThumb2JumpTables ( ) ; verify ( ) ; for ( unsigned i = <NUM_LIT> , e = CPEntries . size ( ) ; i != e ; ++ i ) { for ( unsigned j = <NUM_LIT> , je = CPEntries [ i ] . size ( ) ; j != je ; ++ j ) { const CPEntry & CPE = CPEntries [ i ] [ j ] ; if ( CPE . CPEMI && CPE . CPEMI -> getOperand ( <NUM_LIT> ) . isCPI ( ) ) AFI -> recordCPEClone ( i , CPE . CPI ) ; } } LLVM_DEBUG ( dbgs ( ) << '\n' ; dumpBBs ( ) ) ; BBUtils -> clear ( ) ; WaterList . clear ( ) ; CPUsers . clear ( ) ; CPEntries . clear ( ) ; JumpTableEntryIndices . clear ( ) ; JumpTableUserIndices . clear ( ) ; ImmBranches . clear ( ) ; PushPopMIs . clear ( ) ; T2JumpTables . clear ( ) ; return MadeChange ;


<s> for ( MachineInstr & I : * MBB ) { BBI . Size += TII -> GetInstSizeInBytes ( I ) ; if ( I . isInlineAsm ( ) ) BBI . Unalign = isThumb ? <NUM_LIT> : <NUM_LIT> ;

else if ( isThumb && mayOptimizeThumb2Instruction ( & I ) ) BBI . Unalign = <NUM_LIT> ;


<s> if ( ! isThumb ) BuildMI ( UserMBB , DebugLoc ( ) , TII -> get ( UncondBr ) ) . addMBB ( NewMBB ) ; else BuildMI ( UserMBB , DebugLoc ( ) , TII -> get ( UncondBr ) ) . addMBB ( NewMBB ) . addImm ( <STR_LIT> ) . addReg ( <NUM_LIT> ) ; unsigned MaxDisp = getUnconditionalBrDisp ( UncondBr ) ; ImmBranches . push_back ( ImmBranch ( & UserMBB -> back ( ) , MaxDisp , false , UncondBr ) ) ; computeBlockSize ( UserMBB ) ; adjustBBOffsetsAfter ( UserMBB ) ; return ; } } unsigned LogAlign = MF -> getAlignment ( ) ; assert ( LogAlign >= CPELogAlign && <STR_LIT> ) ; unsigned KnownBits = UserBBI . internalKnownBits ( ) ; unsigned UPad = UnknownPadding ( LogAlign , KnownBits ) ; unsigned BaseInsertOffset = UserOffset + U . getMaxDisp ( ) - UPad ; DEBUG ( dbgs ( ) << format ( <STR_LIT> , BaseInsertOffset ) ) ; BaseInsertOffset -= <NUM_LIT> ; DEBUG ( dbgs ( ) << format ( <STR_LIT> , BaseInsertOffset ) << <STR_LIT> << LogAlign << <STR_LIT> << KnownBits << <STR_LIT> << UPad << '\n' ) ; if ( BaseInsertOffset + <NUM_LIT> >= UserBBI . postOffset ( ) ) { BaseInsertOffset = std :: max ( UserBBI . postOffset ( ) - UPad - <NUM_LIT> , UserOffset + TII -> GetInstSizeInBytes ( * UserMI ) + <NUM_LIT> ) ; DEBUG ( dbgs ( ) << format ( <STR_LIT> , BaseInsertOffset ) ) ; } unsigned EndInsertOffset = BaseInsertOffset + <NUM_LIT> + UPad + CPEMI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; MachineBasicBlock :: iterator MI = UserMI ; ++ MI ; unsigned CPUIndex = CPUserIndex + <NUM_LIT> ; unsigned NumCPUsers = CPUsers . size ( ) ; MachineInstr * LastIT = nullptr ; for ( unsigned Offset = UserOffset + TII -> GetInstSizeInBytes ( * UserMI ) ; Offset < BaseInsertOffset ; Offset += TII -> GetInstSizeInBytes ( * MI ) , MI = std :: next ( MI ) ) { assert ( MI != UserMBB -> end ( ) && <STR_LIT> ) ; if ( CPUIndex < NumCPUsers && CPUsers [ CPUIndex ] . MI == & * MI ) { CPUser & U = CPUsers [ CPUIndex ] ; if ( ! isOffsetInRange ( Offset , EndInsertOffset , U ) ) { BaseInsertOffset -= <NUM_LIT> << LogAlign ; EndInsertOffset -= <NUM_LIT> << LogAlign ; } EndInsertOffset += U . CPEMI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; CPUIndex ++ ; } if ( MI -> getOpcode ( ) == <STR_LIT> ) LastIT = & * MI ; } -- MI ;


<s> unsigned <ISA_LIT>ConstantIslands :: getOffsetOf ( MachineInstr * MI ) const { MachineBasicBlock * MBB = MI -> getParent ( ) ; unsigned Offset = BBInfo [ MBB -> getNumber ( ) ] . Offset ;

for ( MachineBasicBlock :: iterator I = MBB -> begin ( ) ; & * I != MI ; ++ I ) {


<s> break ; case <STR_LIT> : isCond = true ; UOpc = <STR_LIT> ; Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : isCond = true ; UOpc = <STR_LIT> ; Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; } unsigned MaxOffs = ( ( <NUM_LIT> << ( Bits - <NUM_LIT> ) ) - <NUM_LIT> ) * Scale ; ImmBranches . push_back ( ImmBranch ( & I , MaxOffs , isCond , UOpc ) ) ; } if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) PushPopMIs . push_back ( & I ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) continue ; for ( unsigned op = <NUM_LIT> , e = I . getNumOperands ( ) ; op != e ; ++ op ) if ( I . getOperand ( op ) . isCPI ( ) || I . getOperand ( op ) . isJTI ( ) ) { unsigned Bits = <NUM_LIT> ; unsigned Scale = <NUM_LIT> ; bool NegOk = false ; bool IsSoImm = false ; switch ( Opc ) { default : llvm_unreachable ( <STR_LIT> ) ; case <STR_LIT> : case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; NegOk = true ; IsSoImm = true ; break ; case <STR_LIT> : case <STR_LIT> : Bits = <NUM_LIT> ; NegOk = true ; break ; case <STR_LIT> : case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : Bits = <NUM_LIT> ; NegOk = true ; break ; case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Bits = <NUM_LIT> ; Scale = <NUM_LIT> ; NegOk = true ; break ; } unsigned CPI = I . getOperand ( op ) . getIndex ( ) ; if ( I . getOperand ( op ) . isJTI ( ) ) { JumpTableUserIndices . insert ( std :: make_pair ( CPI , CPUsers . size ( ) ) ) ;


<s> if ( HalfWordOk && ( DstOffset - JTOffset ) > TBHLimit ) HalfWordOk = false ; if ( ! ByteOk && ! HalfWordOk ) break ; } if ( ! ByteOk && ! HalfWordOk ) continue ; MachineBasicBlock * MBB = MI -> getParent ( ) ; if ( ! MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ) continue ; unsigned IdxReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool IdxRegKill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; CPUser & User = CPUsers [ JumpTableUserIndices [ JTI ] ] ; unsigned DeadSize = <NUM_LIT> ; bool CanDeleteLEA = false ; bool BaseRegKill = false ; bool PreservedBaseReg = preserveBaseRegister ( MI , User . MI , DeadSize , CanDeleteLEA , BaseRegKill ) ; if ( ! jumpTableFollowsTB ( MI , User . CPEMI ) && ! PreservedBaseReg ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << * MI ) ; MachineInstr * CPEMI = User . CPEMI ; unsigned Opc = ByteOk ? <STR_LIT> : <STR_LIT> ; MachineBasicBlock :: iterator MI_JT = MI ; MachineInstr * NewJTMI = BuildMI ( * MBB , MI_JT , MI -> getDebugLoc ( ) , TII -> get ( Opc ) ) . addReg ( User . MI -> getOperand ( <NUM_LIT> ) . getReg ( ) , getKillRegState ( BaseRegKill ) ) . addReg ( IdxReg , getKillRegState ( IdxRegKill ) ) . addJumpTableIndex ( JTI , JTOP . getTargetFlags ( ) ) . addImm ( CPEMI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> << * NewJTMI ) ; unsigned JTOpc = ByteOk ? <STR_LIT> : <STR_LIT> ; CPEMI -> setDesc ( TII -> get ( JTOpc ) ) ; if ( jumpTableFollowsTB ( MI , User . CPEMI ) ) { NewJTMI -> getOperand ( <NUM_LIT> ) . setReg ( <STR_LIT> ) ; NewJTMI -> getOperand ( <NUM_LIT> ) . setIsKill ( false ) ; if ( CanDeleteLEA ) { User . MI -> eraseFromParent ( ) ; DeadSize += <NUM_LIT> ; User . MI = NewJTMI ; User . MaxDisp = <NUM_LIT> ; User . NegOk = false ; User . IsSoImm = false ; User . KnownAlignment = false ; } else { int CPEntryIdx = JumpTableEntryIndices [ JTI ] ; auto & CPEs = CPEntries [ CPEntryIdx ] ; auto Entry = std :: find_if ( CPEs . begin ( ) , CPEs . end ( ) , [ & ] ( CPEntry & E ) { return E . CPEMI == User . CPEMI ; } ) ; ++ Entry -> RefCount ; CPUsers . emplace_back ( CPUser ( NewJTMI , User . CPEMI , <NUM_LIT> , false , false ) ) ; } } unsigned NewSize = TII -> GetInstSizeInBytes ( * NewJTMI ) ; unsigned OrigSize = TII -> GetInstSizeInBytes ( * MI ) ;


<s> const BasicBlockInfo & BBI = BBInfo [ J ] ;

dbgs ( ) << format ( <STR_LIT> , BBI . Offset , J ) << <STR_LIT> << unsigned ( BBI . KnownBits ) << <STR_LIT> << unsigned ( BBI . Unalign ) << <STR_LIT> << Log2 ( BBI . PostAlign ) << format ( <STR_LIT> , BBInfo [ J ] . Size ) ;


<s> unsigned Size = CPEMI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; CPEMI -> eraseFromParent ( ) ; BBInfoVector & BBInfo = BBUtils -> getBBInfo ( ) ; BBUtils -> adjustBBSize ( CPEBB , - Size ) ; if ( CPEBB -> empty ( ) ) { BBInfo [ CPEBB -> getNumber ( ) ] . Size = <NUM_LIT> ;

CPEBB -> setAlignment ( Align :: None ( ) ) ;


<s> BBInfoVector & BBInfo = BBUtils -> getBBInfo ( ) ; BBUtils -> adjustBBSize ( CPEBB , - Size ) ; if ( CPEBB -> empty ( ) ) { BBInfo [ CPEBB -> getNumber ( ) ] . Size = <NUM_LIT> ; CPEBB -> setAlignment ( Align :: None ( ) ) ; } else { CPEBB -> setAlignment ( getCPEAlign ( & * CPEBB -> begin ( ) ) ) ;


<s> DT = & getAnalysis < MachineDominatorTree > ( ) ; isThumb = AFI -> isThumbFunction ( ) ; isThumb1 = AFI -> isThumb1OnlyFunction ( ) ; isThumb2 = AFI -> isThumb2Function ( ) ; HasFarJump = false ; bool GenerateTBB = isThumb2 || ( isThumb1 && SynthesizeThumb1TBB ) ; MF -> RenumberBlocks ( ) ; bool MadeChange = false ; if ( GenerateTBB && AdjustJumpTableBlocks ) { scanFunctionJumpTables ( ) ; MadeChange |= reorderThumb2JumpTables ( ) ; T2JumpTables . clear ( ) ; MF -> RenumberBlocks ( ) ; } std :: vector < MachineInstr * > CPEMIs ; if ( ! MCP -> isEmpty ( ) ) doInitialConstPlacement ( CPEMIs ) ; if ( MF -> getJumpTableInfo ( ) ) doInitialJumpTablePlacement ( CPEMIs ) ; AFI -> initPICLabelUId ( CPEMIs . size ( ) ) ; initializeFunctionInfo ( CPEMIs ) ; CPEMIs . clear ( ) ; LLVM_DEBUG ( dumpBBs ( ) ) ; if ( ! T2JumpTables . empty ( ) ) MF -> ensureAlignment ( Align ( <NUM_LIT> ) ) ; MadeChange |= removeUnusedCPEntries ( ) ; unsigned NoCPIters = <NUM_LIT> , NoBRIters = <NUM_LIT> ; while ( true ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << NoCPIters << '\n' ) ; bool CPChange = false ; for ( unsigned i = <NUM_LIT> , e = CPUsers . size ( ) ; i != e ; ++ i ) CPChange |= handleConstantPoolUser ( i , NoCPIters >= CPMaxIteration / <NUM_LIT> ) ; if ( CPChange && ++ NoCPIters > CPMaxIteration ) report_fatal_error ( <STR_LIT> ) ; LLVM_DEBUG ( dumpBBs ( ) ) ; NewWaterList . clear ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << NoBRIters << '\n' ) ; bool BRChange = false ; for ( unsigned i = <NUM_LIT> , e = ImmBranches . size ( ) ; i != e ; ++ i ) BRChange |= fixupImmediateBr ( ImmBranches [ i ] ) ; if ( BRChange && ++ NoBRIters > <NUM_LIT> ) report_fatal_error ( <STR_LIT> ) ; LLVM_DEBUG ( dumpBBs ( ) ) ; if ( ! CPChange && ! BRChange ) break ; MadeChange = true ; } if ( isThumb2 && ! STI -> prefers32BitThumb ( ) ) MadeChange |= optimizeThumb2Instructions ( ) ; if ( isThumb && STI -> hasV8MBaselineOps ( ) ) MadeChange |= optimizeThumb2Branches ( ) ; if ( GenerateTBB && ! STI -> genExecuteOnly ( ) ) MadeChange |= optimizeThumb2JumpTables ( ) ; verify ( ) ; if ( isThumb && ! HasFarJump && AFI -> isLRSpilledForFarJump ( ) ) MadeChange |= undoLRSpillRestore ( ) ; for ( unsigned i = <NUM_LIT> , e = CPEntries . size ( ) ; i != e ; ++ i ) { for ( unsigned j = <NUM_LIT> , je = CPEntries [ i ] . size ( ) ; j != je ; ++ j ) {

const CPEntry & CPE = CPEntries [ i ] [ j ] ;


<s> bool <ISA_LIT>ConstantIslands :: fixupConditionalBr ( ImmBranch & Br ) { MachineInstr * MI = Br . MI ; MachineBasicBlock * DestBB = MI -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; <STR_LIT> CC = ( <STR_LIT> ) MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; CC = <STR_LIT> ( CC ) ; unsigned CCReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineBasicBlock * MBB = MI -> getParent ( ) ; MachineInstr * BMI = & MBB -> back ( ) ; bool NeedSplit = ( BMI != MI ) || ! BBHasFallthrough ( MBB ) ; ++ NumCBrFixed ; if ( BMI != MI ) { if ( std :: next ( MachineBasicBlock :: iterator ( MI ) ) == std :: prev ( MBB -> end ( ) ) && BMI -> getOpcode ( ) == Br . UncondBr ) { MachineBasicBlock * NewDest = BMI -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( isBBInRange ( MI , NewDest , Br . MaxDisp ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << * BMI ) ; BMI -> getOperand ( <NUM_LIT> ) . setMBB ( DestBB ) ; MI -> getOperand ( <NUM_LIT> ) . setMBB ( NewDest ) ; MI -> getOperand ( <NUM_LIT> ) . setImm ( CC ) ;


<s> BB -> updateTerminator ( ) ; MF . RenumberBlocks ( ) ; ++ NumJTMoved ; return NULL ; } MachineBasicBlock * NewBB = MF . CreateMachineBasicBlock ( JTBB -> getBasicBlock ( ) ) ; MachineFunction :: iterator MBBI = JTBB ; ++ MBBI ; MF . insert ( MBBI , NewBB ) ; assert ( isThumb2 && <STR_LIT> ) ; BuildMI ( NewBB , DebugLoc :: getUnknownLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( BB ) ; MF . RenumberBlocks ( NewBB ) ; NewBB -> addSuccessor ( BB ) ; JTBB -> removeSuccessor ( BB ) ; JTBB -> addSuccessor ( NewBB ) ;