<s> if ( I == E ) { DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( Reg . R , TRI , Reg . S ) << '\n' ) ; MachineInstr * DefI = MRI -> getVRegDef ( Reg . R ) ; DefI -> eraseFromParent ( ) ; return ; } for ( ; I != E ; ++ I ) { MachineInstr * UseI = I -> getParent ( ) ; if ( isConvertibleToPredForm ( UseI ) ) PUsers . insert ( UseI ) ;


<s> for ( SetOfReg :: iterator I = PredGPRs . begin ( ) , E = PredGPRs . end ( ) ; I != E ; ++ I ) processPredicateGPR ( * I ) ; bool Again ; do { Again = false ; VectOfInst Processed , Copy ; using iterator = VectOfInst :: iterator ; Copy = PUsers ; for ( iterator I = Copy . begin ( ) , E = Copy . end ( ) ; I != E ; ++ I ) { MachineInstr * MI = * I ; bool Done = convertToPredForm ( MI ) ; if ( Done ) { Processed . insert ( MI ) ; Again = true ; } } Changed |= Again ; auto Done = [ Processed ] ( MachineInstr * MI ) -> bool { return Processed . count ( MI ) ; } ; PUsers . remove_if ( Done ) ; } while ( Again ) ; Changed |= eliminatePredCopies ( MF ) ;


<s> initialize<ISA_LIT>GenPredicatePass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> use_iterator I = MRI -> use_begin ( Reg . R ) , E = MRI -> use_end ( ) ; if ( I == E ) { DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( Reg . R , TRI , Reg . S ) << '\n' ) ; MachineInstr * DefI = MRI -> getVRegDef ( Reg . R ) ; DefI -> eraseFromParent ( ) ; return ; }


<s> use_iterator I = MRI -> use_begin ( Reg . R ) , E = MRI -> use_end ( ) ; if ( I == E ) { DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( Reg . R , TRI , Reg . S ) << '\n' ) ;

MachineInstr * DefI = MRI -> getVRegDef ( Reg . R ) ;


<s> Again = false ; VectOfInst Processed , Copy ; typedef VectOfInst :: iterator iterator ; Copy = PUsers ; for ( iterator I = Copy . begin ( ) , E = Copy . end ( ) ; I != E ; ++ I ) { MachineInstr * MI = * I ; bool Done = convertToPredForm ( MI ) ; if ( Done ) { Processed . insert ( MI ) ; Again = true ; } } Changed |= Again ; auto Done = [ Processed ] ( MachineInstr * MI ) -> bool { return Processed . count ( MI ) ; } ;


<s> VectOfInst Processed , Copy ; typedef VectOfInst :: iterator iterator ; Copy = PUsers ; for ( iterator I = Copy . begin ( ) , E = Copy . end ( ) ; I != E ; ++ I ) { MachineInstr * MI = * I ; bool Done = convertToPredForm ( MI ) ; if ( Done ) { Processed . insert ( MI ) ; Again = true ; } } Changed |= Again ; auto Done = [ Processed ] ( MachineInstr * MI ) -> bool { return Processed . count ( MI ) ; } ; PUsers . remove_if ( Done ) ; } while ( Again ) ;


<s> if ( ! TargetRegisterInfo :: isVirtualRegister ( DR . R ) ) continue ; if ( ! TargetRegisterInfo :: isVirtualRegister ( SR . R ) ) continue ; if ( MRI -> getRegClass ( DR . R ) != PredRC ) continue ; if ( MRI -> getRegClass ( SR . R ) != PredRC ) continue ; assert ( ! DR . S && ! SR . S && <STR_LIT> ) ; MRI -> replaceRegWith ( DR . R , SR . R ) ;


<s> if ( MRI -> getRegClass ( PR . R ) != PredRC ) return false ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : for ( const MachineOperand & MO : DefI -> operands ( ) ) if ( MO . isReg ( ) && MO . isUse ( ) ) WorkQ . push ( Register ( MO . getReg ( ) ) ) ; break ;


<s> case <STR_LIT> : case TargetOpcode :: COPY : if ( isPredReg ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { RegisterSubReg RD = MI -> getOperand ( <NUM_LIT> ) ; if ( Register :: isVirtualRegister ( RD . R ) ) PredGPRs . insert ( RD ) ; } break ; } } }


<s> RegisterSubReg OutR ( Op0 ) ; const TargetRegisterClass * PredRC = & <STR_LIT> ; RegisterSubReg NewPR = MRI -> createVirtualRegister ( PredRC ) ; MachineInstrBuilder MIB = BuildMI ( B , MI , DL , TII -> get ( NewOpc ) , NewPR . R ) ; for ( unsigned i = <NUM_LIT> ; i < NumOps ; ++ i ) { RegisterSubReg GPR = MI -> getOperand ( i ) ; RegisterSubReg Pred = getPredRegFor ( GPR ) ; MIB . addReg ( Pred . R , <NUM_LIT> , Pred . S ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; const TargetRegisterClass * RC = MRI -> getRegClass ( OutR . R ) ; Register NewOutR = MRI -> createVirtualRegister ( RC ) ; BuildMI ( B , MI , DL , TII -> get ( TargetOpcode :: COPY ) , NewOutR ) . addReg ( NewPR . R , <NUM_LIT> , NewPR . S ) ; MRI -> replaceRegWith ( OutR . R , NewOutR ) ; MI -> eraseFromParent ( ) ; if ( ! isPredReg ( NewOutR ) ) { RegisterSubReg R ( NewOutR ) ; PredGPRs . insert ( R ) ; processPredicateGPR ( R ) ; } return true ;


<s> RegisterSubReg Reg ( MO ) ; if ( Reg . S && Reg . S != <STR_LIT> ) return false ; if ( ! PredGPRs . count ( Reg ) ) return false ; } MachineBasicBlock & B = * MI -> getParent ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; unsigned NewOpc = getPredForm ( Opc ) ; if ( NewOpc == <NUM_LIT> ) { switch ( Opc ) { case <STR_LIT> : NewOpc = <STR_LIT> ; break ; case <STR_LIT> : NewOpc = TargetOpcode :: COPY ; break ; default : return false ; } RegisterSubReg PR = getPredRegFor ( MI -> getOperand ( <NUM_LIT> ) ) ; if ( ! isScalarPred ( PR ) ) return false ; NumOps = <NUM_LIT> ; } MachineOperand & Op0 = MI -> getOperand ( <NUM_LIT> ) ; assert ( Op0 . isDef ( ) ) ; RegisterSubReg OutR ( Op0 ) ; const TargetRegisterClass * PredRC = & <STR_LIT> ; RegisterSubReg NewPR = MRI -> createVirtualRegister ( PredRC ) ; MachineInstrBuilder MIB = BuildMI ( B , MI , DL , TII -> get ( NewOpc ) , NewPR . R ) ;

for ( unsigned i = <NUM_LIT> ; i < NumOps ; ++ i ) {


<s> if ( ! Register :: isVirtualRegister ( DR . R ) ) continue ; if ( ! Register :: isVirtualRegister ( SR . R ) ) continue ; if ( MRI -> getRegClass ( DR . R ) != PredRC ) continue ; if ( MRI -> getRegClass ( SR . R ) != PredRC ) continue ; assert ( ! DR . S && ! SR . S && <STR_LIT> ) ;

MRI -> replaceRegWith ( DR . R , SR . R ) ;


<s> switch ( Opc ) { case A2_and : case A2_andp : return C2_and ; case A4_andn : case A4_andnp : return C2_andn ; case M4_and_and : return C4_and_and ; case M4_and_andn : return C4_and_andn ; case M4_and_or : return C4_and_or ; case A2_or : case A2_orp : return C2_or ; case A4_orn : case A4_ornp : return C2_orn ; case M4_or_and : return C4_or_and ; case M4_or_andn : return C4_or_andn ; case M4_or_or :


<s> MachineInstr * DefI = MRI -> getVRegDef ( Reg . R ) ; assert ( DefI ) ; unsigned Opc = DefI -> getOpcode ( ) ; if ( Opc == <STR_LIT> || Opc == TargetOpcode :: COPY ) { assert ( DefI -> getOperand ( <NUM_LIT> ) . isDef ( ) && DefI -> getOperand ( <NUM_LIT> ) . isUse ( ) ) ; RegisterSubReg PR = DefI -> getOperand ( <NUM_LIT> ) ; G2P . insert ( std :: make_pair ( Reg , PR ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << PrintRegister ( PR , * TRI ) << '\n' ) ; return PR ;


<s> <ISA_LIT>GenPredicate ( ) : MachineFunctionPass ( ID ) { initialize<ISA_LIT>GenPredicatePass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> const TargetRegisterClass * RC = MRI -> getRegClass ( R ) ;


<s> const MachineInstr * DefI = MRI -> getVRegDef ( PR . R ) ; if ( ! DefI ) return false ; unsigned DefOpc = DefI -> getOpcode ( ) ; switch ( DefOpc ) { case TargetOpcode :: COPY : { const TargetRegisterClass * PredRC = & <STR_LIT> ; if ( MRI -> getRegClass ( PR . R ) != PredRC ) return false ; LLVM_FALLTHROUGH ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : for ( const MachineOperand & MO : DefI -> operands ( ) ) if ( MO . isReg ( ) && MO . isUse ( ) ) WorkQ . push ( RegisterSubReg ( MO . getReg ( ) ) ) ; break ; default : return isScalarCmp ( DefOpc ) ; } } return true ;


<s> PrintRegister ( RegisterSubReg R , const TargetRegisterInfo & I ) : Reg ( R ) , TRI ( I ) {


<s> RegisterSubReg ( const Register & Reg ) : R ( Reg ) , S ( <NUM_LIT> ) {


<s> G2P . clear ( ) ; bool Changed = false ; collectPredicateGPR ( MF ) ; for ( SetOfReg :: iterator I = PredGPRs . begin ( ) , E = PredGPRs . end ( ) ; I != E ; ++ I ) processPredicateGPR ( * I ) ; bool Again ; do { Again = false ; VectOfInst Processed , Copy ; using iterator = VectOfInst :: iterator ; Copy = PUsers ; for ( iterator I = Copy . begin ( ) , E = Copy . end ( ) ; I != E ; ++ I ) { MachineInstr * MI = * I ; bool Done = convertToPredForm ( MI ) ; if ( Done ) { Processed . insert ( MI ) ; Again = true ; } } Changed |= Again ; auto Done = [ Processed ] ( MachineInstr * MI ) -> bool { return Processed . count ( MI ) ; } ; PUsers . remove_if ( Done ) ; } while ( Again ) ; Changed |= eliminatePredCopies ( MF ) ; return Changed ;


<s> MachineInstr * MI = & * I ; unsigned Opc = MI -> getOpcode ( ) ; switch ( Opc ) { case <STR_LIT> : case TargetOpcode :: COPY : if ( isPredReg ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) {

RegisterSubReg RD = MI -> getOperand ( <NUM_LIT> ) ;


<s> MachineOperand & Op0 = MI -> getOperand ( <NUM_LIT> ) ; assert ( Op0 . isDef ( ) ) ; RegisterSubReg OutR ( Op0 ) ; const TargetRegisterClass * PredRC = & <STR_LIT> ; RegisterSubReg NewPR = MRI -> createVirtualRegister ( PredRC ) ; MachineInstrBuilder MIB = BuildMI ( B , MI , DL , TII -> get ( NewOpc ) , NewPR . R ) ; for ( unsigned i = <NUM_LIT> ; i < NumOps ; ++ i ) { RegisterSubReg GPR = MI -> getOperand ( i ) ; RegisterSubReg Pred = getPredRegFor ( GPR ) ; MIB . addReg ( Pred . R , <NUM_LIT> , Pred . S ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MIB ) ; const TargetRegisterClass * RC = MRI -> getRegClass ( OutR . R ) ;

unsigned NewOutR = MRI -> createVirtualRegister ( RC ) ;


<s> if ( MRI -> getRegClass ( DR . R ) != PredRC ) continue ; if ( MRI -> getRegClass ( SR . R ) != PredRC ) continue ; assert ( ! DR . S && ! SR . S && <STR_LIT> ) ; MRI -> replaceRegWith ( DR . R , SR . R ) ; Erase . insert ( & MI ) ; Changed = true ; }


<s> bool Changed = false ; VectOfInst Erase ; for ( MachineBasicBlock & MBB : MF ) { for ( MachineInstr & MI : MBB ) { if ( MI . getOpcode ( ) != TargetOpcode :: COPY ) continue ; RegisterSubReg DR = MI . getOperand ( <NUM_LIT> ) ; RegisterSubReg SR = MI . getOperand ( <NUM_LIT> ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( DR . R ) ) continue ; if ( ! TargetRegisterInfo :: isVirtualRegister ( SR . R ) ) continue ; if ( MRI -> getRegClass ( DR . R ) != PredRC ) continue ; if ( MRI -> getRegClass ( SR . R ) != PredRC ) continue ;

assert ( ! DR . S && ! SR . S && <STR_LIT> ) ;


<s> RegToRegMap :: iterator F = G2P . find ( Reg ) ; if ( F != G2P . end ( ) ) return F -> second ; LLVM_DEBUG ( dbgs ( ) << __func__ << <STR_LIT> << PrintRegister ( Reg , * TRI ) ) ; MachineInstr * DefI = MRI -> getVRegDef ( Reg . R ) ; assert ( DefI ) ; unsigned Opc = DefI -> getOpcode ( ) ; if ( Opc == <STR_LIT> || Opc == TargetOpcode :: COPY ) { assert ( DefI -> getOperand ( <NUM_LIT> ) . isDef ( ) && DefI -> getOperand ( <NUM_LIT> ) . isUse ( ) ) ; RegisterSubReg PR = DefI -> getOperand ( <NUM_LIT> ) ; G2P . insert ( std :: make_pair ( Reg , PR ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << PrintRegister ( PR , * TRI ) << '\n' ) ; return PR ; } MachineBasicBlock & B = * DefI -> getParent ( ) ; DebugLoc DL = DefI -> getDebugLoc ( ) ;

const TargetRegisterClass * PredRC = & <STR_LIT> ;


<s> if ( isConvertibleToPredForm ( DefI ) ) { MachineBasicBlock :: iterator DefIt = DefI ; BuildMI ( B , std :: next ( DefIt ) , DL , TII -> get ( TargetOpcode :: COPY ) , NewPR ) . addReg ( Reg . R , <NUM_LIT> , Reg . S ) ; G2P . insert ( std :: make_pair ( Reg , RegisterSubReg ( NewPR ) ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << PrintRegister ( RegisterSubReg ( NewPR ) , * TRI ) << '\n' ) ; return RegisterSubReg ( NewPR ) ; } llvm_unreachable ( <STR_LIT> ) ;


<s> NewOpc = TargetOpcode :: COPY ; break ; default : return false ; } Register PR = getPredRegFor ( MI -> getOperand ( <NUM_LIT> ) ) ; if ( ! isScalarPred ( PR ) ) return false ; NumOps = <NUM_LIT> ; } MachineOperand & Op0 = MI -> getOperand ( <NUM_LIT> ) ; assert ( Op0 . isDef ( ) ) ; Register OutR ( Op0 ) ; const TargetRegisterClass * PredRC = & <STR_LIT> ; Register NewPR = MRI -> createVirtualRegister ( PredRC ) ; MachineInstrBuilder MIB = BuildMI ( B , MI , DL , TII -> get ( NewOpc ) , NewPR . R ) ; for ( unsigned i = <NUM_LIT> ; i < NumOps ; ++ i ) { Register GPR = MI -> getOperand ( i ) ;


<s> if ( ! MO . isReg ( ) || ! MO . isUse ( ) ) continue ; Register Reg ( MO ) ; if ( Reg . S && Reg . S != <STR_LIT> ) return false ; if ( ! PredGPRs . count ( Reg ) ) return false ; } MachineBasicBlock & B = * MI -> getParent ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; unsigned NewOpc = getPredForm ( Opc ) ; if ( NewOpc == <NUM_LIT> ) { switch ( Opc ) { case <STR_LIT> : NewOpc = <STR_LIT> ; break ; case <STR_LIT> : NewOpc = TargetOpcode :: COPY ; break ; default : return false ; } Register PR = getPredRegFor ( MI -> getOperand ( <NUM_LIT> ) ) ; if ( ! isScalarPred ( PR ) ) return false ; NumOps = <NUM_LIT> ; } MachineOperand & Op0 = MI -> getOperand ( <NUM_LIT> ) ; assert ( Op0 . isDef ( ) ) ; Register OutR ( Op0 ) ;


<s> if ( ! TargetRegisterInfo :: isVirtualRegister ( DR . R ) ) continue ; if ( ! TargetRegisterInfo :: isVirtualRegister ( SR . R ) ) continue ; if ( MRI -> getRegClass ( DR . R ) != PredRC ) continue ; if ( MRI -> getRegClass ( SR . R ) != PredRC ) continue ; assert ( ! DR . S && ! SR . S && <STR_LIT> ) ;


<s> if ( F != G2P . end ( ) ) return F -> second ; LLVM_DEBUG ( dbgs ( ) << __func__ << <STR_LIT> << PrintRegister ( Reg , * TRI ) ) ; MachineInstr * DefI = MRI -> getVRegDef ( Reg . R ) ; assert ( DefI ) ; unsigned Opc = DefI -> getOpcode ( ) ; if ( Opc == <STR_LIT> || Opc == TargetOpcode :: COPY ) { assert ( DefI -> getOperand ( <NUM_LIT> ) . isDef ( ) && DefI -> getOperand ( <NUM_LIT> ) . isUse ( ) ) ; Register PR = DefI -> getOperand ( <NUM_LIT> ) ; G2P . insert ( std :: make_pair ( Reg , PR ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << PrintRegister ( PR , * TRI ) << '\n' ) ; return PR ;


<s> if ( F != G2P . end ( ) ) return F -> second ; LLVM_DEBUG ( dbgs ( ) << __func__ << <STR_LIT> << PrintRegister ( Reg , * TRI ) ) ; MachineInstr * DefI = MRI -> getVRegDef ( Reg . R ) ; assert ( DefI ) ; unsigned Opc = DefI -> getOpcode ( ) ; if ( Opc == <STR_LIT> || Opc == TargetOpcode :: COPY ) {

assert ( DefI -> getOperand ( <NUM_LIT> ) . isDef ( ) && DefI -> getOperand ( <NUM_LIT> ) . isUse ( ) ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg . R , TRI , Reg . S ) << '\n' ) ; MachineInstr * DefI = MRI -> getVRegDef ( Reg . R ) ; DefI -> eraseFromParent ( ) ; return ; } for ( ; I != E ; ++ I ) {

MachineInstr * UseI = I -> getParent ( ) ;


<s> case <STR_LIT> : case TargetOpcode :: COPY : if ( isPredReg ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { RegisterSubReg RD = MI . getOperand ( <NUM_LIT> ) ; if ( RD . R . isVirtual ( ) ) PredGPRs . insert ( RD ) ;


<s> VectOfInst Erase ; for ( MachineBasicBlock & MBB : MF ) { for ( MachineInstr & MI : MBB ) { if ( MI . getOpcode ( ) != TargetOpcode :: COPY ) continue ; RegisterSubReg DR = MI . getOperand ( <NUM_LIT> ) ; RegisterSubReg SR = MI . getOperand ( <NUM_LIT> ) ;

if ( ! DR . R . isVirtual ( ) ) continue ;


<s> if ( Opc == <STR_LIT> || Opc == TargetOpcode :: COPY ) { assert ( DefI -> getOperand ( <NUM_LIT> ) . isDef ( ) && DefI -> getOperand ( <NUM_LIT> ) . isUse ( ) ) ; RegisterSubReg PR = DefI -> getOperand ( <NUM_LIT> ) ; G2P . insert ( std :: make_pair ( Reg , PR ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << PrintRegister ( PR , * TRI ) << '\n' ) ; return PR ; } MachineBasicBlock & B = * DefI -> getParent ( ) ; DebugLoc DL = DefI -> getDebugLoc ( ) ; const TargetRegisterClass * PredRC = & <STR_LIT> ; Register NewPR = MRI -> createVirtualRegister ( PredRC ) ; if ( isConvertibleToPredForm ( DefI ) ) { MachineBasicBlock :: iterator DefIt = DefI ; BuildMI ( B , std :: next ( DefIt ) , DL , TII -> get ( TargetOpcode :: COPY ) , NewPR ) . addReg ( Reg . R , <NUM_LIT> , Reg . S ) ; G2P . insert ( std :: make_pair ( Reg , RegisterSubReg ( NewPR ) ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << PrintRegister ( RegisterSubReg ( NewPR ) , * TRI ) << '\n' ) ; return RegisterSubReg ( NewPR ) ; }


<s> if ( Opc == <STR_LIT> || Opc == TargetOpcode :: COPY ) { assert ( DefI -> getOperand ( <NUM_LIT> ) . isDef ( ) && DefI -> getOperand ( <NUM_LIT> ) . isUse ( ) ) ; RegisterSubReg PR = DefI -> getOperand ( <NUM_LIT> ) ; G2P . insert ( std :: make_pair ( Reg , PR ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << PrintRegister ( PR , * TRI ) << '\n' ) ; return PR ; } MachineBasicBlock & B = * DefI -> getParent ( ) ; DebugLoc DL = DefI -> getDebugLoc ( ) ; const TargetRegisterClass * PredRC = & <STR_LIT> ; Register NewPR = MRI -> createVirtualRegister ( PredRC ) ; if ( isConvertibleToPredForm ( DefI ) ) { MachineBasicBlock :: iterator DefIt = DefI ;

BuildMI ( B , std :: next ( DefIt ) , DL , TII -> get ( TargetOpcode :: COPY ) , NewPR ) . addReg ( Reg . R , <NUM_LIT> , Reg . S ) ;


<s> if ( ! R . isVirtual ( ) ) return false ; const TargetRegisterClass * RC = MRI -> getRegClass ( R ) ;


<s> bool <ISA_LIT>GenPredicate :: runOnMachineFunction ( MachineFunction & MF ) { if ( skipFunction ( MF . getFunction ( ) ) ) return false ; TII = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) ; TRI = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; PredGPRs . clear ( ) ; PUsers . clear ( ) ; G2P . clear ( ) ; bool Changed = false ; collectPredicateGPR ( MF ) ;

for ( SetOfReg :: iterator I = PredGPRs . begin ( ) , E = PredGPRs . end ( ) ; I != E ; ++ I ) processPredicateGPR ( * I ) ;


<s> bool <ISA_LIT>GenPredicate :: runOnMachineFunction ( MachineFunction & MF ) { if ( skipFunction ( MF . getFunction ( ) ) ) return false ; TII = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) ; TRI = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; PredGPRs . clear ( ) ; PUsers . clear ( ) ; G2P . clear ( ) ; bool Changed = false ; collectPredicateGPR ( MF ) ; for ( const RegisterSubReg & R : PredGPRs ) processPredicateGPR ( R ) ; bool Again ; do { Again = false ; VectOfInst Processed , Copy ; Copy = PUsers ;


<s> MachineInstr * MI = & * I ; unsigned Opc = MI -> getOpcode ( ) ; switch ( Opc ) { case <STR_LIT> : case TargetOpcode :: COPY : if ( isPredReg ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { RegisterSubReg RD = MI -> getOperand ( <NUM_LIT> ) ; if ( RD . R . isVirtual ( ) ) PredGPRs . insert ( RD ) ; }


<s> switch ( Opc ) { case <STR_LIT> : case TargetOpcode :: COPY : if ( isPredReg ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { RegisterSubReg RD = MI -> getOperand ( <NUM_LIT> ) ; if ( RD . R . isVirtual ( ) ) PredGPRs . insert ( RD ) ;


<s> RegisterSubReg DR = MI . getOperand ( <NUM_LIT> ) ; RegisterSubReg SR = MI . getOperand ( <NUM_LIT> ) ; if ( ! DR . R . isVirtual ( ) ) continue ; if ( ! SR . R . isVirtual ( ) ) continue ; if ( MRI -> getRegClass ( DR . R ) != PredRC ) continue ; if ( MRI -> getRegClass ( SR . R ) != PredRC ) continue ; assert ( ! DR . S && ! SR . S && <STR_LIT> ) ; MRI -> replaceRegWith ( DR . R , SR . R ) ;


<s> while ( ! WorkQ . empty ( ) ) { Register PR = WorkQ . front ( ) ; WorkQ . pop ( ) ; const MachineInstr * DefI = MRI -> getVRegDef ( PR . R ) ; if ( ! DefI ) return false ; unsigned DefOpc = DefI -> getOpcode ( ) ; switch ( DefOpc ) { case TargetOpcode :: COPY : { const TargetRegisterClass * PredRC = & <STR_LIT> ; if ( MRI -> getRegClass ( PR . R ) != PredRC ) return false ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> uint64_t defaultTextSegmentAddr ( ) const { return <NUM_LIT> ;


<s> uint64_t <STR_LIT> ( ) const { return llvm :: StringSwitch < uint64_t > ( m_Options . getTargetCPU ( ) ) . Case ( <STR_LIT> , V4 ) . Case ( <STR_LIT> , V5 ) . Default ( V4 ) ;


<s> <STR_LIT> ( const TargetOptions & pTargetOptions ) : GNUInfo ( pTargetOptions . triple ( ) ) , m_Options ( pTargetOptions ) {


<s> uint32_t machine ( ) const { return llvm :: ELF :: EM_HEXAGON ;


<s> iterator it = begin ( ) ; for ( size_t i = <NUM_LIT> ; i < <ISA_LIT>GOTPLT0Num ; ++ i ) ++ it ; for ( ; it != end ( ) ; ++ it ) { llvm :: cast < <ISA_LIT>GOTEntry > ( * it ) . setValue ( pPLT . addr ( ) ) ; }


<s> void <ISA_LIT>GOTPLT :: applyGOT0 ( uint64_t pAddress ) {

llvm :: cast < <ISA_LIT>GOTEntry > ( * ( m_SectionData -> getFragmentList ( ) . begin ( ) ) ) . setValue ( pAddress ) ;


<s> return ( m_SectionData -> size ( ) > <ISA_LIT>GOTPLT0Num ) ;


<s> create ( ) ; } pSection . setAlign ( <NUM_LIT> ) ;


<s> for ( size_t i = <NUM_LIT> ; i < <ISA_LIT>GOTPLT0Num ; ++ i ) { create ( ) ; }


<s> <ISA_LIT>GOTPLT :: ~ <ISA_LIT>GOTPLT ( ) {


return new <ISA_LIT>GOTEntry ( <NUM_LIT> , m_SectionData ) ;


<s> <STR_LIT> ( LDSection & pSection ) : GOT ( pSection ) {


<s> <ISA_LIT>GOTEntry ( uint64_t pContent , SectionData * pParent ) : GOT :: Entry < <NUM_LIT> > ( pContent , pParent ) {


<s> Val = MO . getImm ( ) ; return true ; } if ( ! MO . isReg ( ) ) return false ; int64_t TV ; unsigned R = MO . getReg ( ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( R ) ) return false ; MachineInstr * DI = MRI -> getVRegDef ( R ) ; unsigned DOpc = DI -> getOpcode ( ) ; switch ( DOpc ) { case TargetOpcode :: COPY : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( ! checkForImmediate ( DI -> getOperand ( <NUM_LIT> ) , TV ) ) return false ; break ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const MachineOperand & S1 = DI -> getOperand ( <NUM_LIT> ) ; const MachineOperand & S2 = DI -> getOperand ( <NUM_LIT> ) ; int64_t V1 , V2 ; if ( ! checkForImmediate ( S1 , V1 ) || ! checkForImmediate ( S2 , V2 ) ) return false ; TV = V2 | ( V1 << <NUM_LIT> ) ; break ; } case TargetOpcode :: REG_SEQUENCE : { const MachineOperand & S1 = DI -> getOperand ( <NUM_LIT> ) ; const MachineOperand & S3 = DI -> getOperand ( <NUM_LIT> ) ; int64_t V1 , V3 ; if ( ! checkForImmediate ( S1 , V1 ) || ! checkForImmediate ( S3 , V3 ) ) return false ; unsigned Sub2 = DI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; unsigned Sub4 = DI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( Sub2 == <STR_LIT> && Sub4 == <STR_LIT> ) TV = V1 | ( V3 << <NUM_LIT> ) ; else if ( Sub2 == <STR_LIT> && Sub4 == <STR_LIT> ) TV = V3 | ( V1 << <NUM_LIT> ) ; else llvm_unreachable ( <STR_LIT> ) ; break ; } default : return false ; }


<s> const std :: vector < MachineBasicBlock * > & Blocks = L -> getBlocks ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> << Blocks [ <NUM_LIT> ] -> getNumber ( ) ; ) ; for ( unsigned i = <NUM_LIT> , e = Blocks . size ( ) ; i != e ; ++ i ) { MachineBasicBlock * MBB = Blocks [ i ] ; for ( MachineBasicBlock :: iterator MII = MBB -> begin ( ) , E = MBB -> end ( ) ; MII != E ; ++ MII ) { const MachineInstr * MI = & * MII ; if ( isInvalidLoopOperation ( MI , IsInnerHWLoop ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ; MI -> dump ( ) ; ) ; return true ; } }


<s> if ( Changed && L0Used && L1Used ) return Changed ; unsigned LOOP_i ; unsigned LOOP_r ; unsigned ENDLOOP ; unsigned IsInnerHWLoop = <NUM_LIT> ; if ( L0Used ) { LOOP_i = <STR_LIT> ; LOOP_r = <STR_LIT> ; ENDLOOP = <STR_LIT> ; IsInnerHWLoop = <NUM_LIT> ; } else { LOOP_i = <STR_LIT> ; LOOP_r = <STR_LIT> ; ENDLOOP = <STR_LIT> ; } int Limit = HWLoopLimit ; if ( Limit >= <NUM_LIT> ) { if ( Counter >= HWLoopLimit ) return false ; Counter ++ ; } if ( containsInvalidInstruction ( L , IsInnerHWLoop ) ) return false ; MachineBasicBlock * LastMBB = L -> findLoopControlBlock ( ) ; if ( ! LastMBB ) return false ; MachineBasicBlock :: iterator LastI = LastMBB -> getFirstTerminator ( ) ; if ( LastI == LastMBB -> end ( ) ) return false ; if ( ! fixupInductionVariable ( L ) ) return false ; MachineBasicBlock * Preheader = MLI -> findLoopPreheader ( L , SpecPreheader ) ; if ( ! Preheader ) { Preheader = createPreheaderForLoop ( L ) ; if ( ! Preheader ) return false ; } MachineBasicBlock :: iterator InsertPos = Preheader -> getFirstTerminator ( ) ; SmallVector < MachineInstr * , <NUM_LIT> > OldInsts ; CountValue * TripCount = getLoopTripCount ( L , OldInsts ) ; if ( ! TripCount ) return false ; if ( TripCount -> isReg ( ) ) { MachineInstr * TCDef = MRI -> getVRegDef ( TripCount -> getReg ( ) ) ; MachineBasicBlock * BBDef = TCDef -> getParent ( ) ; if ( ! MDT -> dominates ( BBDef , Preheader ) ) return false ; } MachineBasicBlock * TopBlock = L -> getTopBlock ( ) ; MachineBasicBlock * ExitingBlock = L -> findLoopControlBlock ( ) ; MachineBasicBlock * LoopStart = nullptr ; if ( ExitingBlock != L -> getLoopLatch ( ) ) { MachineBasicBlock * TB = nullptr , * FB = nullptr ; SmallVector < MachineOperand , <NUM_LIT> > Cond ; if ( TII -> analyzeBranch ( * ExitingBlock , TB , FB , Cond , false ) ) return false ; if ( L -> contains ( TB ) ) LoopStart = TB ; else if ( L -> contains ( FB ) ) LoopStart = FB ; else return false ; } else LoopStart = TopBlock ; DEBUG ( dbgs ( ) << <STR_LIT> ; L -> dump ( ) ) ; DebugLoc DL ; if ( InsertPos != Preheader -> end ( ) ) DL = InsertPos -> getDebugLoc ( ) ;


<s> } MachineBasicBlock * TopBlock = L -> getTopBlock ( ) ; MachineBasicBlock * ExitingBlock = L -> findLoopControlBlock ( ) ; MachineBasicBlock * LoopStart = nullptr ; if ( ExitingBlock != L -> getLoopLatch ( ) ) { MachineBasicBlock * TB = nullptr , * FB = nullptr ; SmallVector < MachineOperand , <NUM_LIT> > Cond ; if ( TII -> analyzeBranch ( * ExitingBlock , TB , FB , Cond , false ) ) return false ; if ( L -> contains ( TB ) ) LoopStart = TB ; else if ( L -> contains ( FB ) ) LoopStart = FB ; else return false ; } else LoopStart = TopBlock ; DEBUG ( dbgs ( ) << <STR_LIT> ; L -> dump ( ) ) ; DebugLoc DL ; if ( InsertPos != Preheader -> end ( ) ) DL = InsertPos -> getDebugLoc ( ) ; if ( TripCount -> isReg ( ) ) { unsigned CountReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * Preheader , InsertPos , DL , TII -> get ( TargetOpcode :: COPY ) , CountReg ) . addReg ( TripCount -> getReg ( ) , <NUM_LIT> , TripCount -> getSubReg ( ) ) ; BuildMI ( * Preheader , InsertPos , DL , TII -> get ( LOOP_r ) ) . addMBB ( LoopStart ) . addReg ( CountReg ) ; } else { assert ( TripCount -> isImm ( ) && <STR_LIT> ) ; int64_t CountImm = TripCount -> getImm ( ) ; if ( ! TII -> isValidOffset ( LOOP_i , CountImm ) ) { unsigned CountReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * Preheader , InsertPos , DL , TII -> get ( <STR_LIT> ) , CountReg ) . addImm ( CountImm ) ; BuildMI ( * Preheader , InsertPos , DL , TII -> get ( LOOP_r ) ) . addMBB ( LoopStart ) . addReg ( CountReg ) ;

} else BuildMI ( * Preheader , InsertPos , DL , TII -> get ( LOOP_i ) ) . addMBB ( LoopStart ) . addImm ( CountImm ) ;


<s> } } SmallVector < MachineOperand , <NUM_LIT> > Tmp2 ; SmallVector < MachineOperand , <NUM_LIT> > EmptyCond ; TB = FB = nullptr ; for ( MBBVector :: iterator I = Preds . begin ( ) , E = Preds . end ( ) ; I != E ; ++ I ) { MachineBasicBlock * PB = * I ; if ( PB != Latch ) { Tmp2 . clear ( ) ; bool NotAnalyzed = TII -> analyzeBranch ( * PB , TB , FB , Tmp2 , false ) ; ( void ) NotAnalyzed ; assert ( ! NotAnalyzed && <STR_LIT> ) ; if ( TB != Header && ( Tmp2 . empty ( ) || FB != Header ) ) TII -> insertBranch ( * PB , NewPH , nullptr , EmptyCond , DL ) ; PB -> ReplaceUsesOfBlockWith ( Header , NewPH ) ; } } TB = FB = nullptr ; bool LatchNotAnalyzed = TII -> analyzeBranch ( * Latch , TB , FB , Tmp2 , false ) ; ( void ) LatchNotAnalyzed ; assert ( ! LatchNotAnalyzed && <STR_LIT> ) ; if ( ! TB && ! FB ) TII -> insertBranch ( * Latch , Header , nullptr , EmptyCond , DL ) ; TII -> insertBranch ( * NewPH , Header , nullptr , EmptyCond , DL ) ; NewPH -> addSuccessor ( Header ) ; MachineLoop * ParentLoop = L -> getParentLoop ( ) ; if ( ParentLoop ) ParentLoop -> addBasicBlockToLoop ( NewPH , MLI -> getBase ( ) ) ; if ( MDT ) { if ( MachineDomTreeNode * HN = MDT -> getNode ( Header ) ) { if ( MachineDomTreeNode * DHN = HN -> getIDom ( ) ) { MDT -> addNewBlock ( NewPH , DHN -> getBlock ( ) ) ; MDT -> changeImmediateDominator ( Header , NewPH ) ; } } } return NewPH ;


<s> if ( PredB != Latch ) { PN -> RemoveOperand ( i + <NUM_LIT> ) ; PN -> RemoveOperand ( i ) ; } } PN -> addOperand ( MachineOperand :: CreateReg ( NewPR , false ) ) ; PN -> addOperand ( MachineOperand :: CreateMBB ( NewPH ) ) ; } } else { assert ( Header -> pred_size ( ) == <NUM_LIT> ) ; for ( instr_iterator I = Header -> instr_begin ( ) , E = Header -> instr_end ( ) ; I != E && I -> isPHI ( ) ; ++ I ) { MachineInstr * PN = & * I ; for ( unsigned i = <NUM_LIT> , n = PN -> getNumOperands ( ) ; i < n ; i += <NUM_LIT> ) { MachineOperand & MO = PN -> getOperand ( i + <NUM_LIT> ) ; if ( MO . getMBB ( ) != Latch ) MO . setMBB ( NewPH ) ; } } } SmallVector < MachineOperand , <NUM_LIT> > Tmp2 ; SmallVector < MachineOperand , <NUM_LIT> > EmptyCond ; TB = FB = nullptr ; for ( MBBVector :: iterator I = Preds . begin ( ) , E = Preds . end ( ) ; I != E ; ++ I ) { MachineBasicBlock * PB = * I ; if ( PB != Latch ) { Tmp2 . clear ( ) ; bool NotAnalyzed = TII -> analyzeBranch ( * PB , TB , FB , Tmp2 , false ) ; ( void ) NotAnalyzed ; assert ( ! NotAnalyzed && <STR_LIT> ) ; if ( TB != Header && ( Tmp2 . empty ( ) || FB != Header ) ) TII -> insertBranch ( * PB , NewPH , nullptr , EmptyCond , DL ) ; PB -> ReplaceUsesOfBlockWith ( Header , NewPH ) ; } } TB = FB = nullptr ; bool LatchNotAnalyzed = TII -> analyzeBranch ( * Latch , TB , FB , Tmp2 , false ) ; ( void ) LatchNotAnalyzed ; assert ( ! LatchNotAnalyzed && <STR_LIT> ) ; if ( ! TB && ! FB ) TII -> insertBranch ( * Latch , Header , nullptr , EmptyCond , DL ) ;

TII -> insertBranch ( * NewPH , Header , nullptr , EmptyCond , DL ) ;


<s> MachineBasicBlock * ExitingBlock = L -> findLoopControlBlock ( ) ; if ( ! Header || ! Preheader || ! Latch || ! ExitingBlock ) return false ; typedef std :: pair < unsigned , int64_t > RegisterBump ; typedef std :: map < unsigned , RegisterBump > InductionMap ; InductionMap IndMap ; typedef MachineBasicBlock :: instr_iterator instr_iterator ; for ( instr_iterator I = Header -> instr_begin ( ) , E = Header -> instr_end ( ) ; I != E && I -> isPHI ( ) ; ++ I ) { MachineInstr * Phi = & * I ; for ( unsigned i = <NUM_LIT> , n = Phi -> getNumOperands ( ) ; i < n ; i += <NUM_LIT> ) { if ( Phi -> getOperand ( i + <NUM_LIT> ) . getMBB ( ) != Latch ) continue ; unsigned PhiOpReg = Phi -> getOperand ( i ) . getReg ( ) ; MachineInstr * DI = MRI -> getVRegDef ( PhiOpReg ) ; if ( DI -> getDesc ( ) . isAdd ( ) ) { unsigned IndReg = DI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineOperand & Opnd2 = DI -> getOperand ( <NUM_LIT> ) ; int64_t V ; if ( MRI -> getVRegDef ( IndReg ) == Phi && checkForImmediate ( Opnd2 , V ) ) { unsigned UpdReg = DI -> getOperand ( <NUM_LIT> ) . getReg ( ) ;

IndMap . insert ( std :: make_pair ( UpdReg , std :: make_pair ( IndReg , V ) ) ) ;


<s> break ; case <STR_LIT> : Cmp = Comparison :: LEs ; break ; case <STR_LIT> : Cmp = Comparison :: LEu ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : Cmp = Comparison :: GTu ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : Cmp = Comparison :: GTs ; break ; default : return ( Comparison :: Kind ) <NUM_LIT> ; }


<s> else if ( MBB == Latch ) IVReg = IV_Phi -> getOperand ( i ) . getReg ( ) ; } if ( ! InitialValue ) return nullptr ; SmallVector < MachineOperand , <NUM_LIT> > Cond ; MachineBasicBlock * TB = nullptr , * FB = nullptr ; bool NotAnalyzed = TII -> analyzeBranch ( * ExitingBlock , TB , FB , Cond , false ) ; if ( NotAnalyzed ) return nullptr ; MachineBasicBlock * Header = L -> getHeader ( ) ; assert ( TB && <STR_LIT> ) ; if ( ExitingBlock != Latch && ( TB == Latch || FB == Latch ) ) { MachineBasicBlock * LTB = nullptr , * LFB = nullptr ; SmallVector < MachineOperand , <NUM_LIT> > LCond ; bool NotAnalyzed = TII -> analyzeBranch ( * Latch , LTB , LFB , LCond , false ) ; if ( NotAnalyzed ) return nullptr ; if ( TB == Latch ) TB = ( LTB == Header ) ? LTB : LFB ; else FB = ( LTB == Header ) ? LTB : LFB ; } assert ( ( ! FB || TB == Header || FB == Header ) && <STR_LIT> ) ; if ( ! TB || ( FB && TB != Header && FB != Header ) ) return nullptr ; bool Negated = TII -> predOpcodeHasNot ( Cond ) ^ ( TB != Header ) ; unsigned PredReg , PredPos , PredRegFlags ; if ( ! TII -> getPredReg ( Cond , PredReg , PredPos , PredRegFlags ) ) return nullptr ; MachineInstr * CondI = MRI -> getVRegDef ( PredReg ) ; unsigned CondOpc = CondI -> getOpcode ( ) ; unsigned CmpReg1 = <NUM_LIT> , CmpReg2 = <NUM_LIT> ; int Mask = <NUM_LIT> , ImmValue = <NUM_LIT> ;

bool AnalyzedCmp = TII -> analyzeCompare ( * CondI , CmpReg1 , CmpReg2 , Mask , ImmValue ) ;


<s> const MachineOperand & MO = MI -> getOperand ( i ) ; if ( ! MO . isReg ( ) || ! MO . isDef ( ) ) continue ; unsigned R = MO . getReg ( ) ;

if ( IsInnerHWLoop && ( R == <STR_LIT> || R == <STR_LIT> || R == <STR_LIT> || R == <STR_LIT> ) ) return true ;


<s> if ( MI -> getDesc ( ) . isCall ( ) ) return ! TII -> doesNotReturn ( * MI ) ; for ( unsigned i = <NUM_LIT> , e = MI -> getNumOperands ( ) ; i != e ; ++ i ) { const MachineOperand & MO = MI -> getOperand ( i ) ; if ( ! MO . isReg ( ) || ! MO . isDef ( ) ) continue ; unsigned R = MO . getReg ( ) ; if ( IsInnerHWLoop && ( R == <STR_LIT> || R == <STR_LIT> || R == <STR_LIT> || R == <STR_LIT> ) ) return true ;

if ( ! IsInnerHWLoop && ( R == <STR_LIT> || R == <STR_LIT> ) ) return true ;


<s> DEBUG ( dbgs ( ) << <STR_LIT> << Blocks [ <NUM_LIT> ] -> getNumber ( ) ; ) ; for ( unsigned i = <NUM_LIT> , e = Blocks . size ( ) ; i != e ; ++ i ) { MachineBasicBlock * MBB = Blocks [ i ] ; if ( A == MBB ) return false ; }

MachineInstr * Def = MRI -> getVRegDef ( MO -> getReg ( ) ) ;


<s> typedef MachineBasicBlock :: instr_iterator instr_iterator ; for ( instr_iterator I ( BumpI ) , E = BB -> instr_end ( ) ; I != E ; ++ I ) if ( & * I == CmpI ) return true ; unsigned PredR = CmpI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool FoundBump = false ; instr_iterator CmpIt = CmpI -> getIterator ( ) , NextIt = std :: next ( CmpIt ) ; for ( instr_iterator I = NextIt , E = BB -> instr_end ( ) ; I != E ; ++ I ) { MachineInstr * In = & * I ; for ( unsigned i = <NUM_LIT> , n = In -> getNumOperands ( ) ; i < n ; ++ i ) { MachineOperand & MO = In -> getOperand ( i ) ; if ( MO . isReg ( ) && MO . isUse ( ) ) {

if ( MO . getReg ( ) == PredR ) return false ;


<s> bool <ISA_LIT>HardwareLoops :: orderBumpCompare ( MachineInstr * BumpI , MachineInstr * CmpI ) { assert ( BumpI != CmpI && <STR_LIT> ) ; MachineBasicBlock * BB = BumpI -> getParent ( ) ; if ( CmpI -> getParent ( ) != BB ) return false ; typedef MachineBasicBlock :: instr_iterator instr_iterator ; for ( instr_iterator I ( BumpI ) , E = BB -> instr_end ( ) ; I != E ; ++ I ) if ( & * I == CmpI ) return true ; unsigned PredR = CmpI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool FoundBump = false ; instr_iterator CmpIt = CmpI -> getIterator ( ) , NextIt = std :: next ( CmpIt ) ; for ( instr_iterator I = NextIt , E = BB -> instr_end ( ) ; I != E ; ++ I ) { MachineInstr * In = & * I ; for ( unsigned i = <NUM_LIT> , n = In -> getNumOperands ( ) ; i < n ; ++ i ) { MachineOperand & MO = In -> getOperand ( i ) ; if ( MO . isReg ( ) && MO . isUse ( ) ) { if ( MO . getReg ( ) == PredR ) return false ; } } if ( In == BumpI ) { BB -> splice ( ++ BumpI -> getIterator ( ) , BB , CmpI -> getIterator ( ) ) ; FoundBump = true ; break ; } } assert ( FoundBump && <STR_LIT> ) ; return FoundBump ;


<s> void print ( raw_ostream & OS , const TargetRegisterInfo * TRI = nullptr ) const { if ( isReg ( ) ) { OS << PrintReg ( Contents . R . Reg , TRI , Contents . R . Sub ) ; } if ( isImm ( ) ) { OS << Contents . ImmVal ; }


<s> DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( skipFunction ( * MF . getFunction ( ) ) ) return false ; bool Changed = false ; MLI = & getAnalysis < MachineLoopInfo > ( ) ; MRI = & MF . getRegInfo ( ) ; MDT = & getAnalysis < MachineDominatorTree > ( ) ; TII = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) ; for ( auto & L : * MLI ) if ( ! L -> getParentLoop ( ) ) {


<s> MachineBasicBlock :: iterator InsertPos = PH -> getFirstTerminator ( ) ; DebugLoc DL = ( InsertPos != PH -> end ( ) ) ? InsertPos -> getDebugLoc ( ) : DebugLoc ( ) ; if ( IVBump < <NUM_LIT> ) { std :: swap ( Start , End ) ; IVBump = - IVBump ; } bool RegToImm = Start -> isReg ( ) && End -> isImm ( ) ; bool RegToReg = Start -> isReg ( ) && End -> isReg ( ) ; int64_t StartV = <NUM_LIT> , EndV = <NUM_LIT> ; if ( Start -> isImm ( ) ) StartV = Start -> getImm ( ) ; if ( End -> isImm ( ) ) EndV = End -> getImm ( ) ; int64_t AdjV = <NUM_LIT> ; if ( CmpHasEqual ) { if ( Start -> isImm ( ) ) StartV -- ; else if ( End -> isImm ( ) ) EndV ++ ; else AdjV += <NUM_LIT> ; } if ( Cmp != Comparison :: NE ) { if ( Start -> isImm ( ) ) StartV -= ( IVBump - <NUM_LIT> ) ; else if ( End -> isImm ( ) ) EndV += ( IVBump - <NUM_LIT> ) ; else AdjV += ( IVBump - <NUM_LIT> ) ; } unsigned R = <NUM_LIT> , SR = <NUM_LIT> ; if ( Start -> isReg ( ) ) { R = Start -> getReg ( ) ; SR = Start -> getSubReg ( ) ; } else { R = End -> getReg ( ) ; SR = End -> getSubReg ( ) ; } const TargetRegisterClass * RC = MRI -> getRegClass ( R ) ; if ( ! SR && RC == & <STR_LIT> ) return <NUM_LIT> ; const TargetRegisterClass * IntRC = & <STR_LIT> ; unsigned DistR , DistSR ; if ( Start -> isImm ( ) && StartV == <NUM_LIT> ) { DistR = End -> getReg ( ) ; DistSR = End -> getSubReg ( ) ; } else { const MCInstrDesc & SubD = RegToReg ? TII -> get ( <STR_LIT> ) : ( RegToImm ? TII -> get ( <STR_LIT> ) : TII -> get ( <STR_LIT> ) ) ; unsigned SubR = MRI -> createVirtualRegister ( IntRC ) ; MachineInstrBuilder SubIB = BuildMI ( * PH , InsertPos , DL , SubD , SubR ) ; if ( RegToReg ) { SubIB . addReg ( End -> getReg ( ) , <NUM_LIT> , End -> getSubReg ( ) ) . addReg ( Start -> getReg ( ) , <NUM_LIT> , Start -> getSubReg ( ) ) ; } else if ( RegToImm ) { SubIB . addImm ( EndV ) . addReg ( Start -> getReg ( ) , <NUM_LIT> , Start -> getSubReg ( ) ) ; } else { SubIB . addReg ( End -> getReg ( ) , <NUM_LIT> , End -> getSubReg ( ) ) . addImm ( - StartV ) ; } DistR = SubR ; DistSR = <NUM_LIT> ;


<s> MachineBasicBlock * MBB = Blocks [ i ] ; for ( MachineBasicBlock :: iterator MII = MBB -> begin ( ) , E = MBB -> end ( ) ; MII != E ; ++ MII ) {


<s> if ( LastI == LastMBB -> end ( ) ) return false ; bool NewPreheader = false ; MachineBasicBlock * Preheader = L -> getLoopPreheader ( ) ; if ( ! Preheader ) { Preheader = createPreheaderForLoop ( L ) ; if ( ! Preheader ) return false ; NewPreheader = true ; } MachineBasicBlock :: iterator InsertPos = Preheader -> getFirstTerminator ( ) ; SmallVector < MachineInstr * , <NUM_LIT> > OldInsts ; CountValue * TripCount = getLoopTripCount ( L , OldInsts ) ; if ( TripCount == <NUM_LIT> ) return false ; if ( TripCount -> isReg ( ) ) { MachineInstr * TCDef = MRI -> getVRegDef ( TripCount -> getReg ( ) ) ; MachineBasicBlock * BBDef = TCDef -> getParent ( ) ; if ( ! NewPreheader ) { if ( ! MDT -> dominates ( BBDef , Preheader ) ) return false ; } else { if ( ! MDT -> properlyDominates ( BBDef , L -> getHeader ( ) ) ) return false ; } } MachineBasicBlock * LoopStart = L -> getTopBlock ( ) ; if ( L -> getLoopLatch ( ) != LastMBB ) { LoopStart = L -> getLoopLatch ( ) ; if ( ! LastMBB -> isSuccessor ( LoopStart ) ) return false ; } DEBUG ( dbgs ( ) << <STR_LIT> ; L -> dump ( ) ) ; DebugLoc DL ; if ( InsertPos != Preheader -> end ( ) ) DL = InsertPos -> getDebugLoc ( ) ; if ( TripCount -> isReg ( ) ) { unsigned CountReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * Preheader , InsertPos , DL , TII -> get ( TargetOpcode :: COPY ) , CountReg ) . addReg ( TripCount -> getReg ( ) , <NUM_LIT> , TripCount -> getSubReg ( ) ) ;

BuildMI ( * Preheader , InsertPos , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( LoopStart ) . addReg ( CountReg ) ;


<s> assert ( L -> getHeader ( ) && <STR_LIT> ) ; bool Changed = false ; for ( MachineLoop :: iterator I = L -> begin ( ) , E = L -> end ( ) ; I != E ; ++ I ) Changed |= convertToHardwareLoop ( * I ) ; if ( Changed ) return Changed ; int Limit = HWLoopLimit ; if ( Limit >= <NUM_LIT> ) { if ( Counter >= HWLoopLimit ) return false ; Counter ++ ; } if ( containsInvalidInstruction ( L ) ) return false ; if ( ! fixupInductionVariable ( L ) ) return false ; MachineBasicBlock * LastMBB = L -> getExitingBlock ( ) ; if ( LastMBB == <NUM_LIT> ) return false ; MachineBasicBlock :: iterator LastI = LastMBB -> getFirstTerminator ( ) ; if ( LastI == LastMBB -> end ( ) ) return false ; bool NewPreheader = false ; MachineBasicBlock * Preheader = L -> getLoopPreheader ( ) ; if ( ! Preheader ) { Preheader = createPreheaderForLoop ( L ) ; if ( ! Preheader ) return false ; NewPreheader = true ; } MachineBasicBlock :: iterator InsertPos = Preheader -> getFirstTerminator ( ) ; SmallVector < MachineInstr * , <NUM_LIT> > OldInsts ; CountValue * TripCount = getLoopTripCount ( L , OldInsts ) ; if ( TripCount == <NUM_LIT> ) return false ; if ( TripCount -> isReg ( ) ) { MachineInstr * TCDef = MRI -> getVRegDef ( TripCount -> getReg ( ) ) ; MachineBasicBlock * BBDef = TCDef -> getParent ( ) ; if ( ! NewPreheader ) { if ( ! MDT -> dominates ( BBDef , Preheader ) ) return false ; } else { if ( ! MDT -> properlyDominates ( BBDef , L -> getHeader ( ) ) ) return false ; } } MachineBasicBlock * LoopStart = L -> getTopBlock ( ) ; if ( L -> getLoopLatch ( ) != LastMBB ) { LoopStart = L -> getLoopLatch ( ) ; if ( ! LastMBB -> isSuccessor ( LoopStart ) ) return false ; } DEBUG ( dbgs ( ) << <STR_LIT> ; L -> dump ( ) ) ; DebugLoc DL ; if ( InsertPos != Preheader -> end ( ) ) DL = InsertPos -> getDebugLoc ( ) ; if ( TripCount -> isReg ( ) ) { unsigned CountReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * Preheader , InsertPos , DL , TII -> get ( TargetOpcode :: COPY ) , CountReg ) . addReg ( TripCount -> getReg ( ) , <NUM_LIT> , TripCount -> getSubReg ( ) ) ; BuildMI ( * Preheader , InsertPos , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( LoopStart ) . addReg ( CountReg ) ; } else {

assert ( TripCount -> isImm ( ) && <STR_LIT> ) ;


<s> if ( PredB != Latch ) { PN -> RemoveOperand ( i + <NUM_LIT> ) ; PN -> RemoveOperand ( i ) ; } } PN -> addOperand ( MachineOperand :: CreateReg ( NewPR , false ) ) ; PN -> addOperand ( MachineOperand :: CreateMBB ( NewPH ) ) ; } } else { assert ( Header -> pred_size ( ) == <NUM_LIT> ) ; for ( instr_iterator I = Header -> instr_begin ( ) , E = Header -> instr_end ( ) ; I != E && I -> isPHI ( ) ; ++ I ) { MachineInstr * PN = & * I ; for ( unsigned i = <NUM_LIT> , n = PN -> getNumOperands ( ) ; i < n ; i += <NUM_LIT> ) { MachineOperand & MO = PN -> getOperand ( i + <NUM_LIT> ) ; if ( MO . getMBB ( ) != Latch ) MO . setMBB ( NewPH ) ; } } } SmallVector < MachineOperand , <NUM_LIT> > Tmp2 ; SmallVector < MachineOperand , <NUM_LIT> > EmptyCond ; TB = FB = <NUM_LIT> ; for ( MBBVector :: iterator I = Preds . begin ( ) , E = Preds . end ( ) ; I != E ; ++ I ) { MachineBasicBlock * PB = * I ; if ( PB != Latch ) { Tmp2 . clear ( ) ; bool NotAnalyzed = TII -> AnalyzeBranch ( * PB , TB , FB , Tmp2 , false ) ; ( void ) NotAnalyzed ; assert ( ! NotAnalyzed && <STR_LIT> ) ; if ( TB != Header && ( Tmp2 . empty ( ) || FB != Header ) ) TII -> InsertBranch ( * PB , NewPH , <NUM_LIT> , EmptyCond , DL ) ; PB -> ReplaceUsesOfBlockWith ( Header , NewPH ) ; } } TB = FB = <NUM_LIT> ; bool LatchNotAnalyzed = TII -> AnalyzeBranch ( * Latch , TB , FB , Tmp2 , false ) ; ( void ) LatchNotAnalyzed ; assert ( ! LatchNotAnalyzed && <STR_LIT> ) ; if ( ! TB && ! FB ) TII -> InsertBranch ( * Latch , Header , <NUM_LIT> , EmptyCond , DL ) ; TII -> InsertBranch ( * NewPH , Header , <NUM_LIT> , EmptyCond , DL ) ; NewPH -> addSuccessor ( Header ) ;


<s> for ( unsigned i = <NUM_LIT> , n = PN -> getNumOperands ( ) ; i < n ; i += <NUM_LIT> ) { unsigned PredR = PN -> getOperand ( i ) . getReg ( ) ; MachineBasicBlock * PredB = PN -> getOperand ( i + <NUM_LIT> ) . getMBB ( ) ; if ( PredB == Latch ) continue ; NewPN -> addOperand ( MachineOperand :: CreateReg ( PredR , false ) ) ; NewPN -> addOperand ( MachineOperand :: CreateMBB ( PredB ) ) ; } for ( int i = PN -> getNumOperands ( ) - <NUM_LIT> ; i > <NUM_LIT> ; i -= <NUM_LIT> ) { MachineBasicBlock * PredB = PN -> getOperand ( i + <NUM_LIT> ) . getMBB ( ) ; if ( PredB != Latch ) { PN -> RemoveOperand ( i + <NUM_LIT> ) ; PN -> RemoveOperand ( i ) ; } } PN -> addOperand ( MachineOperand :: CreateReg ( NewPR , false ) ) ; PN -> addOperand ( MachineOperand :: CreateMBB ( NewPH ) ) ; } } else { assert ( Header -> pred_size ( ) == <NUM_LIT> ) ; for ( instr_iterator I = Header -> instr_begin ( ) , E = Header -> instr_end ( ) ; I != E && I -> isPHI ( ) ; ++ I ) { MachineInstr * PN = & * I ; for ( unsigned i = <NUM_LIT> , n = PN -> getNumOperands ( ) ; i < n ; i += <NUM_LIT> ) {

MachineOperand & MO = PN -> getOperand ( i + <NUM_LIT> ) ;


<s> MachineInstr * <ISA_LIT>HardwareLoops :: defWithImmediate ( unsigned R ) { MachineInstr * DI = MRI -> getVRegDef ( R ) ; unsigned DOpc = DI -> getOpcode ( ) ; switch ( DOpc ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return DI ; }


<s> } } SmallVector < MachineOperand , <NUM_LIT> > Cond ; MachineBasicBlock * TB = <NUM_LIT> , * FB = <NUM_LIT> ; bool NotAnalyzed = TII -> AnalyzeBranch ( * Latch , TB , FB , Cond , false ) ; if ( NotAnalyzed ) return false ; unsigned CSz = Cond . size ( ) ; assert ( CSz == <NUM_LIT> || CSz == <NUM_LIT> ) ; unsigned PredR = Cond [ CSz - <NUM_LIT> ] . getReg ( ) ; MachineInstr * PredI = MRI -> getVRegDef ( PredR ) ; if ( ! PredI -> isCompare ( ) ) return false ; unsigned CmpReg1 = <NUM_LIT> , CmpReg2 = <NUM_LIT> ; int CmpImm = <NUM_LIT> , CmpMask = <NUM_LIT> ; bool CmpAnalyzed = TII -> analyzeCompare ( PredI , CmpReg1 , CmpReg2 , CmpMask , CmpImm ) ; if ( ! CmpAnalyzed ) return false ; InductionMap :: iterator IndMapEnd = IndMap . end ( ) ; InductionMap :: iterator F = IndMapEnd ; if ( CmpReg1 != <NUM_LIT> ) { InductionMap :: iterator F1 = IndMap . find ( CmpReg1 ) ; if ( F1 != IndMapEnd ) F = F1 ; } if ( CmpReg2 != <NUM_LIT> ) { InductionMap :: iterator F2 = IndMap . find ( CmpReg2 ) ; if ( F2 != IndMapEnd ) { if ( F != IndMapEnd ) return false ; F = F2 ; } } if ( F == IndMapEnd ) return false ; Reg = F -> second . first ; IVBump = F -> second . second ; IVOp = MRI -> getVRegDef ( F -> first ) ; return true ;


<s> typedef std :: pair < unsigned , RegisterBump > RegisterInduction ; typedef std :: set < RegisterInduction > RegisterInductionSet ; RegisterInductionSet IndRegs ; typedef MachineBasicBlock :: instr_iterator instr_iterator ; for ( instr_iterator I = Header -> instr_begin ( ) , E = Header -> instr_end ( ) ; I != E && I -> isPHI ( ) ; ++ I ) { MachineInstr * Phi = & * I ; for ( unsigned i = <NUM_LIT> , n = Phi -> getNumOperands ( ) ; i < n ; i += <NUM_LIT> ) { if ( Phi -> getOperand ( i + <NUM_LIT> ) . getMBB ( ) != Latch ) continue ; unsigned PhiReg = Phi -> getOperand ( i ) . getReg ( ) ; MachineInstr * DI = MRI -> getVRegDef ( PhiReg ) ; unsigned UpdOpc = DI -> getOpcode ( ) ; bool isAdd = ( UpdOpc == <STR_LIT> ) ; if ( isAdd ) { unsigned IndReg = DI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( MRI -> getVRegDef ( IndReg ) == Phi ) { unsigned UpdReg = DI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; int64_t V = DI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; IndRegs . insert ( std :: make_pair ( UpdReg , std :: make_pair ( IndReg , V ) ) ) ; } } } } if ( IndRegs . empty ( ) ) return false ; MachineBasicBlock * TB = <NUM_LIT> , * FB = <NUM_LIT> ; SmallVector < MachineOperand , <NUM_LIT> > Cond ; bool NotAnalyzed = TII -> AnalyzeBranch ( * Latch , TB , FB , Cond , false ) ; if ( NotAnalyzed ) return false ; if ( Cond . empty ( ) ) return false ; if ( TB != Header && FB != Header ) return false ; unsigned CSz = Cond . size ( ) ; if ( CSz != <NUM_LIT> && CSz != <NUM_LIT> ) return false ; unsigned P = Cond [ CSz - <NUM_LIT> ] . getReg ( ) ;

MachineInstr * PredDef = MRI -> getVRegDef ( P ) ;


<s> AU . addRequired < MachineLoopInfo > ( ) ;

MachineFunctionPass :: getAnalysisUsage ( AU ) ;


<s> for ( unsigned i = <NUM_LIT> , n = IV_Phi -> getNumOperands ( ) ; i < n ; i += <NUM_LIT> ) { MachineBasicBlock * MBB = IV_Phi -> getOperand ( i + <NUM_LIT> ) . getMBB ( ) ; if ( MBB == Preheader ) InitialValue = & IV_Phi -> getOperand ( i ) ; else if ( MBB == Latch ) IVReg = IV_Phi -> getOperand ( i ) . getReg ( ) ; } if ( ! InitialValue ) return <NUM_LIT> ; SmallVector < MachineOperand , <NUM_LIT> > Cond ; MachineBasicBlock * TB = <NUM_LIT> , * FB = <NUM_LIT> ; bool NotAnalyzed = TII -> AnalyzeBranch ( * Latch , TB , FB , Cond , false ) ; if ( NotAnalyzed ) return <NUM_LIT> ; MachineBasicBlock * Header = L -> getHeader ( ) ; assert ( TB && <STR_LIT> ) ; assert ( ( ! FB || TB == Header || FB == Header ) && <STR_LIT> ) ; if ( ! TB || ( FB && TB != Header && FB != Header ) ) return <NUM_LIT> ; bool Negated = ( Cond . size ( ) > <NUM_LIT> ) ^ ( TB != Header ) ; unsigned PredReg = Cond [ Cond . size ( ) - <NUM_LIT> ] . getReg ( ) ; MachineInstr * CondI = MRI -> getVRegDef ( PredReg ) ; unsigned CondOpc = CondI -> getOpcode ( ) ; unsigned CmpReg1 = <NUM_LIT> , CmpReg2 = <NUM_LIT> ; int Mask = <NUM_LIT> , ImmValue = <NUM_LIT> ; bool AnalyzedCmp = TII -> analyzeCompare ( CondI , CmpReg1 , CmpReg2 , Mask , ImmValue ) ; if ( ! AnalyzedCmp ) return <NUM_LIT> ; OldInsts . push_back ( CondI ) ; OldInsts . push_back ( IVOp ) ; Comparison :: Kind Cmp ; bool isSwapped = false ; const MachineOperand & Op1 = CondI -> getOperand ( <NUM_LIT> ) ; const MachineOperand & Op2 = CondI -> getOperand ( <NUM_LIT> ) ; const MachineOperand * EndValue = <NUM_LIT> ; if ( Op1 . isReg ( ) ) { if ( Op2 . isImm ( ) || Op1 . getReg ( ) == IVReg ) EndValue = & Op2 ; else { EndValue = & Op1 ; isSwapped = true ; } } if ( ! EndValue ) return <NUM_LIT> ; switch ( CondOpc ) {


<s> const char * getPassName ( ) const { return <STR_LIT> ;


<s> const char * getPassName ( ) const {


<s> for ( unsigned j = <NUM_LIT> , f = OnePhi -> getNumOperands ( ) ; j != f ; ++ j ) { const MachineOperand & OPO = OnePhi -> getOperand ( j ) ; if ( ! OPO . isReg ( ) || ! OPO . isDef ( ) ) continue ; unsigned OPReg = OPO . getReg ( ) ; use_nodbg_iterator nextJ ; for ( use_nodbg_iterator J = MRI -> use_nodbg_begin ( OPReg ) ; J != End ; J = nextJ ) { nextJ = llvm :: next ( J ) ; MachineOperand & Use = J . getOperand ( ) ; MachineInstr * UseMI = Use . getParent ( ) ; if ( MI != UseMI ) return false ; } } DeadPhis . push_back ( OnePhi ) ; }


<s> use_nodbg_iterator End = MRI -> use_nodbg_end ( ) ; if ( llvm :: next ( I ) != End || ! I . getOperand ( ) . getParent ( ) -> isPHI ( ) ) return false ; MachineInstr * OnePhi = I . getOperand ( ) . getParent ( ) ; for ( unsigned j = <NUM_LIT> , f = OnePhi -> getNumOperands ( ) ; j != f ; ++ j ) { const MachineOperand & OPO = OnePhi -> getOperand ( j ) ; if ( ! OPO . isReg ( ) || ! OPO . isDef ( ) ) continue ; unsigned OPReg = OPO . getReg ( ) ; use_nodbg_iterator nextJ ; for ( use_nodbg_iterator J = MRI -> use_nodbg_begin ( OPReg ) ; J != End ; J = nextJ ) { nextJ = llvm :: next ( J ) ; MachineOperand & Use = J . getOperand ( ) ; MachineInstr * UseMI = Use . getParent ( ) ; if ( MI != UseMI ) return false ; }


<s> bool FoundBump = false ; instr_iterator CmpIt = CmpI , NextIt = llvm :: next ( CmpIt ) ; for ( instr_iterator I = NextIt , E = BB -> instr_end ( ) ; I != E ; ++ I ) { MachineInstr * In = & * I ; for ( unsigned i = <NUM_LIT> , n = In -> getNumOperands ( ) ; i < n ; ++ i ) { MachineOperand & MO = In -> getOperand ( i ) ; if ( MO . isReg ( ) && MO . isUse ( ) ) { if ( MO . getReg ( ) == PredR ) return false ; } } if ( In == BumpI ) { instr_iterator After = BumpI ;


<s> instr_iterator CmpIt = CmpI , NextIt = llvm :: next ( CmpIt ) ; for ( instr_iterator I = NextIt , E = BB -> instr_end ( ) ; I != E ; ++ I ) { MachineInstr * In = & * I ; for ( unsigned i = <NUM_LIT> , n = In -> getNumOperands ( ) ; i < n ; ++ i ) { MachineOperand & MO = In -> getOperand ( i ) ; if ( MO . isReg ( ) && MO . isUse ( ) ) { if ( MO . getReg ( ) == PredR ) return false ; } } if ( In == BumpI ) { instr_iterator After = BumpI ; instr_iterator From = CmpI ; BB -> splice ( llvm :: next ( After ) , BB , From ) ; FoundBump = true ; break ; }


<s> const MachineOperand & MO = MI -> getOperand ( i ) ; if ( ! MO . isReg ( ) || ! MO . isDef ( ) ) continue ; unsigned Reg = MO . getReg ( ) ; MachineRegisterInfo :: use_iterator nextI ; for ( MachineRegisterInfo :: use_iterator I = MRI -> use_begin ( Reg ) , E = MRI -> use_end ( ) ; I != E ; I = nextI ) { nextI = llvm :: next ( I ) ; MachineOperand & Use = I . getOperand ( ) ; MachineInstr * UseMI = Use . getParent ( ) ; if ( UseMI == MI ) continue ; if ( Use . isDebug ( ) ) UseMI -> getOperand ( <NUM_LIT> ) . setReg ( <NUM_LIT> ) ; } } MI -> eraseFromParent ( ) ; for ( unsigned i = <NUM_LIT> ; i < DeadPhis . size ( ) ; ++ i ) DeadPhis [ i ] -> eraseFromParent ( ) ; }


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const MachineOperand & S1 = DI -> getOperand ( <NUM_LIT> ) ; const MachineOperand & S2 = DI -> getOperand ( <NUM_LIT> ) ; int64_t V1 , V2 ; if ( ! checkForImmediate ( S1 , V1 ) || ! checkForImmediate ( S2 , V2 ) ) return false ; TV = V2 | ( static_cast < uint64_t > ( V1 ) << <NUM_LIT> ) ; break ; } case TargetOpcode :: REG_SEQUENCE : { const MachineOperand & S1 = DI -> getOperand ( <NUM_LIT> ) ; const MachineOperand & S3 = DI -> getOperand ( <NUM_LIT> ) ; int64_t V1 , V3 ; if ( ! checkForImmediate ( S1 , V1 ) || ! checkForImmediate ( S3 , V3 ) ) return false ; unsigned Sub2 = DI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; unsigned Sub4 = DI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( Sub2 == <STR_LIT> && Sub4 == <STR_LIT> ) TV = V1 | ( V3 << <NUM_LIT> ) ; else if ( Sub2 == <STR_LIT> && Sub4 == <STR_LIT> ) TV = V3 | ( V1 << <NUM_LIT> ) ; else llvm_unreachable ( <STR_LIT> ) ; break ; } default : return false ; } switch ( MO . getSubReg ( ) ) { case <STR_LIT> : Val = TV & <NUM_LIT> ; break ; case <STR_LIT> : Val = ( TV >> <NUM_LIT> ) & <NUM_LIT> ;


<s> R = Start -> getReg ( ) ; SR = Start -> getSubReg ( ) ; } else { R = End -> getReg ( ) ; SR = End -> getSubReg ( ) ; } const TargetRegisterClass * RC = MRI -> getRegClass ( R ) ; if ( ! SR && RC == & <STR_LIT> ) return nullptr ; const TargetRegisterClass * IntRC = & <STR_LIT> ; unsigned DistR , DistSR ; if ( Start -> isImm ( ) && StartV == <NUM_LIT> ) { DistR = End -> getReg ( ) ; DistSR = End -> getSubReg ( ) ; } else { const MCInstrDesc & SubD = RegToReg ? TII -> get ( <STR_LIT> ) : ( RegToImm ? TII -> get ( <STR_LIT> ) : TII -> get ( <STR_LIT> ) ) ; if ( RegToReg || RegToImm ) { Register SubR = MRI -> createVirtualRegister ( IntRC ) ; MachineInstrBuilder SubIB = BuildMI ( * PH , InsertPos , DL , SubD , SubR ) ; if ( RegToReg ) SubIB . addReg ( End -> getReg ( ) , <NUM_LIT> , End -> getSubReg ( ) ) . addReg ( Start -> getReg ( ) , <NUM_LIT> , Start -> getSubReg ( ) ) ; else SubIB . addImm ( EndV ) . addReg ( Start -> getReg ( ) , <NUM_LIT> , Start -> getSubReg ( ) ) ; DistR = SubR ; } else { const MachineInstr * EndValInstr = MRI -> getVRegDef ( End -> getReg ( ) ) ; if ( EndValInstr -> getOpcode ( ) == <STR_LIT> && EndValInstr -> getOperand ( <NUM_LIT> ) . getSubReg ( ) == <NUM_LIT> && EndValInstr -> getOperand ( <NUM_LIT> ) . getImm ( ) == StartV ) { DistR = EndValInstr -> getOperand ( <NUM_LIT> ) . getReg ( ) ; } else { Register SubR = MRI -> createVirtualRegister ( IntRC ) ; MachineInstrBuilder SubIB = BuildMI ( * PH , InsertPos , DL , SubD , SubR ) ; SubIB . addReg ( End -> getReg ( ) , <NUM_LIT> , End -> getSubReg ( ) ) . addImm ( - StartV ) ; DistR = SubR ; } } DistSR = <NUM_LIT> ; } unsigned AdjR , AdjSR ; if ( AdjV == <NUM_LIT> ) { AdjR = DistR ; AdjSR = DistSR ; } else { Register AddR = MRI -> createVirtualRegister ( IntRC ) ; MCInstrDesc const & AddD = TII -> get ( <STR_LIT> ) ; BuildMI ( * PH , InsertPos , DL , AddD , AddR ) . addReg ( DistR , <NUM_LIT> , DistSR ) . addImm ( AdjV ) ; AdjR = AddR ;


<s> for ( MachineBasicBlock * MBB : L -> getBlocks ( ) ) { for ( const MachineInstr & MI : * MBB ) { if ( isInvalidLoopOperation ( & MI , IsInnerHWLoop ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; ) ; return true ; } }


<s> MachineBasicBlock :: iterator InsertPos = Preheader -> getFirstTerminator ( ) ; SmallVector < MachineInstr * , <NUM_LIT> > OldInsts ; CountValue * TripCount = getLoopTripCount ( L , OldInsts ) ; if ( ! TripCount ) return false ; if ( TripCount -> isReg ( ) ) { MachineInstr * TCDef = MRI -> getVRegDef ( TripCount -> getReg ( ) ) ; MachineBasicBlock * BBDef = TCDef -> getParent ( ) ; if ( ! MDT -> dominates ( BBDef , Preheader ) ) return false ; } MachineBasicBlock * TopBlock = L -> getTopBlock ( ) ; MachineBasicBlock * ExitingBlock = L -> findLoopControlBlock ( ) ; MachineBasicBlock * LoopStart = nullptr ; if ( ExitingBlock != L -> getLoopLatch ( ) ) { MachineBasicBlock * TB = nullptr , * FB = nullptr ; SmallVector < MachineOperand , <NUM_LIT> > Cond ; if ( TII -> analyzeBranch ( * ExitingBlock , TB , FB , Cond , false ) ) return false ; if ( L -> contains ( TB ) ) LoopStart = TB ; else if ( L -> contains ( FB ) ) LoopStart = FB ; else return false ; } else LoopStart = TopBlock ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; L -> dump ( ) ) ; DebugLoc DL ; if ( InsertPos != Preheader -> end ( ) ) DL = InsertPos -> getDebugLoc ( ) ; if ( TripCount -> isReg ( ) ) { Register CountReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * Preheader , InsertPos , DL , TII -> get ( TargetOpcode :: COPY ) , CountReg ) . addReg ( TripCount -> getReg ( ) , <NUM_LIT> , TripCount -> getSubReg ( ) ) ; BuildMI ( * Preheader , InsertPos , DL , TII -> get ( LOOP_r ) ) . addMBB ( LoopStart ) . addReg ( CountReg ) ; } else { assert ( TripCount -> isImm ( ) && <STR_LIT> ) ; int64_t CountImm = TripCount -> getImm ( ) ;

if ( ! TII -> isValidOffset ( LOOP_i , CountImm , TRI ) ) {


<s> for ( instr_iterator I = Header -> instr_begin ( ) , E = Header -> instr_end ( ) ; I != E && I -> isPHI ( ) ; ++ I ) { MachineInstr * Phi = & * I ; for ( unsigned i = <NUM_LIT> , n = Phi -> getNumOperands ( ) ; i < n ; i += <NUM_LIT> ) { if ( Phi -> getOperand ( i + <NUM_LIT> ) . getMBB ( ) != Latch ) continue ; Register PhiOpReg = Phi -> getOperand ( i ) . getReg ( ) ; MachineInstr * DI = MRI -> getVRegDef ( PhiOpReg ) ; if ( DI -> getDesc ( ) . isAdd ( ) ) { Register IndReg = DI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineOperand & Opnd2 = DI -> getOperand ( <NUM_LIT> ) ; int64_t V ; if ( MRI -> getVRegDef ( IndReg ) == Phi && checkForImmediate ( Opnd2 , V ) ) { Register UpdReg = DI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; IndMap . insert ( std :: make_pair ( UpdReg , std :: make_pair ( IndReg , V ) ) ) ; }


<s> use_nodbg_iterator End = MRI -> use_nodbg_end ( ) ; if ( std :: next ( I ) != End || ! I -> getParent ( ) -> isPHI ( ) ) return false ; MachineInstr * OnePhi = I -> getParent ( ) ; for ( unsigned j = <NUM_LIT> , f = OnePhi -> getNumOperands ( ) ; j != f ; ++ j ) { const MachineOperand & OPO = OnePhi -> getOperand ( j ) ; if ( ! OPO . isReg ( ) || ! OPO . isDef ( ) ) continue ; Register OPReg = OPO . getReg ( ) ; use_nodbg_iterator nextJ ;

for ( use_nodbg_iterator J = MRI -> use_nodbg_begin ( OPReg ) ;


<s> if ( LoopFeederPhi . find ( MO -> getReg ( ) ) == LoopFeederPhi . end ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * * L -> block_begin ( ) ) ) ;

if ( llvm :: is_contained ( L -> getBlocks ( ) , A ) ) return false ;


<s> if ( ! InitVal -> isReg ( ) ) return false ; if ( ! EndVal -> isImm ( ) ) return false ; int64_t Imm ; if ( checkForImmediate ( * InitVal , Imm ) ) return ( EndVal -> getImm ( ) == Imm ) ; Register Reg = InitVal -> getReg ( ) ; if ( ! Reg . isVirtual ( ) ) return true ; MachineInstr * Def = MRI -> getVRegDef ( Reg ) ; if ( ! Def ) return true ; if ( Def -> isPHI ( ) && ! phiMayWrapOrUnderflow ( Def , EndVal , Def -> getParent ( ) , L , LoopFeederPhi ) ) return false ; if ( Def -> isCopy ( ) && ! loopCountMayWrapOrUnderFlow ( & ( Def -> getOperand ( <NUM_LIT> ) ) , EndVal , Def -> getParent ( ) , L , LoopFeederPhi ) ) return false ; for ( MachineRegisterInfo :: use_instr_nodbg_iterator I = MRI -> use_instr_nodbg_begin ( Reg ) , E = MRI -> use_instr_nodbg_end ( ) ; I != E ; ++ I ) { MachineInstr * MI = & * I ; Register CmpReg1 , CmpReg2 ; int64_t CmpMask = <NUM_LIT> , CmpValue = <NUM_LIT> ; if ( ! TII -> analyzeCompare ( * MI , CmpReg1 , CmpReg2 , CmpMask , CmpValue ) ) continue ; MachineBasicBlock * TBB = nullptr , * FBB = nullptr ; SmallVector < MachineOperand , <NUM_LIT> > Cond ; if ( TII -> analyzeBranch ( * MI -> getParent ( ) , TBB , FBB , Cond , false ) ) continue ;

Comparison :: Kind Cmp = getComparisonKind ( MI -> getOpcode ( ) , nullptr , nullptr , <NUM_LIT> ) ;


<s> MachineInstr * In = & * I ; for ( unsigned i = <NUM_LIT> , n = In -> getNumOperands ( ) ; i < n ; ++ i ) { MachineOperand & MO = In -> getOperand ( i ) ; if ( MO . isReg ( ) && MO . isUse ( ) ) { if ( MO . getReg ( ) == PredR ) return false ; } } if ( In == BumpI ) {

BB -> splice ( ++ BumpI -> getIterator ( ) , BB , CmpI -> getIterator ( ) ) ;


<s> if ( ! MO . isReg ( ) || ! MO . isDef ( ) ) continue ; Register Reg = MO . getReg ( ) ; for ( MachineOperand & MO : llvm :: make_early_inc_range ( MRI -> use_operands ( Reg ) ) ) { MachineInstr * UseMI = MO . getParent ( ) ; if ( UseMI == MI ) continue ; if ( MO . isDebug ( ) ) MO . setReg ( <NUM_LIT> ) ; } } MI -> eraseFromParent ( ) ; for ( unsigned i = <NUM_LIT> ; i < DeadPhis . size ( ) ; ++ i ) DeadPhis [ i ] -> eraseFromParent ( ) ;


<s> const <ISA_LIT>Subtarget & HST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = HST . getInstrInfo ( ) ; TRI = HST . getRegisterInfo ( ) ; for ( auto & L : * MLI ) if ( L -> isOutermost ( ) ) { bool L0Used = false ; bool L1Used = false ; Changed |= convertToHardwareLoop ( L , L0Used , L1Used ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( skipFunction ( MF . getFunction ( ) ) ) return false ; bool Changed = false ; MLI = & getAnalysis < MachineLoopInfo > ( ) ; MRI = & MF . getRegInfo ( ) ; MDT = & getAnalysis < MachineDominatorTree > ( ) ; const <ISA_LIT>Subtarget & HST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = HST . getInstrInfo ( ) ;


<s> Register R = MO . getReg ( ) ; MachineInstr * DI = MRI -> getVRegDef ( R ) ; const TargetRegisterClass * RC = MRI -> getRegClass ( R ) ; Register NewR = MRI -> createVirtualRegister ( RC ) ; MachineBasicBlock & B = * DI -> getParent ( ) ;


<s> MRI = & MF . getRegInfo ( ) ; MDT = & getAnalysis < MachineDominatorTree > ( ) ; const <ISA_LIT>Subtarget & HST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = HST . getInstrInfo ( ) ; TRI = HST . getRegisterInfo ( ) ; for ( auto & L : * MLI ) if ( ! L -> getParentLoop ( ) ) { bool L0Used = false ; bool L1Used = false ; Changed |= convertToHardwareLoop ( L , L0Used , L1Used ) ; }


<s> } unsigned R = <NUM_LIT> , SR = <NUM_LIT> ; if ( Start -> isReg ( ) ) { R = Start -> getReg ( ) ; SR = Start -> getSubReg ( ) ; } else { R = End -> getReg ( ) ; SR = End -> getSubReg ( ) ; } const TargetRegisterClass * RC = MRI -> getRegClass ( R ) ; if ( ! SR && RC == & <STR_LIT> ) return nullptr ; const TargetRegisterClass * IntRC = & <STR_LIT> ; unsigned DistR , DistSR ; if ( Start -> isImm ( ) && StartV == <NUM_LIT> ) { DistR = End -> getReg ( ) ; DistSR = End -> getSubReg ( ) ; } else { const MCInstrDesc & SubD = RegToReg ? TII -> get ( <STR_LIT> ) : ( RegToImm ? TII -> get ( <STR_LIT> ) : TII -> get ( <STR_LIT> ) ) ; unsigned SubR = MRI -> createVirtualRegister ( IntRC ) ; MachineInstrBuilder SubIB = BuildMI ( * PH , InsertPos , DL , SubD , SubR ) ; if ( RegToReg ) { SubIB . addReg ( End -> getReg ( ) , <NUM_LIT> , End -> getSubReg ( ) ) . addReg ( Start -> getReg ( ) , <NUM_LIT> , Start -> getSubReg ( ) ) ; } else if ( RegToImm ) { SubIB . addImm ( EndV ) . addReg ( Start -> getReg ( ) , <NUM_LIT> , Start -> getSubReg ( ) ) ; } else { SubIB . addReg ( End -> getReg ( ) , <NUM_LIT> , End -> getSubReg ( ) ) . addImm ( - StartV ) ; } DistR = SubR ; DistSR = <NUM_LIT> ; } unsigned AdjR , AdjSR ; if ( AdjV == <NUM_LIT> ) { AdjR = DistR ; AdjSR = DistSR ; } else { unsigned AddR = MRI -> createVirtualRegister ( IntRC ) ; MCInstrDesc const & AddD = TII -> get ( <STR_LIT> ) ; BuildMI ( * PH , InsertPos , DL , AddD , AddR ) . addReg ( DistR , <NUM_LIT> , DistSR ) . addImm ( AdjV ) ; AdjR = AddR ; AdjSR = <NUM_LIT> ; } unsigned CountR , CountSR ; if ( IVBump == <NUM_LIT> ) { CountR = AdjR ; CountSR = AdjSR ; } else { unsigned Shift = Log2_32 ( IVBump ) ; unsigned LsrR = MRI -> createVirtualRegister ( IntRC ) ; const MCInstrDesc & LsrD = TII -> get ( <STR_LIT> ) ; BuildMI ( * PH , InsertPos , DL , LsrD , LsrR ) . addReg ( AdjR , <NUM_LIT> , AdjSR ) . addImm ( Shift ) ; CountR = LsrR ;


<s> assert ( Dist1 > <NUM_LIT> && <STR_LIT> ) ; uint64_t Count = Dist1 ; if ( Count > <NUM_LIT> ) return nullptr ; return new CountValue ( CountValue :: CV_Immediate , Count ) ; } if ( ! isPowerOf2_64 ( abs64 ( IVBump ) ) ) return nullptr ; MachineBasicBlock * PH = Loop -> getLoopPreheader ( ) ; assert ( PH && <STR_LIT> ) ; MachineBasicBlock :: iterator InsertPos = PH -> getFirstTerminator ( ) ; DebugLoc DL = ( InsertPos != PH -> end ( ) ) ? InsertPos -> getDebugLoc ( ) : DebugLoc ( ) ; if ( IVBump < <NUM_LIT> ) { std :: swap ( Start , End ) ; IVBump = - IVBump ; } bool RegToImm = Start -> isReg ( ) && End -> isImm ( ) ; bool RegToReg = Start -> isReg ( ) && End -> isReg ( ) ; int64_t StartV = <NUM_LIT> , EndV = <NUM_LIT> ; if ( Start -> isImm ( ) ) StartV = Start -> getImm ( ) ; if ( End -> isImm ( ) ) EndV = End -> getImm ( ) ; int64_t AdjV = <NUM_LIT> ; if ( CmpHasEqual ) { if ( Start -> isImm ( ) ) StartV -- ; else if ( End -> isImm ( ) ) EndV ++ ; else AdjV += <NUM_LIT> ; } if ( Cmp != Comparison :: NE ) { if ( Start -> isImm ( ) ) StartV -= ( IVBump - <NUM_LIT> ) ; else if ( End -> isImm ( ) ) EndV += ( IVBump - <NUM_LIT> ) ; else AdjV += ( IVBump - <NUM_LIT> ) ; } unsigned R = <NUM_LIT> , SR = <NUM_LIT> ; if ( Start -> isReg ( ) ) { R = Start -> getReg ( ) ; SR = Start -> getSubReg ( ) ; } else { R = End -> getReg ( ) ; SR = End -> getSubReg ( ) ; } const TargetRegisterClass * RC = MRI -> getRegClass ( R ) ; if ( ! SR && RC == & <STR_LIT> ) return nullptr ; const TargetRegisterClass * IntRC = & <STR_LIT> ; unsigned DistR , DistSR ; if ( Start -> isImm ( ) && StartV == <NUM_LIT> ) { DistR = End -> getReg ( ) ; DistSR = End -> getSubReg ( ) ; } else { const MCInstrDesc & SubD = RegToReg ? TII -> get ( <STR_LIT> ) : ( RegToImm ? TII -> get ( <STR_LIT> ) : TII -> get ( <STR_LIT> ) ) ; unsigned SubR = MRI -> createVirtualRegister ( IntRC ) ; MachineInstrBuilder SubIB = BuildMI ( * PH , InsertPos , DL , SubD , SubR ) ; if ( RegToReg ) { SubIB . addReg ( End -> getReg ( ) , <NUM_LIT> , End -> getSubReg ( ) ) . addReg ( Start -> getReg ( ) , <NUM_LIT> , Start -> getSubReg ( ) ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return DI ; } return nullptr ;


<s> unsigned R = MO . getReg ( ) ; if ( ! defWithImmediate ( R ) ) { CmpRegs . insert ( MO . getReg ( ) ) ; continue ; } if ( CmpImmOp ) return false ; CmpImmOp = & MO ; } } else if ( MO . isImm ( ) ) { if ( CmpImmOp ) return false ; CmpImmOp = & MO ; } } if ( CmpRegs . empty ( ) ) return false ; for ( RegisterInductionSet :: iterator I = IndRegs . begin ( ) , E = IndRegs . end ( ) ; I != E ; ++ I ) { if ( CmpRegs . count ( I -> first ) ) return true ; const RegisterBump & RB = I -> second ; if ( CmpRegs . count ( RB . first ) ) { if ( ! CmpImmOp ) return false ; int64_t CmpImm = getImmediate ( * CmpImmOp ) ; int64_t V = RB . second ; if ( V > <NUM_LIT> && CmpImm + V < CmpImm ) return false ; if ( V < <NUM_LIT> && CmpImm + V > CmpImm ) return false ; CmpImm += V ; if ( CmpImmOp -> isImm ( ) && ! isInt < <NUM_LIT> > ( CmpImm ) ) return false ; MachineInstr * BumpI = MRI -> getVRegDef ( I -> first ) ; bool Order = orderBumpCompare ( BumpI , PredDef ) ; if ( ! Order ) return false ; setImmediate ( * CmpImmOp , CmpImm ) ; for ( unsigned i = <NUM_LIT> , n = PredDef -> getNumOperands ( ) ; i < n ; ++ i ) { MachineOperand & MO = PredDef -> getOperand ( i ) ; if ( MO . isReg ( ) && MO . getReg ( ) == RB . first ) {


<s> bool Negated = ( Cond . size ( ) > <NUM_LIT> ) ^ ( TB != Header ) ; unsigned PredReg = Cond [ Cond . size ( ) - <NUM_LIT> ] . getReg ( ) ; MachineInstr * CondI = MRI -> getVRegDef ( PredReg ) ; unsigned CondOpc = CondI -> getOpcode ( ) ; unsigned CmpReg1 = <NUM_LIT> , CmpReg2 = <NUM_LIT> ; int Mask = <NUM_LIT> , ImmValue = <NUM_LIT> ; bool AnalyzedCmp = TII -> analyzeCompare ( CondI , CmpReg1 , CmpReg2 , Mask , ImmValue ) ; if ( ! AnalyzedCmp ) return nullptr ; OldInsts . push_back ( CondI ) ; OldInsts . push_back ( IVOp ) ; Comparison :: Kind Cmp ; bool isSwapped = false ; const MachineOperand & Op1 = CondI -> getOperand ( <NUM_LIT> ) ; const MachineOperand & Op2 = CondI -> getOperand ( <NUM_LIT> ) ; const MachineOperand * EndValue = nullptr ; if ( Op1 . isReg ( ) ) { if ( Op2 . isImm ( ) || Op1 . getReg ( ) == IVReg ) EndValue = & Op2 ; else { EndValue = & Op1 ; isSwapped = true ; } } if ( ! EndValue ) return nullptr ; switch ( CondOpc ) { case <STR_LIT> : case <STR_LIT> : Cmp = ! Negated ? Comparison :: EQ : Comparison :: NE ; break ; case <STR_LIT> : case <STR_LIT> : Cmp = ! Negated ? Comparison :: GTu : Comparison :: LEu ; break ; case <STR_LIT> : case <STR_LIT> : Cmp = ! Negated ? Comparison :: GTs : Comparison :: LEs ; break ; case <STR_LIT> : case <STR_LIT> : { if ( IVBump != <NUM_LIT> ) return nullptr ; int64_t InitV , EndV ; assert ( EndValue -> isImm ( ) && <STR_LIT> ) ; EndV = EndValue -> getImm ( ) ; if ( InitialValue -> isReg ( ) ) { if ( ! defWithImmediate ( InitialValue -> getReg ( ) ) ) return nullptr ; InitV = getImmediate ( * InitialValue ) ; } else { assert ( InitialValue -> isImm ( ) ) ; InitV = InitialValue -> getImm ( ) ; } if ( InitV >= EndV ) return nullptr ; if ( CondOpc == <STR_LIT> ) { if ( ! isInt < <NUM_LIT> > ( InitV ) || ! isInt < <NUM_LIT> > ( EndV ) ) return nullptr ; } else { if ( ! isInt < <NUM_LIT> > ( InitV ) || ! isInt < <NUM_LIT> > ( EndV ) ) return nullptr ; } Cmp = ! Negated ? Comparison :: EQ : Comparison :: NE ; break ;


<s> MDT = & getAnalysis < MachineDominatorTree > ( ) ; TII = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) ; for ( MachineLoopInfo :: iterator I = MLI -> begin ( ) , E = MLI -> end ( ) ; I != E ; ++ I ) { MachineLoop * L = * I ; if ( ! L -> getParentLoop ( ) ) Changed |= convertToHardwareLoop ( L ) ; } return Changed ;


<s> bool Changed = false ; MLI = & getAnalysis < MachineLoopInfo > ( ) ; MRI = & MF . getRegInfo ( ) ; MDT = & getAnalysis < MachineDominatorTree > ( ) ; TII = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) ; for ( MachineLoopInfo :: iterator I = MLI -> begin ( ) , E = MLI -> end ( ) ; I != E ; ++ I ) { MachineLoop * L = * I ; if ( ! L -> getParentLoop ( ) ) Changed |= convertToHardwareLoop ( L ) ; }


<s> for ( MachineBasicBlock :: iterator MII = MBB -> begin ( ) , E = MBB -> end ( ) ; MII != E ; ++ MII ) { const MachineInstr * MI = & * MII ; if ( isInvalidLoopOperation ( MI ) ) { return true ; }


<s> BuildMI ( * MBB , MII , DL , TII -> get ( <STR_LIT> ) , Scratch ) . addImm ( MII -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ; BuildMI ( * MBB , MII , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Scratch ) ; } BuildMI ( * MBB , MII , DL , TII -> get ( <STR_LIT> ) , Scratch ) . addMBB ( MII -> getOperand ( <NUM_LIT> ) . getMBB ( ) ) ; BuildMI ( * MBB , MII , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Scratch ) ;


<s> MachineFunction * MF = LastMBB -> getParent ( ) ; const TargetRegisterClass * RC = MF -> getRegInfo ( ) . getRegClass ( TripCount -> getReg ( ) ) ; unsigned CountReg = MF -> getRegInfo ( ) . createVirtualRegister ( RC ) ; BuildMI ( * Preheader , InsertPos , InsertPos -> getDebugLoc ( ) , TII -> get ( TargetOpcode :: COPY ) , CountReg ) . addReg ( TripCount -> getReg ( ) ) ; if ( TripCount -> isNeg ( ) ) { unsigned CountReg1 = CountReg ; CountReg = MF -> getRegInfo ( ) . createVirtualRegister ( RC ) ; BuildMI ( * Preheader , InsertPos , InsertPos -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , CountReg ) . addReg ( CountReg1 ) ; } BuildMI ( * Preheader , InsertPos , InsertPos -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( LoopStart ) . addReg ( CountReg ) ; } else { assert ( TripCount -> isImm ( ) && <STR_LIT> ) ; int64_t CountImm = TripCount -> getImm ( ) ; BuildMI ( * Preheader , InsertPos , InsertPos -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( LoopStart ) . addImm ( CountImm ) ; } LoopStart -> setHasAddressTaken ( ) ; BlockAddress :: get ( const_cast < BasicBlock * > ( LoopStart -> getBasicBlock ( ) ) ) ; DebugLoc dl = LastI -> getDebugLoc ( ) ; BuildMI ( * LastMBB , LastI , dl , TII -> get ( <STR_LIT> ) ) . addMBB ( LoopStart ) ; if ( LastI -> getOpcode ( ) == <STR_LIT> || LastI -> getOpcode ( ) == <STR_LIT> ) { MachineBasicBlock * BranchTarget = LastI -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; LastI = LastMBB -> erase ( LastI ) ; if ( ! L -> contains ( BranchTarget ) ) { if ( LastI != LastMBB -> end ( ) ) {


<s> } MachineBasicBlock :: iterator InsertPos = Preheader -> getFirstTerminator ( ) ; MachineBasicBlock * LastMBB = L -> getExitingBlock ( ) ; if ( LastMBB == <NUM_LIT> ) { return false ; } MachineBasicBlock :: iterator LastI = LastMBB -> getFirstTerminator ( ) ; MachineBasicBlock * LoopStart = L -> getTopBlock ( ) ; if ( L -> getLoopLatch ( ) != LastMBB ) { LoopStart = L -> getLoopLatch ( ) ; if ( ! LastMBB -> isSuccessor ( LoopStart ) ) { return false ; } } DEBUG ( dbgs ( ) << <STR_LIT> ; L -> dump ( ) ) ; if ( TripCount -> isReg ( ) ) { MachineFunction * MF = LastMBB -> getParent ( ) ; const TargetRegisterClass * RC = MF -> getRegInfo ( ) . getRegClass ( TripCount -> getReg ( ) ) ; unsigned CountReg = MF -> getRegInfo ( ) . createVirtualRegister ( RC ) ; BuildMI ( * Preheader , InsertPos , InsertPos -> getDebugLoc ( ) , TII -> get ( TargetOpcode :: COPY ) , CountReg ) . addReg ( TripCount -> getReg ( ) ) ; if ( TripCount -> isNeg ( ) ) { unsigned CountReg1 = CountReg ; CountReg = MF -> getRegInfo ( ) . createVirtualRegister ( RC ) ;

BuildMI ( * Preheader , InsertPos , InsertPos -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , CountReg ) . addReg ( CountReg1 ) ;


<s> explicit CountValue ( int64_t i ) : Kind ( CV_Immediate ) , Contents ( i ) , isNegative ( i < <NUM_LIT> ) {


<s> MBB != MBBe ; ++ MBB ) { BlockToInstOffset [ MBB ] = InstOffset ; InstOffset += ( MBB -> size ( ) * <NUM_LIT> ) ; } InstOffset = <NUM_LIT> ; bool Changed = false ; RegScavenger RS ; for ( MachineFunction :: iterator MBB = MF . begin ( ) , MBBe = MF . end ( ) ; MBB != MBBe ; ++ MBB ) { InstOffset = BlockToInstOffset [ MBB ] ; RS . enterBasicBlock ( MBB ) ; MachineBasicBlock :: iterator MIE = MBB -> end ( ) ; MachineBasicBlock :: iterator MII = MBB -> begin ( ) ; while ( MII != MIE ) { if ( isHardwareLoop ( MII ) ) {


<s> InstOffset = <NUM_LIT> ; bool Changed = false ; RegScavenger RS ; for ( MachineFunction :: iterator MBB = MF . begin ( ) , MBBe = MF . end ( ) ; MBB != MBBe ; ++ MBB ) { InstOffset = BlockToInstOffset [ MBB ] ; RS . enterBasicBlock ( MBB ) ; MachineBasicBlock :: iterator MIE = MBB -> end ( ) ; MachineBasicBlock :: iterator MII = MBB -> begin ( ) ; while ( MII != MIE ) { if ( isHardwareLoop ( MII ) ) { RS . forward ( MII ) ; assert ( MII -> getOperand ( <NUM_LIT> ) . isMBB ( ) && <STR_LIT> ) ; int diff = InstOffset - BlockToInstOffset [ MII -> getOperand ( <NUM_LIT> ) . getMBB ( ) ] ; diff = ( diff > <NUM_LIT> ? diff : - diff ) ; if ( ( unsigned ) diff > MAX_LOOP_DISTANCE ) { convertLoopInstr ( MF , MII , RS ) ; MII = MBB -> erase ( MII ) ; Changed = true ; } else { ++ MII ;


<s> AU . addRequired < MachineDominatorTree > ( ) ; AU . addPreserved < MachineDominatorTree > ( ) ; AU . addRequired < MachineLoopInfo > ( ) ; AU . addPreserved < MachineLoopInfo > ( ) ; MachineFunctionPass :: getAnalysisUsage ( AU ) ;


<s> unsigned DefReg = MPhi -> getOperand ( <NUM_LIT> ) . getReg ( ) ; for ( unsigned i = <NUM_LIT> ; i != MPhi -> getNumOperands ( ) ; i += <NUM_LIT> ) { MachineBasicBlock * MBB = MPhi -> getOperand ( i + <NUM_LIT> ) . getMBB ( ) ; if ( L -> contains ( MBB ) ) { const MachineInstr * DI = MRI -> getVRegDef ( MPhi -> getOperand ( i ) . getReg ( ) ) ;

if ( isInductionOperation ( DI , DefReg ) ) {


<s> unsigned getReg ( ) const { assert ( isReg ( ) && <STR_LIT> ) ;


<s> return new CountValue ( InitialValue -> getReg ( ) , iv_value > <NUM_LIT> ) ; } else { assert ( InitialValue -> isReg ( ) && <STR_LIT> ) ; const MachineInstr * DefInstr = MRI -> getVRegDef ( InitialValue -> getReg ( ) ) ; if ( DefInstr && DefInstr -> getOpcode ( ) == <STR_LIT> ) { int64_t count = ImmVal - DefInstr -> getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( ( count % iv_value ) != <NUM_LIT> ) { return <NUM_LIT> ; } return new CountValue ( count / iv_value ) ; } } } } return <NUM_LIT> ;


<s> CountValueType getType ( ) const { return Kind ;


<s> <ISA_LIT>FixupHwLoops ( ) : MachineFunctionPass ( ID ) {


<s> return MI -> getOpcode ( ) == <STR_LIT> ;


<s> bool <ISA_LIT>HardwareLoops :: isInductionOperation ( const MachineInstr * MI , unsigned IVReg ) const {

return ( MI -> getOpcode ( ) == <STR_LIT> && MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == IVReg ) ;


<s> } for ( unsigned i = <NUM_LIT> , e = MI -> getNumOperands ( ) ; i != e ; ++ i ) { const MachineOperand & MO = MI -> getOperand ( i ) ; if ( MO . isReg ( ) && MO . isDef ( ) && ( MO . getReg ( ) == <STR_LIT> || MO . getReg ( ) == <STR_LIT> || MO . getReg ( ) == <STR_LIT> || MO . getReg ( ) == <STR_LIT> ) ) { return true ; } }


<s> if ( isReg ( ) ) { OS << PrintReg ( getReg ( ) ) ;


<s> bool Changed = fixupLoopInstrs ( MF ) ;


<s> void setReg ( unsigned Val ) { Contents . RegNum = Val ;


<s> Values ( int64_t i ) : ImmVal ( i ) {


<s> DebugLoc DL = MII -> getDebugLoc ( ) ; unsigned Scratch = RS . scavengeRegister ( & <STR_LIT> , MII , <NUM_LIT> ) ; if ( MII -> getOperand ( <NUM_LIT> ) . isReg ( ) ) { BuildMI ( * MBB , MII , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( MII -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; } else { BuildMI ( * MBB , MII , DL , TII -> get ( <STR_LIT> ) , Scratch ) . addImm ( MII -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ; BuildMI ( * MBB , MII , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Scratch ) ;


<s> void <ISA_LIT>FixupHwLoops :: convertLoopInstr ( MachineFunction & MF , MachineBasicBlock :: iterator & MII , RegScavenger & RS ) { const TargetInstrInfo * TII = MF . getTarget ( ) . getInstrInfo ( ) ; MachineBasicBlock * MBB = MII -> getParent ( ) ; DebugLoc DL = MII -> getDebugLoc ( ) ; unsigned Scratch = RS . scavengeRegister ( & <STR_LIT> , MII , <NUM_LIT> ) ;

if ( MII -> getOperand ( <NUM_LIT> ) . isReg ( ) ) {


<s> if ( ! checkForImmediate ( S1 , V1 ) || ! checkForImmediate ( S2 , V2 ) ) return false ; TV = V2 | ( static_cast < uint64_t > ( V1 ) << <NUM_LIT> ) ; break ; } case TargetOpcode :: REG_SEQUENCE : { const MachineOperand & S1 = DI -> getOperand ( <NUM_LIT> ) ; const MachineOperand & S3 = DI -> getOperand ( <NUM_LIT> ) ; int64_t V1 , V3 ; if ( ! checkForImmediate ( S1 , V1 ) || ! checkForImmediate ( S3 , V3 ) ) return false ; unsigned Sub2 = DI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; unsigned Sub4 = DI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( Sub2 == <STR_LIT> && Sub4 == <STR_LIT> ) TV = V1 | ( V3 << <NUM_LIT> ) ; else if ( Sub2 == <STR_LIT> && Sub4 == <STR_LIT> ) TV = V3 | ( V1 << <NUM_LIT> ) ; else llvm_unreachable ( <STR_LIT> ) ; break ; } default : return false ; } switch ( MO . getSubReg ( ) ) {


<s> DEBUG ( dbgs ( ) << <STR_LIT> ; L -> dump ( ) ) ; DebugLoc DL ; if ( InsertPos != Preheader -> end ( ) ) DL = InsertPos -> getDebugLoc ( ) ; if ( TripCount -> isReg ( ) ) { unsigned CountReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * Preheader , InsertPos , DL , TII -> get ( TargetOpcode :: COPY ) , CountReg ) . addReg ( TripCount -> getReg ( ) , <NUM_LIT> , TripCount -> getSubReg ( ) ) ; BuildMI ( * Preheader , InsertPos , DL , TII -> get ( LOOP_r ) ) . addMBB ( LoopStart ) . addReg ( CountReg ) ; } else { assert ( TripCount -> isImm ( ) && <STR_LIT> ) ; int64_t CountImm = TripCount -> getImm ( ) ; if ( ! TII -> isValidOffset ( LOOP_i , CountImm , TRI ) ) { unsigned CountReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * Preheader , InsertPos , DL , TII -> get ( <STR_LIT> ) , CountReg ) . addImm ( CountImm ) ; BuildMI ( * Preheader , InsertPos , DL , TII -> get ( LOOP_r ) ) . addMBB ( LoopStart ) . addReg ( CountReg ) ; } else BuildMI ( * Preheader , InsertPos , DL , TII -> get ( LOOP_i ) ) . addMBB ( LoopStart ) . addImm ( CountImm ) ; } LoopStart -> setHasAddressTaken ( ) ; BlockAddress :: get ( const_cast < BasicBlock * > ( LoopStart -> getBasicBlock ( ) ) ) ; DebugLoc LastIDL = LastI -> getDebugLoc ( ) ; BuildMI ( * LastMBB , LastI , LastIDL , TII -> get ( ENDLOOP ) ) . addMBB ( LoopStart ) ; if ( LastI -> getOpcode ( ) == <STR_LIT> || LastI -> getOpcode ( ) == <STR_LIT> ) { MachineBasicBlock * BranchTarget = LastI -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; LastI = LastMBB -> erase ( LastI ) ; if ( ! L -> contains ( BranchTarget ) ) { if ( LastI != LastMBB -> end ( ) ) LastI = LastMBB -> erase ( LastI ) ; SmallVector < MachineOperand , <NUM_LIT> > Cond ; TII -> insertBranch ( * LastMBB , BranchTarget , nullptr , Cond , LastIDL ) ; } } else { LastMBB -> erase ( LastI ) ; } delete TripCount ; for ( unsigned i = <NUM_LIT> ; i < OldInsts . size ( ) ; ++ i ) removeIfDead ( OldInsts [ i ] ) ; ++ NumHWLoops ; if ( L0Used ) RecL1used = true ; else RecL0used = true ; return true ;


<s> } } PN -> addOperand ( MachineOperand :: CreateReg ( NewPR , false ) ) ; PN -> addOperand ( MachineOperand :: CreateMBB ( NewPH ) ) ; } } else { assert ( Header -> pred_size ( ) == <NUM_LIT> ) ; for ( instr_iterator I = Header -> instr_begin ( ) , E = Header -> instr_end ( ) ; I != E && I -> isPHI ( ) ; ++ I ) { MachineInstr * PN = & * I ; for ( unsigned i = <NUM_LIT> , n = PN -> getNumOperands ( ) ; i < n ; i += <NUM_LIT> ) { MachineOperand & MO = PN -> getOperand ( i + <NUM_LIT> ) ; if ( MO . getMBB ( ) != Latch ) MO . setMBB ( NewPH ) ; } } } SmallVector < MachineOperand , <NUM_LIT> > Tmp2 ; SmallVector < MachineOperand , <NUM_LIT> > EmptyCond ; TB = FB = nullptr ; for ( MBBVector :: iterator I = Preds . begin ( ) , E = Preds . end ( ) ; I != E ; ++ I ) { MachineBasicBlock * PB = * I ; if ( PB != Latch ) { Tmp2 . clear ( ) ; bool NotAnalyzed = TII -> analyzeBranch ( * PB , TB , FB , Tmp2 , false ) ; ( void ) NotAnalyzed ; assert ( ! NotAnalyzed && <STR_LIT> ) ; if ( TB != Header && ( Tmp2 . empty ( ) || FB != Header ) ) TII -> insertBranch ( * PB , NewPH , nullptr , EmptyCond , DL ) ; PB -> ReplaceUsesOfBlockWith ( Header , NewPH ) ; } } TB = FB = nullptr ; bool LatchNotAnalyzed = TII -> analyzeBranch ( * Latch , TB , FB , Tmp2 , false ) ; ( void ) LatchNotAnalyzed ; assert ( ! LatchNotAnalyzed && <STR_LIT> ) ; if ( ! TB && ! FB ) TII -> insertBranch ( * Latch , Header , nullptr , EmptyCond , DL ) ;

TII -> insertBranch ( * NewPH , Header , nullptr , EmptyCond , DL ) ;


<s> bool <ISA_LIT>HardwareLoops :: findInductionRegister ( MachineLoop * L , unsigned & Reg , int64_t & IVBump , MachineInstr * & IVOp ) const { MachineBasicBlock * Header = L -> getHeader ( ) ; MachineBasicBlock * Preheader = MLI -> findLoopPreheader ( L , SpecPreheader ) ; MachineBasicBlock * Latch = L -> getLoopLatch ( ) ; MachineBasicBlock * ExitingBlock = L -> findLoopControlBlock ( ) ; if ( ! Header || ! Preheader || ! Latch || ! ExitingBlock ) return false ; using RegisterBump = std :: pair < unsigned , int64_t > ; using InductionMap = std :: map < unsigned , RegisterBump > ; InductionMap IndMap ; using instr_iterator = MachineBasicBlock :: instr_iterator ; for ( instr_iterator I = Header -> instr_begin ( ) , E = Header -> instr_end ( ) ; I != E && I -> isPHI ( ) ; ++ I ) { MachineInstr * Phi = & * I ; for ( unsigned i = <NUM_LIT> , n = Phi -> getNumOperands ( ) ; i < n ; i += <NUM_LIT> ) { if ( Phi -> getOperand ( i + <NUM_LIT> ) . getMBB ( ) != Latch ) continue ; unsigned PhiOpReg = Phi -> getOperand ( i ) . getReg ( ) ; MachineInstr * DI = MRI -> getVRegDef ( PhiOpReg ) ; if ( DI -> getDesc ( ) . isAdd ( ) ) { unsigned IndReg = DI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineOperand & Opnd2 = DI -> getOperand ( <NUM_LIT> ) ; int64_t V ; if ( MRI -> getVRegDef ( IndReg ) == Phi && checkForImmediate ( Opnd2 , V ) ) { unsigned UpdReg = DI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; IndMap . insert ( std :: make_pair ( UpdReg , std :: make_pair ( IndReg , V ) ) ) ; } } } } SmallVector < MachineOperand , <NUM_LIT> > Cond ; MachineBasicBlock * TB = nullptr , * FB = nullptr ; bool NotAnalyzed = TII -> analyzeBranch ( * ExitingBlock , TB , FB , Cond , false ) ; if ( NotAnalyzed ) return false ; unsigned PredR , PredPos , PredRegFlags ; if ( ! TII -> getPredReg ( Cond , PredR , PredPos , PredRegFlags ) ) return false ; MachineInstr * PredI = MRI -> getVRegDef ( PredR ) ;

if ( ! PredI -> isCompare ( ) ) return false ;


<s> using RegisterInductionSet = std :: set < RegisterInduction > ; RegisterInductionSet IndRegs ; using instr_iterator = MachineBasicBlock :: instr_iterator ; for ( instr_iterator I = Header -> instr_begin ( ) , E = Header -> instr_end ( ) ; I != E && I -> isPHI ( ) ; ++ I ) { MachineInstr * Phi = & * I ; for ( unsigned i = <NUM_LIT> , n = Phi -> getNumOperands ( ) ; i < n ; i += <NUM_LIT> ) { if ( Phi -> getOperand ( i + <NUM_LIT> ) . getMBB ( ) != Latch ) continue ; unsigned PhiReg = Phi -> getOperand ( i ) . getReg ( ) ; MachineInstr * DI = MRI -> getVRegDef ( PhiReg ) ; if ( DI -> getDesc ( ) . isAdd ( ) ) { unsigned IndReg = DI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineOperand & Opnd2 = DI -> getOperand ( <NUM_LIT> ) ; int64_t V ; if ( MRI -> getVRegDef ( IndReg ) == Phi && checkForImmediate ( Opnd2 , V ) ) { unsigned UpdReg = DI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; IndRegs . insert ( std :: make_pair ( UpdReg , std :: make_pair ( IndReg , V ) ) ) ; } } } } if ( IndRegs . empty ( ) ) return false ; MachineBasicBlock * TB = nullptr , * FB = nullptr ; SmallVector < MachineOperand , <NUM_LIT> > Cond ; bool NotAnalyzed = TII -> analyzeBranch ( * ExitingBlock , TB , FB , Cond , false ) ; if ( NotAnalyzed || Cond . empty ( ) ) return false ; if ( ExitingBlock != Latch && ( TB == Latch || FB == Latch ) ) { MachineBasicBlock * LTB = nullptr , * LFB = nullptr ; SmallVector < MachineOperand , <NUM_LIT> > LCond ; bool NotAnalyzed = TII -> analyzeBranch ( * Latch , LTB , LFB , LCond , false ) ; if ( NotAnalyzed ) return false ; if ( TB == Latch ) TB = ( LTB == Header ) ? LTB : LFB ; else FB = ( LTB == Header ) ? LTB : LFB ; } if ( TB != Header ) { if ( FB != Header ) { return false ; } if ( MDT -> dominates ( TB , FB ) ) return false ; } unsigned CSz = Cond . size ( ) ; if ( CSz != <NUM_LIT> && CSz != <NUM_LIT> ) return false ; if ( ! Cond [ CSz - <NUM_LIT> ] . isReg ( ) ) return false ; unsigned P = Cond [ CSz - <NUM_LIT> ] . getReg ( ) ; MachineInstr * PredDef = MRI -> getVRegDef ( P ) ; if ( ! PredDef -> isCompare ( ) ) return false ; SmallSet < unsigned , <NUM_LIT> > CmpRegs ;


<s> Cmp = Comparison :: LTs ; break ; case <STR_LIT> : Cmp = Comparison :: LTu ; break ; case <STR_LIT> : case <STR_LIT> : Cmp = Comparison :: LEs ; break ; case <STR_LIT> : case <STR_LIT> : Cmp = Comparison :: LEu ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : Cmp = Comparison :: GTs ; break ; case <STR_LIT> :


<s> case <STR_LIT> : case <STR_LIT> : Cmp = Comparison :: EQ ; break ; case <STR_LIT> : case <STR_LIT> : Cmp = Comparison :: NE ; break ; case <STR_LIT> : Cmp = Comparison :: LTs ; break ; case <STR_LIT> : Cmp = Comparison :: LTu ; break ; case <STR_LIT> : case <STR_LIT> : Cmp = Comparison :: LEs ; break ; case <STR_LIT> : case <STR_LIT> : Cmp = Comparison :: LEu ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : Cmp = Comparison :: GTs ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : Cmp = Comparison :: GTu ; break ; case <STR_LIT> : Cmp = Comparison :: GEs ; break ; case <STR_LIT> : Cmp = Comparison :: GEs ; break ; default : return ( Comparison :: Kind ) <NUM_LIT> ;


<s> static Kind getNegatedComparison ( Kind Cmp ) { if ( ( Cmp & L ) || ( Cmp & G ) ) return ( Kind ) ( ( Cmp ^ ( L | G ) ) ^ EQ ) ; if ( ( Cmp & NE ) || ( Cmp & EQ ) ) return ( Kind ) ( Cmp ^ ( EQ | NE ) ) ;


<s> <ISA_LIT>HardwareLoops ( ) : MachineFunctionPass ( ID ) {


<s> MachineInstr * OnePhi = I -> getParent ( ) ; for ( unsigned j = <NUM_LIT> , f = OnePhi -> getNumOperands ( ) ; j != f ; ++ j ) { const MachineOperand & OPO = OnePhi -> getOperand ( j ) ; if ( ! OPO . isReg ( ) || ! OPO . isDef ( ) ) continue ; unsigned OPReg = OPO . getReg ( ) ; use_nodbg_iterator nextJ ; for ( use_nodbg_iterator J = MRI -> use_nodbg_begin ( OPReg ) ; J != End ; J = nextJ ) { nextJ = std :: next ( J ) ;


<s> bool isImmediate ( const MachineOperand & MO ) const { int64_t V ; return checkForImmediate ( MO , V ) ;


<s> return checkForImmediate ( MO , V ) ;


<s> if ( CmpOpc == <STR_LIT> ) return isUInt < <NUM_LIT> > ( Imm ) ; if ( CmpOpc == <STR_LIT> ) return isInt < <NUM_LIT> > ( Imm ) ; return true ;


<s> using namespace <ISA_LIT> ; static const unsigned Regs01 [ ] = { LC0 , SA0 , LC1 , SA1 } ; static const unsigned Regs1 [ ] = { LC1 , SA1 } ; auto CheckRegs = IsInnerHWLoop ? makeArrayRef ( Regs01 , array_lengthof ( Regs01 ) ) : makeArrayRef ( Regs1 , array_lengthof ( Regs1 ) ) ; for ( unsigned R : CheckRegs ) if ( MI -> modifiesRegister ( R , TRI ) ) return true ;


<s> static const unsigned Regs01 [ ] = { LC0 , SA0 , LC1 , SA1 } ; static const unsigned Regs1 [ ] = { LC1 , SA1 } ; auto CheckRegs = IsInnerHWLoop ? makeArrayRef ( Regs01 , array_lengthof ( Regs01 ) ) : makeArrayRef ( Regs1 , array_lengthof ( Regs1 ) ) ; for ( unsigned R : CheckRegs ) if ( MI -> modifiesRegister ( R , TRI ) ) return true ;


<s> MachineBasicBlock * MBB = Blocks [ i ] ; if ( A == MBB ) return false ; } MachineInstr * Def = MRI -> getVRegDef ( MO -> getReg ( ) ) ; LoopFeederPhi . insert ( std :: make_pair ( MO -> getReg ( ) , Def ) ) ; return true ;


<s> static bool isSigned ( Kind Cmp ) { return ( Cmp & ( L | G ) && ! ( Cmp & U ) ) ;


<s> static bool isUnsigned ( Kind Cmp ) {


<s> if ( ! TII -> analyzeCompare ( * MI , CmpReg1 , CmpReg2 , CmpMask , CmpValue ) ) continue ; MachineBasicBlock * TBB = nullptr , * FBB = nullptr ; SmallVector < MachineOperand , <NUM_LIT> > Cond ; if ( TII -> analyzeBranch ( * MI -> getParent ( ) , TBB , FBB , Cond , false ) ) continue ; Comparison :: Kind Cmp = getComparisonKind ( MI -> getOpcode ( ) , nullptr , nullptr , <NUM_LIT> ) ; if ( Cmp == <NUM_LIT> ) continue ; if ( TII -> predOpcodeHasNot ( Cond ) ^ ( TBB != MBB ) ) Cmp = Comparison :: getNegatedComparison ( Cmp ) ; if ( CmpReg2 != <NUM_LIT> && CmpReg2 == Reg ) Cmp = Comparison :: getSwappedComparison ( Cmp ) ; if ( Comparison :: isSigned ( Cmp ) ) return false ; if ( ( Cmp & Comparison :: G ) || Cmp == Comparison :: NE ) return false ;


<s> for ( instr_iterator I = NextIt , E = BB -> instr_end ( ) ; I != E ; ++ I ) { MachineInstr * In = & * I ; for ( unsigned i = <NUM_LIT> , n = In -> getNumOperands ( ) ; i < n ; ++ i ) { MachineOperand & MO = In -> getOperand ( i ) ; if ( MO . isReg ( ) && MO . isUse ( ) ) {

if ( MO . getReg ( ) == PredR ) return false ;


<s> assert ( Phi -> isPHI ( ) && <STR_LIT> ) ; for ( int i = <NUM_LIT> , n = Phi -> getNumOperands ( ) ; i < n ; i += <NUM_LIT> ) if ( isLoopFeeder ( L , MBB , Phi , & ( Phi -> getOperand ( i ) ) , LoopFeederPhi ) ) if ( loopCountMayWrapOrUnderFlow ( & ( Phi -> getOperand ( i ) ) , EndVal , Phi -> getParent ( ) , L , LoopFeederPhi ) ) return true ;


<s> for ( int i = <NUM_LIT> , n = Phi -> getNumOperands ( ) ; i < n ; i += <NUM_LIT> ) if ( isLoopFeeder ( L , MBB , Phi , & ( Phi -> getOperand ( i ) ) , LoopFeederPhi ) ) if ( loopCountMayWrapOrUnderFlow ( & ( Phi -> getOperand ( i ) ) , EndVal , Phi -> getParent ( ) , L , LoopFeederPhi ) ) return true ;


OS << printReg ( Contents . R . Reg , TRI , Contents . R . Sub ) ;


<s> bool <ISA_LIT>HardwareLoops :: runOnMachineFunction ( MachineFunction & MF ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( skipFunction ( MF . getFunction ( ) ) ) return false ; bool Changed = false ; MLI = & getAnalysis < MachineLoopInfo > ( ) ; MRI = & MF . getRegInfo ( ) ; MDT = & getAnalysis < MachineDominatorTree > ( ) ; const <ISA_LIT>Subtarget & HST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = HST . getInstrInfo ( ) ; TRI = HST . getRegisterInfo ( ) ; for ( auto & L : * MLI ) if ( ! L -> getParentLoop ( ) ) { bool L0Used = false ; bool L1Used = false ; Changed |= convertToHardwareLoop ( L , L0Used , L1Used ) ;


<s> DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( skipFunction ( MF . getFunction ( ) ) ) return false ; bool Changed = false ; MLI = & getAnalysis < MachineLoopInfo > ( ) ; MRI = & MF . getRegInfo ( ) ; MDT = & getAnalysis < MachineDominatorTree > ( ) ; const <ISA_LIT>Subtarget & HST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = HST . getInstrInfo ( ) ; TRI = HST . getRegisterInfo ( ) ;

for ( auto & L : * MLI ) if ( ! L -> getParentLoop ( ) ) {


<s> MachineInstr * DI = MRI -> getVRegDef ( R ) ; const TargetRegisterClass * RC = MRI -> getRegClass ( R ) ; unsigned NewR = MRI -> createVirtualRegister ( RC ) ; MachineBasicBlock & B = * DI -> getParent ( ) ; DebugLoc DL = DI -> getDebugLoc ( ) ;

BuildMI ( B , DI , DL , TII -> get ( DI -> getOpcode ( ) ) , NewR ) . addImm ( Val ) ;


<s> unsigned R = MO . getReg ( ) ; MachineInstr * DI = MRI -> getVRegDef ( R ) ; const TargetRegisterClass * RC = MRI -> getRegClass ( R ) ; unsigned NewR = MRI -> createVirtualRegister ( RC ) ; MachineBasicBlock & B = * DI -> getParent ( ) ; DebugLoc DL = DI -> getDebugLoc ( ) ;


<s> if ( IV_Inst == <NUM_LIT> ) return <NUM_LIT> ; const MachineOperand * IV_Opnd ; const MachineOperand * InitialValue ; if ( ! L -> contains ( IV_Inst -> getOperand ( <NUM_LIT> ) . getMBB ( ) ) ) { InitialValue = & IV_Inst -> getOperand ( <NUM_LIT> ) ; IV_Opnd = & IV_Inst -> getOperand ( <NUM_LIT> ) ; } else { InitialValue = & IV_Inst -> getOperand ( <NUM_LIT> ) ; IV_Opnd = & IV_Inst -> getOperand ( <NUM_LIT> ) ; } for ( MachineRegisterInfo :: reg_iterator RI = MRI -> reg_begin ( IV_Opnd -> getReg ( ) ) , RE = MRI -> reg_end ( ) ; RI != RE ; ++ RI ) { IV_Opnd = & RI . getOperand ( ) ; const MachineInstr * MI = IV_Opnd -> getParent ( ) ; if ( L -> contains ( MI ) && isCompareEqualsImm ( MI ) ) { const MachineOperand & MO = MI -> getOperand ( <NUM_LIT> ) ; assert ( MO . isImm ( ) && <STR_LIT> ) ; int64_t ImmVal = MO . getImm ( ) ; const MachineInstr * IV_DefInstr = MRI -> getVRegDef ( IV_Opnd -> getReg ( ) ) ; assert ( L -> contains ( IV_DefInstr -> getParent ( ) ) && <STR_LIT> ) ; int64_t iv_value = IV_DefInstr -> getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( ImmVal == <NUM_LIT> ) { if ( iv_value != <NUM_LIT> && iv_value != - <NUM_LIT> ) { return <NUM_LIT> ; } return new CountValue ( InitialValue -> getReg ( ) , iv_value > <NUM_LIT> ) ; } else { assert ( InitialValue -> isReg ( ) && <STR_LIT> ) ; const MachineInstr * DefInstr = MRI -> getVRegDef ( InitialValue -> getReg ( ) ) ; if ( DefInstr && DefInstr -> getOpcode ( ) == <STR_LIT> ) { int64_t count = ImmVal - DefInstr -> getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( ( count % iv_value ) != <NUM_LIT> ) { return <NUM_LIT> ; } return new CountValue ( count / iv_value ) ; } } } } return <NUM_LIT> ;


<s> } MachineBasicBlock * TopBlock = L -> getTopBlock ( ) ; MachineBasicBlock * ExitingBlock = L -> findLoopControlBlock ( ) ; MachineBasicBlock * LoopStart = nullptr ; if ( ExitingBlock != L -> getLoopLatch ( ) ) { MachineBasicBlock * TB = nullptr , * FB = nullptr ; SmallVector < MachineOperand , <NUM_LIT> > Cond ; if ( TII -> analyzeBranch ( * ExitingBlock , TB , FB , Cond , false ) ) return false ; if ( L -> contains ( TB ) ) LoopStart = TB ; else if ( L -> contains ( FB ) ) LoopStart = FB ; else return false ; } else LoopStart = TopBlock ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; L -> dump ( ) ) ; DebugLoc DL ; if ( InsertPos != Preheader -> end ( ) ) DL = InsertPos -> getDebugLoc ( ) ; if ( TripCount -> isReg ( ) ) { Register CountReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * Preheader , InsertPos , DL , TII -> get ( TargetOpcode :: COPY ) , CountReg ) . addReg ( TripCount -> getReg ( ) , <NUM_LIT> , TripCount -> getSubReg ( ) ) ; BuildMI ( * Preheader , InsertPos , DL , TII -> get ( LOOP_r ) ) . addMBB ( LoopStart ) . addReg ( CountReg ) ; } else { assert ( TripCount -> isImm ( ) && <STR_LIT> ) ; int64_t CountImm = TripCount -> getImm ( ) ; if ( ! TII -> isValidOffset ( LOOP_i , CountImm , TRI ) ) { Register CountReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * Preheader , InsertPos , DL , TII -> get ( <STR_LIT> ) , CountReg ) . addImm ( CountImm ) ;


<s> const TargetRegisterInfo * TRI = TM ? TM -> getSubtargetImpl ( ) -> getRegisterInfo ( ) : nullptr ; if ( isReg ( ) ) { OS << PrintReg ( Contents . R . Reg , TRI , Contents . R . Sub ) ; } if ( isImm ( ) ) { OS << Contents . ImmVal ; }


<s> TII = static_cast < const <ISA_LIT>InstrInfo * > ( TM -> getSubtargetImpl ( ) -> getInstrInfo ( ) ) ; TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( TM -> getSubtargetImpl ( ) -> getRegisterInfo ( ) ) ; for ( MachineLoopInfo :: iterator I = MLI -> begin ( ) , E = MLI -> end ( ) ; I != E ; ++ I ) { MachineLoop * L = * I ;

if ( ! L -> getParentLoop ( ) ) Changed |= convertToHardwareLoop ( L ) ;


<s> for ( MachineBasicBlock * MBB : L -> getBlocks ( ) ) {

for ( MachineBasicBlock :: iterator MII = MBB -> begin ( ) , E = MBB -> end ( ) ; MII != E ; ++ MII ) {


<s> Register PR = PN -> getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * RC = MRI -> getRegClass ( PR ) ; Register NewPR = MRI -> createVirtualRegister ( RC ) ; NewPN -> addOperand ( MachineOperand :: CreateReg ( NewPR , true ) ) ; for ( unsigned i = <NUM_LIT> , n = PN -> getNumOperands ( ) ; i < n ; i += <NUM_LIT> ) { Register PredR = PN -> getOperand ( i ) . getReg ( ) ; unsigned PredRSub = PN -> getOperand ( i ) . getSubReg ( ) ; MachineBasicBlock * PredB = PN -> getOperand ( i + <NUM_LIT> ) . getMBB ( ) ; if ( PredB == Latch ) continue ; MachineOperand MO = MachineOperand :: CreateReg ( PredR , false ) ; MO . setSubReg ( PredRSub ) ; NewPN -> addOperand ( MO ) ; NewPN -> addOperand ( MachineOperand :: CreateMBB ( PredB ) ) ; } for ( int i = PN -> getNumOperands ( ) - <NUM_LIT> ; i > <NUM_LIT> ; i -= <NUM_LIT> ) { MachineBasicBlock * PredB = PN -> getOperand ( i + <NUM_LIT> ) . getMBB ( ) ; if ( PredB != Latch ) { PN -> RemoveOperand ( i + <NUM_LIT> ) ; PN -> RemoveOperand ( i ) ; } } PN -> addOperand ( MachineOperand :: CreateReg ( NewPR , false ) ) ; PN -> addOperand ( MachineOperand :: CreateMBB ( NewPH ) ) ; } } else { assert ( Header -> pred_size ( ) == <NUM_LIT> ) ; for ( instr_iterator I = Header -> instr_begin ( ) , E = Header -> instr_end ( ) ; I != E && I -> isPHI ( ) ; ++ I ) { MachineInstr * PN = & * I ; for ( unsigned i = <NUM_LIT> , n = PN -> getNumOperands ( ) ; i < n ; i += <NUM_LIT> ) { MachineOperand & MO = PN -> getOperand ( i + <NUM_LIT> ) ; if ( MO . getMBB ( ) != Latch ) MO . setMBB ( NewPH ) ; } } } SmallVector < MachineOperand , <NUM_LIT> > Tmp2 ; SmallVector < MachineOperand , <NUM_LIT> > EmptyCond ; TB = FB = nullptr ; for ( MBBVector :: iterator I = Preds . begin ( ) , E = Preds . end ( ) ; I != E ; ++ I ) { MachineBasicBlock * PB = * I ; if ( PB != Latch ) { Tmp2 . clear ( ) ; bool NotAnalyzed = TII -> analyzeBranch ( * PB , TB , FB , Tmp2 , false ) ;