<s> static void addAsmInstr ( MachineBasicBlock * MBB , unsigned Reg , MachineBasicBlock :: instr_iterator I , const DebugLoc & DL , const <ISA_LIT>InstrInfo * QII , MachineFunction & Fn ) { std :: string VDescStr = <STR_LIT> + getStringReg ( Reg ) ; const char * cstr = Fn . createExternalSymbolName ( VDescStr ) ; unsigned ExtraInfo = InlineAsm :: Extra_HasSideEffects ;

BuildMI ( * MBB , I , DL , QII -> get ( TargetOpcode :: INLINEASM ) ) . addExternalSymbol ( cstr ) . addImm ( ExtraInfo ) ;


<s> const char * cstr = Fn . createExternalSymbolName ( VDescStr ) ; unsigned ExtraInfo = InlineAsm :: Extra_HasSideEffects ;


<s> StringRef getPassName ( ) const override {


<s> initialize<ISA_LIT>VectorPrintPass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MII -> dump ( ) ) ; } } } else { unsigned Reg = <NUM_LIT> ; if ( getInstrVecReg ( MI , Reg ) ) { VecPrintList . push_back ( & MI ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MI . dump ( ) ) ; } } } Changed = ! VecPrintList . empty ( ) ; if ( ! Changed ) return Changed ; for ( auto * I : VecPrintList ) { DebugLoc DL = I -> getDebugLoc ( ) ; MachineBasicBlock * MBB = I -> getParent ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; I -> dump ( ) ) ; unsigned Reg = <NUM_LIT> ; if ( ! getInstrVecReg ( * I , Reg ) ) llvm_unreachable ( <STR_LIT> ) ; MachineBasicBlock :: instr_iterator MII = I -> getIterator ( ) ; if ( I -> isInsideBundle ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; I -> dump ( ) ) ; while ( MBB -> instr_end ( ) != MII && MII -> isInsideBundle ( ) ) MII ++ ;


<s> QST = & Fn . getSubtarget < <ISA_LIT>Subtarget > ( ) ; QRI = QST -> getRegisterInfo ( ) ; QII = QST -> getInstrInfo ( ) ; std :: vector < MachineInstr * > VecPrintList ; for ( auto & MBB : Fn ) for ( auto & MI : MBB ) { if ( MI . isBundle ( ) ) { MachineBasicBlock :: instr_iterator MII = MI . getIterator ( ) ; for ( ++ MII ; MII != MBB . instr_end ( ) && MII -> isInsideBundle ( ) ; ++ MII ) { if ( MII -> getNumOperands ( ) < <NUM_LIT> ) continue ; unsigned Reg = <NUM_LIT> ; if ( getInstrVecReg ( * MII , Reg ) ) { VecPrintList . push_back ( ( & * MII ) ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; MII -> dump ( ) ) ; } } } else { unsigned Reg = <NUM_LIT> ; if ( getInstrVecReg ( MI , Reg ) ) { VecPrintList . push_back ( & MI ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; MI . dump ( ) ) ; } } } Changed = ! VecPrintList . empty ( ) ; if ( ! Changed ) return Changed ; for ( auto * I : VecPrintList ) { DebugLoc DL = I -> getDebugLoc ( ) ; MachineBasicBlock * MBB = I -> getParent ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; I -> dump ( ) ) ; unsigned Reg = <NUM_LIT> ; if ( ! getInstrVecReg ( * I , Reg ) ) llvm_unreachable ( <STR_LIT> ) ; MachineBasicBlock :: instr_iterator MII = I -> getIterator ( ) ; if ( I -> isInsideBundle ( ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ; I -> dump ( ) ) ; while ( MBB -> instr_end ( ) != MII && MII -> isInsideBundle ( ) ) MII ++ ; } else {


<s> static bool isVecReg ( unsigned Reg ) {

return ( Reg >= <STR_LIT> && Reg <= <STR_LIT> ) || ( Reg >= <STR_LIT> && Reg <= <STR_LIT> ) || ( Reg >= <STR_LIT> && Reg <= <STR_LIT> ) || ( Reg >= <STR_LIT> && Reg <= <STR_LIT> ) ;


<s> <ISA_LIT>VectorPrint ( ) : MachineFunctionPass ( ID ) , QST ( nullptr ) , QII ( nullptr ) , QRI ( nullptr ) {

initialize<ISA_LIT>VectorPrintPass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> std :: string VDescStr = <STR_LIT> + getStringReg ( Reg ) ; const char * cstr = Fn . createExternalSymbolName ( VDescStr . c_str ( ) ) ;

unsigned ExtraInfo = InlineAsm :: Extra_HasSideEffects ;


<s> FunctionPass * llvm :: create<ISA_LIT>VectorPrint ( ) { return new <ISA_LIT>VectorPrint ( ) ;


<s> Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( isVecReg ( Reg ) ) return true ; } if ( MI . mayStore ( ) && MI . getNumOperands ( ) >= <NUM_LIT> && MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) {


<s> const char * getPassName ( ) const override { return <STR_LIT> ;


static const char * S [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ;


<s> return S [ R - <STR_LIT> ] ; } if ( R >= <STR_LIT> && R <= <STR_LIT> ) { static const char * S [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; return S [ R - <STR_LIT> ] ; }


<s> <ISA_LIT>VectorPrint ( ) : MachineFunctionPass ( ID ) , QST ( <NUM_LIT> ) , QII ( <NUM_LIT> ) , QRI ( <NUM_LIT> ) { initialize<ISA_LIT>VectorPrintPass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> static bool isVecReg ( unsigned Reg ) { return ( Reg >= <STR_LIT> && Reg <= <STR_LIT> ) || ( Reg >= <STR_LIT> && Reg <= <STR_LIT> ) || ( Reg >= <STR_LIT> && Reg <= <STR_LIT> ) ;


<s> } } } else { unsigned Reg = <NUM_LIT> ; if ( getInstrVecReg ( MI , Reg ) ) { VecPrintList . push_back ( & MI ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; MI . dump ( ) ) ; } } } Changed = VecPrintList . size ( ) > <NUM_LIT> ; if ( ! Changed ) return Changed ; for ( auto * I : VecPrintList ) { DebugLoc DL = I -> getDebugLoc ( ) ; MachineBasicBlock * MBB = I -> getParent ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; I -> dump ( ) ) ; unsigned Reg = <NUM_LIT> ; if ( ! getInstrVecReg ( * I , Reg ) ) llvm_unreachable ( <STR_LIT> ) ; MachineBasicBlock :: instr_iterator MII = I -> getIterator ( ) ; if ( I -> isInsideBundle ( ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ; I -> dump ( ) ) ; while ( MII -> isInsideBundle ( ) ) ++ MII ; } else { DEBUG ( dbgs ( ) << <STR_LIT> ; I -> dump ( ) ) ; MII ++ ; } if ( Reg >= <STR_LIT> && Reg <= <STR_LIT> ) { DEBUG ( dbgs ( ) << <STR_LIT> << Reg - <STR_LIT> << '\n' ) ; addAsmInstr ( MBB , Reg , MII , DL , QII , Fn ) ; } else if ( Reg >= <STR_LIT> && Reg <= <STR_LIT> ) { DEBUG ( dbgs ( ) << <STR_LIT> << Reg - <STR_LIT> << '\n' ) ; addAsmInstr ( MBB , <STR_LIT> + ( Reg - <STR_LIT> ) * <NUM_LIT> + <NUM_LIT> , MII , DL , QII , Fn ) ; addAsmInstr ( MBB , <STR_LIT> + ( Reg - <STR_LIT> ) * <NUM_LIT> , MII , DL , QII , Fn ) ; } else if ( Reg >= <STR_LIT> && Reg <= <STR_LIT> ) { DEBUG ( dbgs ( ) << <STR_LIT> << Reg - <STR_LIT> << '\n' ) ; addAsmInstr ( MBB , Reg , MII , DL , QII , Fn ) ;


<s> unsigned Reg = <NUM_LIT> ; if ( ! getInstrVecReg ( * I , Reg ) ) llvm_unreachable ( <STR_LIT> ) ; MachineBasicBlock :: instr_iterator MII = I -> getIterator ( ) ; if ( I -> isInsideBundle ( ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ; I -> dump ( ) ) ; while ( MII -> isInsideBundle ( ) ) ++ MII ; } else { DEBUG ( dbgs ( ) << <STR_LIT> ; I -> dump ( ) ) ; MII ++ ; } if ( Reg >= <STR_LIT> && Reg <= <STR_LIT> ) { DEBUG ( dbgs ( ) << <STR_LIT> << Reg - <STR_LIT> << '\n' ) ; addAsmInstr ( MBB , Reg , MII , DL , QII , Fn ) ; } else if ( Reg >= <STR_LIT> && Reg <= <STR_LIT> ) { DEBUG ( dbgs ( ) << <STR_LIT> << Reg - <STR_LIT> << '\n' ) ; addAsmInstr ( MBB , <STR_LIT> + ( Reg - <STR_LIT> ) * <NUM_LIT> + <NUM_LIT> , MII , DL , QII , Fn ) ; addAsmInstr ( MBB , <STR_LIT> + ( Reg - <STR_LIT> ) * <NUM_LIT> , MII , DL , QII , Fn ) ; } else if ( Reg >= <STR_LIT> && Reg <= <STR_LIT> ) { DEBUG ( dbgs ( ) << <STR_LIT> << Reg - <STR_LIT> << '\n' ) ; addAsmInstr ( MBB , Reg , MII , DL , QII , Fn ) ; } else llvm_unreachable ( <STR_LIT> ) ; }


<s> unsigned StoreOpc = VecRC . getID ( ) == <STR_LIT> ? <STR_LIT> : <STR_LIT> ; BuildMI ( DefB , At , DefI -> getDebugLoc ( ) , HII -> get ( StoreOpc ) ) . addFrameIndex ( FI ) . addImm ( <NUM_LIT> ) . addReg ( VecR ) ; unsigned VecSize = HRI . getRegSizeInBits ( VecRC ) / <NUM_LIT> ; for ( MachineInstr * ExtI : P . second ) { assert ( ExtI -> getOpcode ( ) == <STR_LIT> ) ; unsigned SR = ExtI -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ; assert ( ExtI -> getOperand ( <NUM_LIT> ) . getReg ( ) == VecR ) ; MachineBasicBlock & ExtB = * ExtI -> getParent ( ) ; DebugLoc DL = ExtI -> getDebugLoc ( ) ; Register BaseR = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( ExtB , ExtI , DL , HII -> get ( <STR_LIT> ) , BaseR ) . addFrameIndex ( FI ) . addImm ( SR == <NUM_LIT> ? <NUM_LIT> : VecSize / <NUM_LIT> ) ; unsigned ElemR = genElemLoad ( ExtI , BaseR , MRI ) ; Register ExtR = ExtI -> getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> HII = HST -> getInstrInfo ( ) ; const auto & HRI = * HST -> getRegisterInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; std :: map < unsigned , SmallVector < MachineInstr * , <NUM_LIT> >> VExtractMap ; bool Changed = false ; for ( MachineBasicBlock & MBB : MF ) { for ( MachineInstr & MI : MBB ) { unsigned Opc = MI . getOpcode ( ) ; if ( Opc != <STR_LIT> ) continue ; Register VecR = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; VExtractMap [ VecR ] . push_back ( & MI ) ; } } for ( auto & P : VExtractMap ) { unsigned VecR = P . first ; if ( P . second . size ( ) <= VExtractThreshold ) continue ; const auto & VecRC = * MRI . getRegClass ( VecR ) ; int FI = MFI . CreateSpillStackObject ( HRI . getSpillSize ( VecRC ) , HRI . getSpillAlignment ( VecRC ) ) ; MachineInstr * DefI = MRI . getVRegDef ( VecR ) ; MachineBasicBlock :: iterator At = std :: next ( DefI -> getIterator ( ) ) ; MachineBasicBlock & DefB = * DefI -> getParent ( ) ; unsigned StoreOpc = VecRC . getID ( ) == <STR_LIT> ? <STR_LIT> : <STR_LIT> ; BuildMI ( DefB , At , DefI -> getDebugLoc ( ) , HII -> get ( StoreOpc ) ) . addFrameIndex ( FI ) . addImm ( <NUM_LIT> ) . addReg ( VecR ) ; unsigned VecSize = HRI . getRegSizeInBits ( VecRC ) / <NUM_LIT> ; for ( MachineInstr * ExtI : P . second ) { assert ( ExtI -> getOpcode ( ) == <STR_LIT> ) ; unsigned SR = ExtI -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ; assert ( ExtI -> getOperand ( <NUM_LIT> ) . getReg ( ) == VecR ) ; MachineBasicBlock & ExtB = * ExtI -> getParent ( ) ; DebugLoc DL = ExtI -> getDebugLoc ( ) ; Register BaseR = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( ExtB , ExtI , DL , HII -> get ( <STR_LIT> ) , BaseR ) . addFrameIndex ( FI ) . addImm ( SR == <NUM_LIT> ? <NUM_LIT> : VecSize / <NUM_LIT> ) ; unsigned ElemR = genElemLoad ( ExtI , BaseR , MRI ) ; Register ExtR = ExtI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MRI . replaceRegWith ( ExtR , ElemR ) ; ExtB . erase ( ExtI ) ;


<s> MachineBasicBlock & ExtB = * ExtI -> getParent ( ) ; DebugLoc DL = ExtI -> getDebugLoc ( ) ; unsigned ElemR = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned ExtIdxR = ExtI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned ExtIdxS = ExtI -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ; if ( ExtIdxS == <NUM_LIT> ) { MachineInstr * DI = MRI . getVRegDef ( ExtIdxR ) ; if ( DI -> getOpcode ( ) == <STR_LIT> ) { unsigned V = DI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; V &= ( HST -> getVectorLength ( ) - <NUM_LIT> ) & - <NUM_LIT> ; BuildMI ( ExtB , ExtI , DL , HII -> get ( <STR_LIT> ) , ElemR ) . addReg ( BaseR ) . addImm ( V ) ;


<s> for ( auto & P : VExtractMap ) { unsigned VecR = P . first ; if ( P . second . size ( ) <= VExtractThreshold ) continue ; const auto & VecRC = * MRI . getRegClass ( VecR ) ; int FI = MFI . CreateSpillStackObject ( HRI . getSpillSize ( VecRC ) , HRI . getSpillAlignment ( VecRC ) ) ; MachineInstr * DefI = MRI . getVRegDef ( VecR ) ; MachineBasicBlock :: iterator At = std :: next ( DefI -> getIterator ( ) ) ; MachineBasicBlock & DefB = * DefI -> getParent ( ) ; unsigned StoreOpc = VecRC . getID ( ) == <STR_LIT> ? <STR_LIT> : <STR_LIT> ; BuildMI ( DefB , At , DefI -> getDebugLoc ( ) , HII -> get ( StoreOpc ) ) . addFrameIndex ( FI ) . addImm ( <NUM_LIT> ) . addReg ( VecR ) ; unsigned VecSize = HRI . getRegSizeInBits ( VecRC ) / <NUM_LIT> ; for ( MachineInstr * ExtI : P . second ) { assert ( ExtI -> getOpcode ( ) == <STR_LIT> ) ; unsigned SR = ExtI -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ; assert ( ExtI -> getOperand ( <NUM_LIT> ) . getReg ( ) == VecR ) ; MachineBasicBlock & ExtB = * ExtI -> getParent ( ) ; DebugLoc DL = ExtI -> getDebugLoc ( ) ; unsigned BaseR = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( ExtB , ExtI , DL , HII -> get ( <STR_LIT> ) , BaseR ) . addFrameIndex ( FI ) . addImm ( SR == <NUM_LIT> ? <NUM_LIT> : VecSize / <NUM_LIT> ) ; unsigned ElemR = genElemLoad ( ExtI , BaseR , MRI ) ; unsigned ExtR = ExtI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MRI . replaceRegWith ( ExtR , ElemR ) ; ExtB . erase ( ExtI ) ; Changed = true ; } } return Changed ;


<s> Align Alignment = HRI . getSpillAlign ( VecRC ) ; MaxAlign = max ( MaxAlign , Alignment ) ; int FI = MFI . CreateStackObject ( HRI . getSpillSize ( VecRC ) , Alignment , false ) ; MachineInstr * DefI = MRI . getVRegDef ( VecR ) ; MachineBasicBlock :: iterator At = std :: next ( DefI -> getIterator ( ) ) ; MachineBasicBlock & DefB = * DefI -> getParent ( ) ; unsigned StoreOpc = VecRC . getID ( ) == <STR_LIT> ? <STR_LIT> : <STR_LIT> ; Register AddrR = EmitAddr ( DefB , At , DefI -> getDebugLoc ( ) , FI , <NUM_LIT> ) ; BuildMI ( DefB , At , DefI -> getDebugLoc ( ) , HII -> get ( StoreOpc ) ) . addReg ( AddrR ) . addImm ( <NUM_LIT> ) . addReg ( VecR ) ; unsigned VecSize = HRI . getRegSizeInBits ( VecRC ) / <NUM_LIT> ; for ( MachineInstr * ExtI : P . second ) { assert ( ExtI -> getOpcode ( ) == <STR_LIT> ) ; unsigned SR = ExtI -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ; assert ( ExtI -> getOperand ( <NUM_LIT> ) . getReg ( ) == VecR ) ;

MachineBasicBlock & ExtB = * ExtI -> getParent ( ) ;


<s> HST = & MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; HII = HST -> getInstrInfo ( ) ; const auto & HRI = * HST -> getRegisterInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; Register AR = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> getStackAlignBaseVReg ( ) ; std :: map < unsigned , SmallVector < MachineInstr * , <NUM_LIT> >> VExtractMap ; MaybeAlign MaxAlign ; bool Changed = false ; for ( MachineBasicBlock & MBB : MF ) { for ( MachineInstr & MI : MBB ) { unsigned Opc = MI . getOpcode ( ) ; if ( Opc != <STR_LIT> ) continue ; Register VecR = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; VExtractMap [ VecR ] . push_back ( & MI ) ; } } auto EmitAddr = [ & ] ( MachineBasicBlock & BB , MachineBasicBlock :: iterator At , DebugLoc dl , int FI , unsigned Offset ) { Register AddrR = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned FiOpc = AR != <NUM_LIT> ? <STR_LIT> : <STR_LIT> ; auto MIB = BuildMI ( BB , At , dl , HII -> get ( FiOpc ) , AddrR ) ; if ( AR ) MIB . addReg ( AR ) ; MIB . addFrameIndex ( FI ) . addImm ( Offset ) ; return AddrR ; } ; for ( auto & P : VExtractMap ) { unsigned VecR = P . first ; if ( P . second . size ( ) <= VExtractThreshold ) continue ; const auto & VecRC = * MRI . getRegClass ( VecR ) ; Align Alignment = HRI . getSpillAlign ( VecRC ) ; MaxAlign = max ( MaxAlign , Alignment ) ; int FI = MFI . CreateStackObject ( HRI . getSpillSize ( VecRC ) , Alignment , false ) ; MachineInstr * DefI = MRI . getVRegDef ( VecR ) ; MachineBasicBlock :: iterator At = std :: next ( DefI -> getIterator ( ) ) ; MachineBasicBlock & DefB = * DefI -> getParent ( ) ; unsigned StoreOpc = VecRC . getID ( ) == <STR_LIT> ? <STR_LIT> : <STR_LIT> ; Register AddrR = EmitAddr ( DefB , At , DefI -> getDebugLoc ( ) , FI , <NUM_LIT> ) ; BuildMI ( DefB , At , DefI -> getDebugLoc ( ) , HII -> get ( StoreOpc ) ) . addReg ( AddrR ) . addImm ( <NUM_LIT> ) . addReg ( VecR ) ; unsigned VecSize = HRI . getRegSizeInBits ( VecRC ) / <NUM_LIT> ; for ( MachineInstr * ExtI : P . second ) { assert ( ExtI -> getOpcode ( ) == <STR_LIT> ) ; unsigned SR = ExtI -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ; assert ( ExtI -> getOperand ( <NUM_LIT> ) . getReg ( ) == VecR ) ;


<s> FunctionPass * llvm :: create<ISA_LIT>VExtract ( ) {


<s> unsigned <ISA_LIT>VExtract :: genElemLoad ( MachineInstr * ExtI , unsigned BaseR , MachineRegisterInfo & MRI ) { MachineBasicBlock & ExtB = * ExtI -> getParent ( ) ; DebugLoc DL = ExtI -> getDebugLoc ( ) ; Register ElemR = MRI . createVirtualRegister ( & <STR_LIT> ) ; Register ExtIdxR = ExtI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned ExtIdxS = ExtI -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ; if ( ExtIdxS == <NUM_LIT> ) { MachineInstr * DI = MRI . getVRegDef ( ExtIdxR ) ; if ( DI -> getOpcode ( ) == <STR_LIT> ) { unsigned V = DI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; V &= ( HST -> getVectorLength ( ) - <NUM_LIT> ) & - <NUM_LIT> ; BuildMI ( ExtB , ExtI , DL , HII -> get ( <STR_LIT> ) , ElemR ) . addReg ( BaseR ) . addImm ( V ) ; return ElemR ; }


<s> StringRef getPassName ( ) const override {


<s> <ISA_LIT>VExtract ( ) : MachineFunctionPass ( ID ) {


<s> } } auto EmitAddr = [ & ] ( MachineBasicBlock & BB , MachineBasicBlock :: iterator At , DebugLoc dl , int FI , unsigned Offset ) { Register AddrR = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned FiOpc = AR != <NUM_LIT> ? <STR_LIT> : <STR_LIT> ; auto MIB = BuildMI ( BB , At , dl , HII -> get ( FiOpc ) , AddrR ) ; if ( AR ) MIB . addReg ( AR ) ; MIB . addFrameIndex ( FI ) . addImm ( Offset ) ; return AddrR ; } ; for ( auto & P : VExtractMap ) { unsigned VecR = P . first ; if ( P . second . size ( ) <= VExtractThreshold ) continue ; const auto & VecRC = * MRI . getRegClass ( VecR ) ; unsigned Align = HRI . getSpillAlignment ( VecRC ) ; MaxAlign = std :: max ( MaxAlign , Align ) ; int FI = MFI . CreateStackObject ( HRI . getSpillSize ( VecRC ) , Align , false ) ; MachineInstr * DefI = MRI . getVRegDef ( VecR ) ; MachineBasicBlock :: iterator At = std :: next ( DefI -> getIterator ( ) ) ; MachineBasicBlock & DefB = * DefI -> getParent ( ) ; unsigned StoreOpc = VecRC . getID ( ) == <STR_LIT> ? <STR_LIT> : <STR_LIT> ; Register AddrR = EmitAddr ( DefB , At , DefI -> getDebugLoc ( ) , FI , <NUM_LIT> ) ; BuildMI ( DefB , At , DefI -> getDebugLoc ( ) , HII -> get ( StoreOpc ) ) . addReg ( AddrR ) . addImm ( <NUM_LIT> ) . addReg ( VecR ) ; unsigned VecSize = HRI . getRegSizeInBits ( VecRC ) / <NUM_LIT> ; for ( MachineInstr * ExtI : P . second ) { assert ( ExtI -> getOpcode ( ) == <STR_LIT> ) ; unsigned SR = ExtI -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ; assert ( ExtI -> getOperand ( <NUM_LIT> ) . getReg ( ) == VecR ) ; MachineBasicBlock & ExtB = * ExtI -> getParent ( ) ; DebugLoc DL = ExtI -> getDebugLoc ( ) ; Register BaseR = EmitAddr ( ExtB , ExtI , ExtI -> getDebugLoc ( ) , FI , SR == <NUM_LIT> ? <NUM_LIT> : VecSize / <NUM_LIT> ) ; unsigned ElemR = genElemLoad ( ExtI , BaseR , MRI ) ; Register ExtR = ExtI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MRI . replaceRegWith ( ExtR , ElemR ) ; ExtB . erase ( ExtI ) ; Changed = true ; } } if ( AR ) { MachineInstr * AlignaI = MRI . getVRegDef ( AR ) ; assert ( AlignaI -> getOpcode ( ) == <STR_LIT> ) ; MachineOperand & Op = AlignaI -> getOperand ( <NUM_LIT> ) ; if ( MaxAlign > Op . getImm ( ) ) Op . setImm ( MaxAlign ) ;


<s> if ( Dep . getSUnit ( ) == SU && Dep . getKind ( ) == SDep :: Data && <STR_LIT> . contains ( Dep . getReg ( ) ) ) { if ( restrictingDepExistInPacket ( * I , Dep . getReg ( ) ) ) return false ; } } } } unsigned PReg1 = getPredicatedRegister ( MI1 , HII ) ; unsigned PReg2 = getPredicatedRegister ( MI2 , HII ) ; return PReg1 == PReg2 && <STR_LIT> . contains ( PReg1 ) && <STR_LIT> . contains ( PReg2 ) && getPredicateSense ( MI1 , HII ) != getPredicateSense ( MI2 , HII ) && HII -> isDotNewInst ( MI1 ) == HII -> isDotNewInst ( MI2 ) ;


<s> if ( getPredicateSense ( MI1 , HII ) == PK_Unknown || getPredicateSense ( MI2 , HII ) == PK_Unknown ) return false ; SUnit * SU = MIToSUnit [ & MI1 ] ; for ( auto I : CurrentPacketMIs ) { SUnit * PacketSU = MIToSUnit . find ( I ) -> second ; if ( PacketSU -> isSucc ( SU ) ) { for ( unsigned i = <NUM_LIT> ; i < PacketSU -> Succs . size ( ) ; ++ i ) { auto Dep = PacketSU -> Succs [ i ] ; if ( Dep . getSUnit ( ) == SU && Dep . getKind ( ) == SDep :: Data && <STR_LIT> . contains ( Dep . getReg ( ) ) ) { if ( restrictingDepExistInPacket ( * I , Dep . getReg ( ) ) ) return false ; }


<s> bool <ISA_LIT>PacketizerList :: cannotCoexist ( const MachineInstr & MI , const MachineInstr & MJ ) {

return cannotCoexistAsymm ( MI , MJ , * HII ) || cannotCoexistAsymm ( MJ , MI , * HII ) ;


<s> bool <ISA_LIT>PacketizerList :: cannotCoexist ( const MachineInstr & MI , const MachineInstr & MJ ) { return cannotCoexistAsymm ( MI , MJ , * HII ) || cannotCoexistAsymm ( MJ , MI , * HII ) ;


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { unsigned TJ = HII . getType ( MJ ) ; if ( TJ != <STR_LIT> && TJ != <STR_LIT> && TJ != <STR_LIT> ) return true ;


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { unsigned TJ = HII . getType ( MJ ) ; if ( TJ != <STR_LIT> && TJ != <STR_LIT> && TJ != <STR_LIT> ) return true ; break ;


<s> if ( PacketSU -> getInstr ( ) -> isInlineAsm ( ) ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; dbgs ( ) << <STR_LIT> ; ) ; MachineInstr & MJ = * MII ; LLVM_DEBUG ( { dbgs ( ) << <STR_LIT> ; MJ . dump ( ) ;


<s> bool <ISA_LIT>PacketizerList :: canPromoteToNewValue ( const MachineInstr & MI , const SUnit * PacketSU , unsigned DepReg , MachineBasicBlock :: iterator & MII ) { if ( ! HII -> mayBeNewStore ( MI ) ) return false ; MachineInstr & PacketMI = * PacketSU -> getInstr ( ) ; if ( canPromoteToNewValueStore ( MI , PacketMI , DepReg ) ) return true ; return false ;


<s> if ( ! HII -> mayBeNewStore ( MI ) ) return false ; MachineInstr & PacketMI = * PacketSU -> getInstr ( ) ; if ( canPromoteToNewValueStore ( MI , PacketMI , DepReg ) ) return true ; return false ;


<s> if ( ! MO . isReg ( ) ) continue ; predRegNumDst = MO . getReg ( ) ; predRegClass = HRI -> getMinimalPhysRegClass ( predRegNumDst ) ; if ( predRegClass == & <STR_LIT> ) break ; } assert ( ( predRegClass == & <STR_LIT> ) && <STR_LIT> ) ; if ( predRegNumDst != predRegNumSrc || HII -> isDotNewInst ( PacketMI ) != HII -> isDotNewInst ( MI ) || getPredicateSense ( MI , HII ) != getPredicateSense ( PacketMI , HII ) ) return false ; } unsigned StartCheck = <NUM_LIT> ; for ( auto I : CurrentPacketMIs ) { SUnit * TempSU = MIToSUnit . find ( I ) -> second ; MachineInstr & TempMI = * TempSU -> getInstr ( ) ; if ( & TempMI != & PacketMI && ! StartCheck ) continue ; StartCheck = <NUM_LIT> ; if ( & TempMI == & PacketMI ) continue ; for ( auto & MO : MI . operands ( ) ) if ( MO . isReg ( ) && TempSU -> getInstr ( ) -> modifiesRegister ( MO . getReg ( ) , HRI ) ) return false ; } if ( ! HII -> isPostIncrement ( MI ) ) { for ( unsigned opNum = <NUM_LIT> ; opNum < MI . getNumOperands ( ) - <NUM_LIT> ; opNum ++ ) { const MachineOperand & MO = MI . getOperand ( opNum ) ; if ( MO . isReg ( ) && MO . getReg ( ) == DepReg ) return false ; } } for ( auto & MO : PacketMI . operands ( ) ) { if ( MO . isRegMask ( ) && MO . clobbersPhysReg ( DepReg ) ) return false ; if ( ! MO . isReg ( ) || ! MO . isDef ( ) || ! MO . isImplicit ( ) ) continue ; unsigned R = MO . getReg ( ) ; if ( R == DepReg || HRI -> isSuperRegister ( DepReg , R ) ) return false ; } for ( auto & MO : MI . operands ( ) ) { if ( MO . isReg ( ) && MO . isUse ( ) && MO . isImplicit ( ) && MO . getReg ( ) == DepReg ) return false ; }


<s> bool <ISA_LIT>PacketizerList :: canReserveResourcesForConstExt ( ) { return tryAllocateResourcesForConstExt ( false ) ;


<s> bool <ISA_LIT>PacketizerList :: canReserveResourcesForConstExt ( ) {

return tryAllocateResourcesForConstExt ( false ) ;


<s> } if ( MI ) { for ( auto & MO : BI -> operands ( ) ) if ( MO . isReg ( ) && MO . getReg ( ) == MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) return ; } } if ( ! MI ) return ; MI -> setDesc ( HII -> get ( HII -> getNonDotCurOp ( * MI ) ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MI -> dump ( ) ; ) ;


<s> return new <ISA_LIT>Packetizer ( Minimal ) ;


<s> MI . setDesc ( HII -> get ( NewOpcode ) ) ;


<s> bool <ISA_LIT>PacketizerList :: demoteToDotOld ( MachineInstr & MI ) { int NewOpcode = HII -> getDotOldOp ( MI ) ;

MI . setDesc ( HII -> get ( NewOpcode ) ) ;


<s> const MachineFunction & MF = * MI . getParent ( ) -> getParent ( ) ; for ( auto * CSR = TRI -> getCalleeSavedRegs ( & MF ) ; CSR && * CSR ; ++ CSR ) if ( MI . modifiesRegister ( * CSR , TRI ) ) return true ; return false ;


<s> for ( MachineInstr * MI : CurrentPacketMIs ) { MachineBasicBlock :: instr_iterator NextMI = std :: next ( MI -> getIterator ( ) ) ; for ( auto & I : make_range ( HII -> expandVGatherPseudo ( * MI ) , NextMI ) ) OldPacketMIs . push_back ( & I ) ; } CurrentPacketMIs . clear ( ) ; if ( OldPacketMIs . size ( ) > <NUM_LIT> ) { MachineBasicBlock :: instr_iterator FirstMI ( OldPacketMIs . front ( ) ) ; MachineBasicBlock :: instr_iterator LastMI ( EndMI . getInstrIterator ( ) ) ; finalizeBundle ( * MBB , FirstMI , LastMI ) ; auto BundleMII = std :: prev ( FirstMI ) ; if ( memShufDisabled ) HII -> setBundleNoShuf ( BundleMII ) ; setmemShufDisabled ( false ) ; }


<s> if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) continue ; if ( HII -> isMemOp ( * MJ ) ) continue ; if ( MJ -> mayLoad ( ) ) FoundLoad = true ; if ( MJ -> mayStore ( ) && ! HII -> isNewValueStore ( * MJ ) ) FoundStore = true ; }


<s> static const MachineOperand & getAbsSetOperand ( const MachineInstr & MI ) {


<s> AU . addRequired < MachineDominatorTree > ( ) ; AU . addRequired < MachineLoopInfo > ( ) ;

AU . addPreserved < MachineDominatorTree > ( ) ;


<s> StringRef getPassName ( ) const override {


<s> DenseSet < unsigned > DefRegsSet ; for ( auto & MO : MI . operands ( ) ) if ( MO . isReg ( ) && MO . isDef ( ) ) DefRegsSet . insert ( MO . getReg ( ) ) ; for ( auto & MO : MI . operands ( ) ) if ( MO . isReg ( ) && MO . isUse ( ) && DefRegsSet . count ( MO . getReg ( ) ) ) return MO ; if ( MI . mayLoad ( ) ) { const MachineOperand & Op1 = MI . getOperand ( <NUM_LIT> ) ; assert ( Op1 . isReg ( ) && <STR_LIT> ) ; return Op1 ; } if ( MI . getDesc ( ) . mayStore ( ) ) { const MachineOperand & Op0 = MI . getOperand ( <NUM_LIT> ) ; assert ( Op0 . isReg ( ) && <STR_LIT> ) ; return Op0 ; } llvm_unreachable ( <STR_LIT> ) ;


<s> for ( auto & MO : MI . operands ( ) ) if ( MO . isReg ( ) && MO . isUse ( ) && DefRegsSet . count ( MO . getReg ( ) ) ) return MO ; if ( MI . mayLoad ( ) ) { const MachineOperand & Op1 = MI . getOperand ( <NUM_LIT> ) ; assert ( Op1 . isReg ( ) && <STR_LIT> ) ; return Op1 ;


<s> for ( auto & Op : MI . operands ( ) ) { if ( Op . isReg ( ) && Op . getReg ( ) && Op . isUse ( ) && <STR_LIT> . contains ( Op . getReg ( ) ) ) return Op . getReg ( ) ; } llvm_unreachable ( <STR_LIT> ) ; return <NUM_LIT> ;


<s> if ( Op . isReg ( ) && Op . getReg ( ) && Op . isUse ( ) && <STR_LIT> . contains ( Op . getReg ( ) ) ) return Op . getReg ( ) ; } llvm_unreachable ( <STR_LIT> ) ; return <NUM_LIT> ;


<s> static PredicateKind getPredicateSense ( const MachineInstr & MI , const <ISA_LIT>InstrInfo * HII ) { if ( ! HII -> isPredicated ( MI ) ) return PK_Unknown ; if ( HII -> isPredicatedTrue ( MI ) ) return PK_True ; return PK_False ;


<s> return MI . getOperand ( MI . getNumOperands ( ) - <NUM_LIT> ) ;


<s> if ( isControlFlow ( I ) && isControlFlow ( J ) ) return true ; auto isBadForLoopN = [ this ] ( const MachineInstr & MI ) -> bool { if ( MI . isCall ( ) || HII -> isDeallocRet ( MI ) || HII -> isNewValueJump ( MI ) ) return true ; if ( HII -> isPredicated ( MI ) && HII -> isPredicatedNew ( MI ) && HII -> isJumpR ( MI ) ) return true ; return false ; } ; if ( HII -> isLoopN ( I ) && isBadForLoopN ( J ) ) return true ; if ( HII -> isLoopN ( J ) && isBadForLoopN ( I ) ) return true ;

return HII -> isDeallocRet ( I ) && ( J . isBranch ( ) || J . isCall ( ) || J . isBarrier ( ) ) ;


<s> return false ; } ; if ( HII -> isLoopN ( I ) && isBadForLoopN ( J ) ) return true ; if ( HII -> isLoopN ( J ) && isBadForLoopN ( I ) ) return true ;


<s> BitVector DeadDefs ( <STR_LIT> ) ; for ( auto & MO : I . operands ( ) ) { if ( ! MO . isReg ( ) || ! MO . isDef ( ) || ! MO . isDead ( ) ) continue ; DeadDefs [ MO . getReg ( ) ] = true ; } for ( auto & MO : J . operands ( ) ) { if ( ! MO . isReg ( ) || ! MO . isDef ( ) || ! MO . isDead ( ) ) continue ; unsigned R = MO . getReg ( ) ; if ( R != <STR_LIT> && DeadDefs [ R ] ) return true ; }


<s> DeadDefs [ MO . getReg ( ) ] = true ; } for ( auto & MO : J . operands ( ) ) { if ( ! MO . isReg ( ) || ! MO . isDef ( ) || ! MO . isDead ( ) ) continue ; unsigned R = MO . getReg ( ) ; if ( R != <STR_LIT> && DeadDefs [ R ] ) return true ; }


<s> bool <ISA_LIT>PacketizerList :: hasDualStoreDependence ( const MachineInstr & I , const MachineInstr & J ) { bool SysI = isSystemInstr ( I ) , SysJ = isSystemInstr ( J ) ; bool StoreI = I . mayStore ( ) , StoreJ = J . mayStore ( ) ; if ( ( SysI && StoreJ ) || ( SysJ && StoreI ) ) return true ; if ( StoreI && StoreJ ) { if ( HII -> isNewValueInst ( J ) || HII -> isMemOp ( J ) || HII -> isMemOp ( I ) ) return true ; } else { bool MopStI = HII -> isMemOp ( I ) || StoreI ; bool MopStJ = HII -> isMemOp ( J ) || StoreJ ; if ( MopStI && MopStJ ) return true ; }

return ( StoreJ && HII -> isDeallocRet ( I ) ) || ( StoreI && HII -> isDeallocRet ( J ) ) ;


<s> if ( HII -> isNewValueInst ( J ) || HII -> isMemOp ( J ) || HII -> isMemOp ( I ) ) return true ; } else { bool MopStI = HII -> isMemOp ( I ) || StoreI ; bool MopStJ = HII -> isMemOp ( J ) || StoreJ ; if ( MopStI && MopStJ ) return true ; } return ( StoreJ && HII -> isDeallocRet ( I ) ) || ( StoreI && HII -> isDeallocRet ( J ) ) ;


<s> if ( OpI . isReg ( ) ) { if ( OpJ . clobbersPhysReg ( OpI . getReg ( ) ) ) return true ; } else if ( OpI . isRegMask ( ) ) { return true ; } } } return false ;


<s> unsigned R = MO . getReg ( ) ; if ( SecondI . readsRegister ( R , TRI ) ) return true ; } return false ;


<s> <ISA_LIT>Packetizer ( bool Min = false ) : MachineFunctionPass ( ID ) , Minimal ( Min ) {


<s> HII = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) ; HRI = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getRegisterInfo ( ) ; addMutation ( llvm :: make_unique < <ISA_LIT>Subtarget :: UsrOverflowMutation > ( ) ) ; addMutation ( llvm :: make_unique < <ISA_LIT>Subtarget :: HVXMemLatencyMutation > ( ) ) ; addMutation ( llvm :: make_unique < <ISA_LIT>Subtarget :: BankConflictMutation > ( ) ) ;


<s> HII = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) ; HRI = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getRegisterInfo ( ) ;

addMutation ( llvm :: make_unique < <ISA_LIT>Subtarget :: UsrOverflowMutation > ( ) ) ;


<s> if ( MI . isCFIInstruction ( ) ) return false ; if ( MI . isInlineAsm ( ) ) return false ; if ( MI . isImplicitDef ( ) ) return false ; const MCInstrDesc & TID = MI . getDesc ( ) ; auto * IS = ResourceTracker -> getInstrItins ( ) -> beginStage ( TID . getSchedClass ( ) ) ;

unsigned FuncUnits = IS -> getUnits ( ) ;


<s> Dependence = false ; PromotedToDotNew = false ; GlueToNewValueJump = false ; GlueAllocframeStore = false ; FoundSequentialDependence = false ;


<s> GlueToNewValueJump = false ; GlueAllocframeStore = false ; FoundSequentialDependence = false ;


<s> if ( HII -> isDeallocRet ( MI ) ) if ( DepReg == HRI -> getFrameRegister ( ) || DepReg == HRI -> getStackRegister ( ) ) return true ;


<s> if ( DepReg == HRI -> getRARegister ( ) ) return true ; if ( HII -> isDeallocRet ( MI ) ) if ( DepReg == HRI -> getFrameRegister ( ) || DepReg == HRI -> getStackRegister ( ) ) return true ; if ( DepType == SDep :: Data ) { for ( const MachineOperand MO : MI . operands ( ) ) if ( MO . isReg ( ) && MO . getReg ( ) == DepReg && ! MO . isImplicit ( ) ) return true ; }


<s> static bool isControlFlow ( const MachineInstr & MI ) {

return MI . getDesc ( ) . isTerminator ( ) || MI . getDesc ( ) . isCall ( ) ;


<s> static bool isDirectJump ( const MachineInstr & MI ) {

return MI . getOpcode ( ) == <STR_LIT> ;


<s> for ( auto & MO : I . operands ( ) ) { if ( CheckDef && MO . isRegMask ( ) && MO . clobbersPhysReg ( DepReg ) ) return true ; if ( ! MO . isReg ( ) || MO . getReg ( ) != DepReg || ! MO . isImplicit ( ) ) continue ; if ( CheckDef == MO . isDef ( ) ) return true ; }


<s> if ( ! MO . isReg ( ) || MO . getReg ( ) != DepReg || ! MO . isImplicit ( ) ) continue ;

if ( CheckDef == MO . isDef ( ) ) return true ;


<s> } } GlueToNewValueJump = true ; if ( Dependence ) return false ; } if ( ! SUJ -> isSucc ( SUI ) ) return true ; for ( unsigned i = <NUM_LIT> ; i < SUJ -> Succs . size ( ) ; ++ i ) { if ( FoundSequentialDependence ) break ; if ( SUJ -> Succs [ i ] . getSUnit ( ) != SUI ) continue ; SDep :: Kind DepType = SUJ -> Succs [ i ] . getKind ( ) ; unsigned DepReg = <NUM_LIT> ; const TargetRegisterClass * RC = nullptr ; if ( DepType == SDep :: Data ) { DepReg = SUJ -> Succs [ i ] . getReg ( ) ; RC = HRI -> getMinimalPhysRegClass ( DepReg ) ; } if ( I . isCall ( ) || HII -> isJumpR ( I ) || I . isReturn ( ) || HII -> isTailCall ( I ) ) { if ( ! isRegDependence ( DepType ) ) continue ; if ( ! isCallDependent ( I , DepType , SUJ -> Succs [ i ] . getReg ( ) ) ) continue ; } if ( DepType == SDep :: Data ) { if ( canPromoteToDotCur ( J , SUJ , DepReg , II , RC ) ) if ( promoteToDotCur ( J , DepType , II , RC ) ) continue ; } if ( DepType == SDep :: Data && HII -> isDotCurInst ( J ) ) { if ( HII -> isHVXVec ( I ) ) continue ; } if ( DepType == SDep :: Data ) { if ( canPromoteToDotNew ( I , SUJ , DepReg , II , RC ) ) { if ( promoteToDotNew ( I , DepType , II , RC ) ) { PromotedToDotNew = true ; if ( cannotCoexist ( I , J ) ) FoundSequentialDependence = true ; continue ; } } if ( HII -> isNewValueJump ( I ) ) continue ; } if ( HII -> isPredicated ( I ) && HII -> isPredicated ( J ) && arePredicatesComplements ( I , J ) ) { auto Itr = find ( IgnoreDepMIs , & J ) ; if ( Itr != IgnoreDepMIs . end ( ) ) { Dependence = true ; return false ; } IgnoreDepMIs . push_back ( & I ) ; continue ; } if ( isDirectJump ( I ) && ! J . isBranch ( ) && ! J . isCall ( ) && DepType == SDep :: Order ) continue ; if ( I . isConditionalBranch ( ) && DepType != SDep :: Data && DepType != SDep :: Output ) continue ; if ( DepType == SDep :: Output ) { FoundSequentialDependence = true ; break ; } if ( DepType == SDep :: Order ) { if ( ! PacketizeVolatiles ) { bool OrdRefs = I . hasOrderedMemoryRef ( ) || J . hasOrderedMemoryRef ( ) ; if ( OrdRefs ) { FoundSequentialDependence = true ; break ;


<s> continue ; } if ( isDirectJump ( I ) && ! J . isBranch ( ) && ! J . isCall ( ) && DepType == SDep :: Order ) continue ; if ( I . isConditionalBranch ( ) && DepType != SDep :: Data && DepType != SDep :: Output ) continue ; if ( DepType == SDep :: Output ) { FoundSequentialDependence = true ; break ; } if ( DepType == SDep :: Order ) { if ( ! PacketizeVolatiles ) { bool OrdRefs = I . hasOrderedMemoryRef ( ) || J . hasOrderedMemoryRef ( ) ; if ( OrdRefs ) { FoundSequentialDependence = true ; break ; } } bool LoadJ = J . mayLoad ( ) , StoreJ = J . mayStore ( ) ; bool LoadI = I . mayLoad ( ) , StoreI = I . mayStore ( ) ; bool NVStoreJ = HII -> isNewValueStore ( J ) ; bool NVStoreI = HII -> isNewValueStore ( I ) ; bool IsVecJ = HII -> isHVXVec ( J ) ; bool IsVecI = HII -> isHVXVec ( I ) ; if ( Slot1Store && MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . hasV65Ops ( ) && ( ( LoadJ && StoreI && ! NVStoreI ) || ( StoreJ && LoadI && ! NVStoreJ ) ) && ( J . getOpcode ( ) != <STR_LIT> && I . getOpcode ( ) != <STR_LIT> ) && ( J . getOpcode ( ) != <STR_LIT> && I . getOpcode ( ) != <STR_LIT> ) && ( ! HII -> isMemOp ( J ) && ! HII -> isMemOp ( I ) ) && ( ! IsVecJ && ! IsVecI ) ) setmemShufDisabled ( true ) ; else if ( StoreJ && LoadI && alias ( J , I ) ) { FoundSequentialDependence = true ; break ; } if ( ! StoreJ ) if ( ! LoadJ || ( ! LoadI && ! StoreI ) ) { FoundSequentialDependence = true ; break ; } continue ; } if ( DepType == SDep :: Data && J . getOpcode ( ) == <STR_LIT> ) { unsigned Opc = I . getOpcode ( ) ; switch ( Opc ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( I . getOperand ( <NUM_LIT> ) . getReg ( ) == HRI -> getStackRegister ( ) ) { GlueAllocframeStore = useCallersSP ( I ) ; if ( GlueAllocframeStore ) continue ; } break ; default : break ; } } if ( ( DepType == SDep :: Anti || DepType == SDep :: Output ) && J . isCall ( ) ) {

for ( const MachineOperand & Op : I . operands ( ) ) {


<s> if ( Coexist && ! Dependence ) return true ; if ( PromotedToDotNew ) demoteToDotOld ( I ) ; cleanUpDotCur ( ) ; if ( GlueAllocframeStore ) { useCalleesSP ( I ) ; GlueAllocframeStore = false ; } if ( ChangedOffset != INT64_MAX ) undoChangedOffset ( I ) ; if ( GlueToNewValueJump ) {


<s> return HII -> isPredicated ( MI ) && HII -> getDotNewPredOp ( MI , nullptr ) > <NUM_LIT> ; } return HII -> mayBeNewStore ( MI ) ;


<s> return DepType == SDep :: Data || DepType == SDep :: Anti || DepType == SDep :: Output ;


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : return true ; } return false ;


<s> if ( MI . isEHLabel ( ) || MI . isCFIInstruction ( ) ) return true ; if ( MI . isInlineAsm ( ) && ! ScheduleInlineAsm ) return true ;

if ( isSchedBarrier ( MI ) ) return true ;


<s> static bool isSystemInstr ( const MachineInstr & MI ) { unsigned Opc = MI . getOpcode ( ) ; switch ( Opc ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> MI . clearFlag ( MachineInstr :: BundledSucc ) ; MI . clearFlag ( MachineInstr :: BundledPred ) ; } else { MI . unbundleFromPred ( ) ; } B . splice ( InsertPt , & B , MI . getIterator ( ) ) ; MachineBasicBlock :: const_instr_iterator I = BundleIt . getInstrIterator ( ) ; MachineBasicBlock :: const_instr_iterator E = B . instr_end ( ) ; unsigned Size = <NUM_LIT> ; for ( ++ I ; I != E && I -> isBundledWithPred ( ) ; ++ I ) ++ Size ; if ( Size > <NUM_LIT> ) return BundleIt ;

MachineBasicBlock :: iterator NextIt = std :: next ( BundleIt ) ;


<s> MachineBasicBlock & B = * MI . getParent ( ) ; assert ( MI . isBundledWithPred ( ) ) ; if ( MI . isBundledWithSucc ( ) ) { MI . clearFlag ( MachineInstr :: BundledSucc ) ; MI . clearFlag ( MachineInstr :: BundledPred ) ; } else { MI . unbundleFromPred ( ) ; } B . splice ( InsertPt , & B , MI . getIterator ( ) ) ; MachineBasicBlock :: const_instr_iterator I = BundleIt . getInstrIterator ( ) ; MachineBasicBlock :: const_instr_iterator E = B . instr_end ( ) ; unsigned Size = <NUM_LIT> ; for ( ++ I ; I != E && I -> isBundledWithPred ( ) ; ++ I ) ++ Size ; if ( Size > <NUM_LIT> ) return BundleIt ; MachineBasicBlock :: iterator NextIt = std :: next ( BundleIt ) ;

MachineInstr & SingleI = * BundleIt -> getNextNode ( ) ;


<s> SUnit * SUJ = MIToSUnit [ J ] ; for ( auto & Pred : SUI -> Preds ) if ( Pred . getSUnit ( ) == SUJ ) if ( ( Pred . getLatency ( ) == <NUM_LIT> && Pred . isAssignedRegDep ( ) ) || HII -> isNewValueJump ( I ) || HII -> isToBeScheduledASAP ( * J , I ) ) return false ;


<s> for ( auto J : CurrentPacketMIs ) { SUnit * SUJ = MIToSUnit [ J ] ; for ( auto & Pred : SUI -> Preds ) if ( Pred . getSUnit ( ) == SUJ ) if ( ( Pred . getLatency ( ) == <NUM_LIT> && Pred . isAssignedRegDep ( ) ) || HII -> isNewValueJump ( I ) || HII -> isToBeScheduledASAP ( * J , I ) ) return false ; } for ( auto J : OldPacketMIs ) { SUnit * SUJ = MIToSUnit [ J ] ; for ( auto & Pred : SUI -> Preds ) if ( Pred . getSUnit ( ) == SUJ && Pred . getLatency ( ) > <NUM_LIT> ) return true ;


<s> bool <ISA_LIT>PacketizerList :: promoteToDotCur ( MachineInstr & MI , SDep :: Kind DepType , MachineBasicBlock :: iterator & MII , const TargetRegisterClass * RC ) {


<s> bool <ISA_LIT>PacketizerList :: promoteToDotCur ( MachineInstr & MI , SDep :: Kind DepType , MachineBasicBlock :: iterator & MII , const TargetRegisterClass * RC ) { assert ( DepType == SDep :: Data ) ; int CurOpcode = HII -> getDotCurOp ( MI ) ;

MI . setDesc ( HII -> get ( CurOpcode ) ) ;


<s> assert ( DepType == SDep :: Data ) ; int NewOpcode ; if ( RC == & <STR_LIT> ) NewOpcode = HII -> getDotNewPredOp ( MI , MBPI ) ; else NewOpcode = HII -> getDotNewOp ( MI ) ;


<s> if ( ! tryAllocateResourcesForConstExt ( true ) ) llvm_unreachable ( <STR_LIT> ) ;


<s> bool <ISA_LIT>PacketizerList :: restrictingDepExistInPacket ( MachineInstr & MI , unsigned DepReg ) { SUnit * PacketSUDep = MIToSUnit . find ( & MI ) -> second ; for ( auto I : CurrentPacketMIs ) { if ( ! HII -> isPredicated ( * I ) ) continue ; SUnit * PacketSU = MIToSUnit . find ( I ) -> second ; if ( PacketSU -> isSucc ( PacketSUDep ) ) { for ( unsigned i = <NUM_LIT> ; i < PacketSU -> Succs . size ( ) ; ++ i ) { auto & Dep = PacketSU -> Succs [ i ] ; if ( Dep . getSUnit ( ) == PacketSUDep && Dep . getKind ( ) == SDep :: Anti && Dep . getReg ( ) == DepReg ) return true ; } } } return false ;


<s> while ( Begin != End ) { MachineBasicBlock :: iterator RB = Begin ; while ( RB != End && HII -> isSchedulingBoundary ( * RB , & MB , MF ) ) ++ RB ; MachineBasicBlock :: iterator RE = RB ; while ( RE != End && ! HII -> isSchedulingBoundary ( * RE , & MB , MF ) ) ++ RE ; if ( RE != End ) ++ RE ; if ( RB != End ) Packetizer . PacketizeMIs ( & MB , RB , RE ) ; Begin = RE ; } } Packetizer . unpacketizeSoloInstrs ( MF ) ;


<s> bool <ISA_LIT>PacketizerList :: shouldAddToPacket ( const MachineInstr & MI ) { if ( Minimal ) return false ; return ! producesStall ( MI ) ;


<s> auto * ExtMI = MF . CreateMachineInstr ( HII -> get ( <STR_LIT> ) , DebugLoc ( ) ) ; bool Avail = ResourceTracker -> canReserveResources ( * ExtMI ) ;

if ( Reserve && Avail ) ResourceTracker -> reserveResources ( * ExtMI ) ;


<s> void <ISA_LIT>PacketizerList :: undoChangedOffset ( MachineInstr & MI ) {


<s> void <ISA_LIT>PacketizerList :: undoChangedOffset ( MachineInstr & MI ) { unsigned BP , OP ; if ( ! HII -> getBaseAndOffsetPosition ( MI , BP , OP ) ) llvm_unreachable ( <STR_LIT> ) ; MI . getOperand ( OP ) . setImm ( ChangedOffset ) ;


<s> assert ( SUI -> getInstr ( ) && SUJ -> getInstr ( ) ) ; MachineInstr & MI = * SUI -> getInstr ( ) ; MachineInstr & MJ = * SUJ -> getInstr ( ) ; unsigned BPI , OPI ; if ( ! HII -> getBaseAndOffsetPosition ( MI , BPI , OPI ) ) return false ; unsigned BPJ , OPJ ; if ( ! HII -> getBaseAndOffsetPosition ( MJ , BPJ , OPJ ) ) return false ; unsigned Reg = MI . getOperand ( BPI ) . getReg ( ) ; if ( Reg != MJ . getOperand ( BPJ ) . getReg ( ) ) return false ; for ( const auto & PI : SUI -> Preds ) if ( PI . getKind ( ) != SDep :: Anti && ( PI . getKind ( ) != SDep :: Data || PI . getReg ( ) != Reg ) ) return false ; int Incr ; if ( ! HII -> getIncrementValue ( MJ , Incr ) ) return false ; int64_t Offset = MI . getOperand ( OPI ) . getImm ( ) ; if ( ! HII -> isValidOffset ( MI . getOpcode ( ) , Offset + Incr , HRI ) ) return false ; MI . getOperand ( OPI ) . setImm ( Offset + Incr ) ;


<s> } unsigned FrameSize = MF . getFrameInfo ( ) . getStackSize ( ) ; MachineOperand & Off = MI . getOperand ( <NUM_LIT> ) ;

Off . setImm ( Off . getImm ( ) + FrameSize + HEXAGON_LRFP_SIZE ) ;


<s> switch ( Opc ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } unsigned FrameSize = MF . getFrameInfo ( ) . getStackSize ( ) ; MachineOperand & Off = MI . getOperand ( <NUM_LIT> ) ; int64_t NewOff = Off . getImm ( ) - ( FrameSize + HEXAGON_LRFP_SIZE ) ; if ( HII -> isValidOffset ( Opc , NewOff , HRI ) ) { Off . setImm ( NewOff ) ; return true ; } return false ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } unsigned FrameSize = MF . getFrameInfo ( ) . getStackSize ( ) ; MachineOperand & Off = MI . getOperand ( <NUM_LIT> ) ; int64_t NewOff = Off . getImm ( ) - ( FrameSize + HEXAGON_LRFP_SIZE ) ; if ( HII -> isValidOffset ( Opc , NewOff , HRI ) ) { Off . setImm ( NewOff ) ; return true ;


<s> MachineInstr & NvjMI = * ++ MII ; ResourceTracker -> reserveResources ( MI ) ; if ( ExtMI ) Good = tryAllocateResourcesForConstExt ( true ) ; bool ExtNvjMI = HII -> isExtended ( NvjMI ) || HII -> isConstExtended ( NvjMI ) ; if ( Good ) { if ( ResourceTracker -> canReserveResources ( NvjMI ) ) ResourceTracker -> reserveResources ( NvjMI ) ; else Good = false ; } if ( Good && ExtNvjMI ) Good = tryAllocateResourcesForConstExt ( true ) ; if ( ! Good ) { endPacket ( MBB , MI ) ; assert ( ResourceTracker -> canReserveResources ( MI ) ) ; ResourceTracker -> reserveResources ( MI ) ; if ( ExtMI ) { assert ( canReserveResourcesForConstExt ( ) ) ; tryAllocateResourcesForConstExt ( true ) ; } assert ( ResourceTracker -> canReserveResources ( NvjMI ) ) ; ResourceTracker -> reserveResources ( NvjMI ) ; if ( ExtNvjMI ) { assert ( canReserveResourcesForConstExt ( ) ) ; reserveResourcesForConstExt ( ) ; } }


<s> if ( MI . isInlineAsm ( ) ) return MJ . isInlineAsm ( ) || MJ . isBranch ( ) || MJ . isBarrier ( ) || MJ . isCall ( ) || MJ . isTerminator ( ) ; switch ( MI . getOpcode ( ) ) { case ( <STR_LIT> ) : case ( <STR_LIT> ) : case ( <STR_LIT> ) : case ( <STR_LIT> ) : case ( <STR_LIT> ) : { unsigned TJ = HII . getType ( MJ ) ; if ( TJ != <STR_LIT> && TJ != <STR_LIT> && TJ != <STR_LIT> ) return true ; break ; } default : break ; } return false ;


<s> if ( ! HII -> isHVXVec ( MI ) ) return false ; if ( ! HII -> isHVXVec ( * MII ) ) return false ; if ( HII -> isDotCurInst ( MI ) && ! HII -> mayBeCurLoad ( MI ) ) return false ; if ( ! HII -> mayBeCurLoad ( MI ) ) return false ; if ( PacketSU -> getInstr ( ) -> isInlineAsm ( ) ) return false ; DEBUG ( dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; dbgs ( ) << <STR_LIT> ; ) ; MachineInstr & MJ = * MII ; DEBUG ( { dbgs ( ) << <STR_LIT> ; MJ . dump ( ) ;


<s> if ( ! isNewifiable ( MI , RC ) ) return false ; const MachineInstr & PI = * PacketSU -> getInstr ( ) ; if ( PI . isInlineAsm ( ) ) return false ; if ( PI . isImplicitDef ( ) ) return false ; if ( isImplicitDependency ( PI , true , DepReg ) || isImplicitDependency ( MI , false , DepReg ) ) return false ; const MCInstrDesc & MCID = PI . getDesc ( ) ; const TargetRegisterClass * VecRC = HII -> getRegClass ( MCID , <NUM_LIT> , HRI , MF ) ; if ( DisableVecDblNVStores && VecRC == & <STR_LIT> ) return false ;

if ( RC == & <STR_LIT> ) return HII -> predCanBeUsedAsDotNew ( PI , DepReg ) ;


<s> } if ( MI ) { for ( auto & MO : BI -> operands ( ) ) if ( MO . isReg ( ) && MO . getReg ( ) == MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) return ; } } if ( ! MI ) return ;

MI -> setDesc ( HII -> get ( HII -> getNonDotCurOp ( * MI ) ) ) ;


<s> if ( HII -> isDotCurInst ( * BI ) ) { MI = BI ; continue ; } if ( MI ) {

for ( auto & MO : BI -> operands ( ) ) if ( MO . isReg ( ) && MO . getReg ( ) == MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) return ;


<s> VLIWPacketizerList :: endPacket ( MBB , MI ) ;


VLIWPacketizerList :: endPacket ( MBB , MI ) ;


<s> if ( HII -> isNewValueInst ( J ) || HII -> isMemOp ( J ) || HII -> isMemOp ( I ) ) return true ; } else { bool MopStI = HII -> isMemOp ( I ) || StoreI ; bool MopStJ = HII -> isMemOp ( J ) || StoreJ ; if ( MopStI && MopStJ ) return true ;


<s> HII = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) ; HRI = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getRegisterInfo ( ) ;

addMutation ( make_unique < <ISA_LIT>Subtarget :: <ISA_LIT>DAGMutation > ( ) ) ;


<s> if ( MI . isImplicitDef ( ) ) return false ; const MCInstrDesc & TID = MI . getDesc ( ) ; auto * IS = ResourceTracker -> getInstrItins ( ) -> beginStage ( TID . getSchedClass ( ) ) ; unsigned FuncUnits = IS -> getUnits ( ) ; return ! FuncUnits ;


<s> void <ISA_LIT>PacketizerList :: initPacketizerState ( ) {


<s> bool <ISA_LIT>PacketizerList :: isLegalToPruneDependencies ( SUnit * SUI , SUnit * SUJ ) { assert ( SUI -> getInstr ( ) && SUJ -> getInstr ( ) ) ; MachineInstr & I = * SUI -> getInstr ( ) ; MachineInstr & J = * SUJ -> getInstr ( ) ; bool Coexist = ! cannotCoexist ( I , J ) ; if ( Coexist && ! Dependence ) return true ; if ( PromotedToDotNew ) demoteToDotOld ( I ) ; cleanUpDotCur ( ) ; if ( GlueAllocframeStore ) { useCalleesSP ( I ) ; GlueAllocframeStore = false ; } return false ;


<s> bool <ISA_LIT>PacketizerList :: isSoloInstruction ( const MachineInstr & MI ) { if ( MI . isEHLabel ( ) || MI . isCFIInstruction ( ) ) return true ; if ( MI . isInlineAsm ( ) && ! ScheduleInlineAsm ) return true ; if ( isSchedBarrier ( MI ) ) return true ; if ( HII -> isSolo ( MI ) ) return true ;

if ( MI . getOpcode ( ) == <STR_LIT> ) return true ;


<s> } for ( auto J : OldPacketMIs ) { SUnit * SUJ = MIToSUnit [ J ] ; for ( auto & Pred : SUI -> Preds ) if ( Pred . getSUnit ( ) == SUJ && Pred . getLatency ( ) > <NUM_LIT> ) return true ; } for ( auto J : OldPacketMIs ) { SUnit * SUJ = MIToSUnit [ J ] ; for ( auto & Pred : SUI -> Preds ) if ( Pred . getSUnit ( ) == SUJ && Pred . getLatency ( ) > <NUM_LIT> ) return true ;


<s> if ( PacketStalls ) return false ; if ( ! OldPacketMIs . empty ( ) ) { auto * OldBB = OldPacketMIs . front ( ) -> getParent ( ) ; auto * ThisBB = I . getParent ( ) ; if ( MLI -> getLoopFor ( OldBB ) != MLI -> getLoopFor ( ThisBB ) ) return false ; } SUnit * SUI = MIToSUnit [ const_cast < MachineInstr * > ( & I ) ] ; for ( auto J : CurrentPacketMIs ) { SUnit * SUJ = MIToSUnit [ J ] ; for ( auto & Pred : SUI -> Preds ) if ( Pred . getSUnit ( ) == SUJ && ( Pred . getLatency ( ) == <NUM_LIT> || HII -> isNewValueJump ( I ) || HII -> isToBeScheduledASAP ( * J , I ) ) ) return false ; } for ( auto J : OldPacketMIs ) { SUnit * SUJ = MIToSUnit [ J ] ; for ( auto & Pred : SUI -> Preds ) if ( Pred . getSUnit ( ) == SUJ && Pred . getLatency ( ) > <NUM_LIT> ) return true ; } for ( auto J : OldPacketMIs ) { SUnit * SUJ = MIToSUnit [ J ] ; for ( auto & Pred : SUI -> Preds ) if ( Pred . getSUnit ( ) == SUJ && Pred . getLatency ( ) > <NUM_LIT> ) return true ;


<s> bool <ISA_LIT>PacketizerList :: promoteToDotNew ( MachineInstr & MI , SDep :: Kind DepType , MachineBasicBlock :: iterator & MII , const TargetRegisterClass * RC ) { assert ( DepType == SDep :: Data ) ; int NewOpcode ; if ( RC == & <STR_LIT> ) NewOpcode = HII -> getDotNewPredOp ( MI , MBPI ) ; else NewOpcode = HII -> getDotNewOp ( MI ) ; MI . setDesc ( HII -> get ( NewOpcode ) ) ;


<s> while ( MI != End ) { auto NextI = std :: next ( MI ) ; if ( MI -> isKill ( ) ) { MB . erase ( MI ) ; End = MB . end ( ) ; } MI = NextI ; } } for ( auto & MB : MF ) { auto Begin = MB . begin ( ) , End = MB . end ( ) ; while ( Begin != End ) { MachineBasicBlock :: iterator RB = Begin ;

while ( RB != End && HII -> isSchedulingBoundary ( * RB , & MB , MF ) ) ++ RB ;


<s> bool <ISA_LIT>PacketizerList :: shouldAddToPacket ( const MachineInstr & MI ) {


<s> case <STR_LIT> : break ; default : llvm_unreachable ( <STR_LIT> ) ; } unsigned FrameSize = MF . getFrameInfo ( ) . getStackSize ( ) ; MachineOperand & Off = MI . getOperand ( <NUM_LIT> ) ; int64_t NewOff = Off . getImm ( ) - ( FrameSize + HEXAGON_LRFP_SIZE ) ;

if ( HII -> isValidOffset ( Opc , NewOff ) ) {


<s> bool <ISA_LIT>Packetizer :: runOnMachineFunction ( MachineFunction & MF ) { MF . getProperties ( ) . set ( MachineFunctionProperties :: Property :: FailsVerification ) ; auto & HST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; HII = HST . getInstrInfo ( ) ; HRI = HST . getRegisterInfo ( ) ; auto & MLI = getAnalysis < MachineLoopInfo > ( ) ; auto * AA = & getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; auto * MBPI = & getAnalysis < MachineBranchProbabilityInfo > ( ) ; if ( EnableGenAllInsnClass ) HII -> genAllInsnTimingClasses ( MF ) ; bool MinOnly = Minimal || DisablePacketizer || ! HST . usePackets ( ) || skipFunction ( MF . getFunction ( ) ) ; <ISA_LIT>PacketizerList Packetizer ( MF , MLI , AA , MBPI , MinOnly ) ; assert ( Packetizer . getResourceTracker ( ) && <STR_LIT> ) ; for ( MachineBasicBlock & MB : MF ) { for ( MachineInstr & MI : llvm :: make_early_inc_range ( MB ) ) if ( MI . isKill ( ) ) MB . erase ( & MI ) ; } if ( HST . isTinyCoreWithDuplex ( ) ) HII -> translateInstrsForDup ( MF , true ) ; for ( auto & MB : MF ) { auto Begin = MB . begin ( ) , End = MB . end ( ) ; while ( Begin != End ) { MachineBasicBlock :: iterator RB = Begin ;


<s> assert ( Packetizer . getResourceTracker ( ) && <STR_LIT> ) ; for ( MachineBasicBlock & MB : MF ) { for ( MachineInstr & MI : llvm :: make_early_inc_range ( MB ) ) if ( MI . isKill ( ) ) MB . erase ( & MI ) ; } if ( HST . isTinyCoreWithDuplex ( ) ) HII -> translateInstrsForDup ( MF , true ) ; for ( auto & MB : MF ) { auto Begin = MB . begin ( ) , End = MB . end ( ) ; while ( Begin != End ) { MachineBasicBlock :: iterator RB = Begin ; while ( RB != End && HII -> isSchedulingBoundary ( * RB , & MB , MF ) ) ++ RB ; MachineBasicBlock :: iterator RE = RB ; while ( RE != End && ! HII -> isSchedulingBoundary ( * RE , & MB , MF ) ) ++ RE ; if ( RE != End ) ++ RE ; if ( RB != End ) Packetizer . PacketizeMIs ( & MB , RB , RE ) ; Begin = RE ; } } if ( HST . isTinyCoreWithDuplex ( ) ) HII -> translateInstrsForDup ( MF , false ) ;

Packetizer . unpacketizeSoloInstrs ( MF ) ;


<s> bool InsertBeforeBundle ; if ( MI . isInlineAsm ( ) ) InsertBeforeBundle = ! hasWriteToReadDep ( MI , * BundleIt , HRI ) ; else if ( MI . isDebugValue ( ) ) InsertBeforeBundle = true ; else continue ; BundleIt = moveInstrOut ( MI , BundleIt , InsertBeforeBundle ) ;


<s> <ISA_LIT>Packetizer ( ) : MachineFunctionPass ( ID ) {


addMutation ( llvm :: make_unique < <ISA_LIT>Subtarget :: UsrOverflowMutation > ( ) ) ;


<s> const MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; if ( MF -> getSubtarget < <ISA_LIT>Subtarget > ( ) . hasV60TOpsOnly ( ) && HII . isHVXMemWithAIndirect ( MI , MJ ) ) return true ; if ( MI . isInlineAsm ( ) ) return MJ . isInlineAsm ( ) || MJ . isBranch ( ) || MJ . isBarrier ( ) || MJ . isCall ( ) || MJ . isTerminator ( ) ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> MachineBasicBlock :: instr_iterator FirstMI ( CurrentPacketMIs . front ( ) ) ; MachineBasicBlock :: instr_iterator LastMI ( MI . getInstrIterator ( ) ) ; finalizeBundle ( * MBB , FirstMI , LastMI ) ; auto BundleMII = std :: prev ( FirstMI ) ; if ( memShufDisabled ) HII -> setBundleNoShuf ( BundleMII ) ; setmemShufDisabled ( false ) ; } OldPacketMIs = CurrentPacketMIs ; CurrentPacketMIs . clear ( ) ; ResourceTracker -> clearResources ( ) ;


<s> inline bool isPromotedToDotNew ( ) const { return PromotedToDotNew ;


<s> auto MI = MB . begin ( ) ; while ( MI != End ) { auto NextI = std :: next ( MI ) ; if ( MI -> isKill ( ) ) { MB . erase ( MI ) ; End = MB . end ( ) ; } MI = NextI ; } } for ( auto & MB : MF ) { auto Begin = MB . begin ( ) , End = MB . end ( ) ; while ( Begin != End ) { MachineBasicBlock :: iterator RB = Begin ; while ( RB != End && HII -> isSchedulingBoundary ( * RB , & MB , MF ) ) ++ RB ; MachineBasicBlock :: iterator RE = RB ; while ( RE != End && ! HII -> isSchedulingBoundary ( * RE , & MB , MF ) ) ++ RE ; if ( RE != End ) ++ RE ;

if ( RB != End ) Packetizer . PacketizeMIs ( & MB , RB , RE ) ;


<s> <ISA_LIT>PacketizerList Packetizer ( MF , MLI , AA , MBPI ) ; assert ( Packetizer . getResourceTracker ( ) && <STR_LIT> ) ; for ( auto & MB : MF ) { auto End = MB . end ( ) ; auto MI = MB . begin ( ) ; while ( MI != End ) { auto NextI = std :: next ( MI ) ; if ( MI -> isKill ( ) ) { MB . erase ( MI ) ; End = MB . end ( ) ; } MI = NextI ; } } for ( auto & MB : MF ) { auto Begin = MB . begin ( ) , End = MB . end ( ) ; while ( Begin != End ) { MachineBasicBlock :: iterator RB = Begin ; while ( RB != End && HII -> isSchedulingBoundary ( * RB , & MB , MF ) ) ++ RB ; MachineBasicBlock :: iterator RE = RB ; while ( RE != End && ! HII -> isSchedulingBoundary ( * RE , & MB , MF ) ) ++ RE ;


<s> assert ( ( QII -> isExtended ( MI ) || QII -> isConstExtended ( MI ) ) && <STR_LIT> ) ; MachineFunction * MF = MI -> getParent ( ) -> getParent ( ) ; MachineInstr * PseudoMI = MF -> CreateMachineInstr ( QII -> get ( <STR_LIT> ) , MI -> getDebugLoc ( ) ) ;

bool CanReserve = ResourceTracker -> canReserveResources ( PseudoMI ) ;


<s> for ( const MCPhysReg * CSR = TRI -> getCalleeSavedRegs ( ) ; * CSR ; ++ CSR ) { unsigned CalleeSavedReg = * CSR ; if ( MI -> modifiesRegister ( CalleeSavedReg , TRI ) ) return true ; } return false ;


<s> unsigned SchedClass = TID . getSchedClass ( ) ; const InstrStage * IS = ResourceTracker -> getInstrItins ( ) -> beginStage ( SchedClass ) ;

unsigned FuncUnits = IS -> getUnits ( ) ;


<s> const <ISA_LIT>InstrInfo * QII = ( const <ISA_LIT>InstrInfo * ) TII ; if ( isCondInst ( MI ) || QII -> mayBeNewStore ( MI ) ) return true ;


<s> if ( MI -> isInlineAsm ( ) ) return true ; if ( MI -> isEHLabel ( ) ) return true ; if ( IsSchedBarrier ( MI ) ) return true ;


<s> MachineFunction * MF = MI -> getParent ( ) -> getParent ( ) ; MachineInstr * PseudoMI = MF -> CreateMachineInstr ( QII -> get ( <STR_LIT> ) , MI -> getDebugLoc ( ) ) ; if ( ResourceTracker -> canReserveResources ( PseudoMI ) ) {

ResourceTracker -> reserveResources ( PseudoMI ) ;


<s> MachineFunction * MF = MI -> getParent ( ) -> getParent ( ) ; MachineInstr * PseudoMI = MF -> CreateMachineInstr ( QII -> get ( <STR_LIT> ) , MI -> getDebugLoc ( ) ) ; if ( ResourceTracker -> canReserveResources ( PseudoMI ) ) { ResourceTracker -> reserveResources ( PseudoMI ) ;

MI -> getParent ( ) -> getParent ( ) -> DeleteMachineInstr ( PseudoMI ) ;


<s> bool <ISA_LIT>PacketizerList :: tryAllocateResourcesForConstExt ( MachineInstr * MI ) { const <ISA_LIT>InstrInfo * QII = ( const <ISA_LIT>InstrInfo * ) TII ;

MachineFunction * MF = MI -> getParent ( ) -> getParent ( ) ;


<s> bool <ISA_LIT>PacketizerList :: CanPromoteToDotNew ( MachineInstr * MI , SUnit * PacketSU , unsigned DepReg , std :: map < MachineInstr * , SUnit * > MIToSUnit , MachineBasicBlock :: iterator & MII , const TargetRegisterClass * RC ) { const <ISA_LIT>InstrInfo * QII = ( const <ISA_LIT>InstrInfo * ) TII ; if ( QII -> isDotNewInst ( MI ) && ! QII -> mayBeNewStore ( MI ) ) return false ; if ( ! isNewifiable ( MI ) ) return false ; if ( RC == & <STR_LIT> && isCondInst ( MI ) ) return true ; else if ( RC != & <STR_LIT> && ! QII -> mayBeNewStore ( MI ) ) return false ; else { int NewOpcode = QII -> GetDotNewOp ( MI ) ; const MCInstrDesc & desc = QII -> get ( NewOpcode ) ; DebugLoc dl ; MachineInstr * NewMI = MI -> getParent ( ) -> getParent ( ) -> CreateMachineInstr ( desc , dl ) ; bool ResourcesAvailable = ResourceTracker -> canReserveResources ( NewMI ) ;

MI -> getParent ( ) -> getParent ( ) -> DeleteMachineInstr ( NewMI ) ;


<s> else if ( RC != & <STR_LIT> && ! QII -> mayBeNewStore ( MI ) ) return false ; else { int NewOpcode = QII -> GetDotNewOp ( MI ) ; const MCInstrDesc & desc = QII -> get ( NewOpcode ) ; DebugLoc dl ; MachineInstr * NewMI = MI -> getParent ( ) -> getParent ( ) -> CreateMachineInstr ( desc , dl ) ; bool ResourcesAvailable = ResourceTracker -> canReserveResources ( NewMI ) ; MI -> getParent ( ) -> getParent ( ) -> DeleteMachineInstr ( NewMI ) ; if ( ! ResourcesAvailable ) return false ; if ( ! CanPromoteToNewValue ( MI , PacketSU , DepReg , MIToSUnit , MII ) ) { return false ; } }


<s> bool <ISA_LIT>PacketizerList :: CanPromoteToNewValue ( MachineInstr * MI , SUnit * PacketSU , unsigned DepReg , std :: map < MachineInstr * , SUnit * > MIToSUnit , MachineBasicBlock :: iterator & MII ) { const <ISA_LIT>InstrInfo * QII = ( const <ISA_LIT>InstrInfo * ) TII ; const <ISA_LIT>RegisterInfo * QRI = ( const <ISA_LIT>RegisterInfo * ) TM . getRegisterInfo ( ) ; if ( ! QRI -> Subtarget . hasV4TOps ( ) || ! QII -> mayBeNewStore ( MI ) ) return false ;

MachineInstr * PacketMI = PacketSU -> getInstr ( ) ;


<s> const <ISA_LIT>InstrInfo * QII = ( const <ISA_LIT>InstrInfo * ) TII ; const <ISA_LIT>RegisterInfo * QRI = ( const <ISA_LIT>RegisterInfo * ) TM . getRegisterInfo ( ) ; if ( ! QRI -> Subtarget . hasV4TOps ( ) || ! QII -> mayBeNewStore ( MI ) ) return false ;


<s> if ( ! QII -> mayBeNewStore ( MI ) ) return false ; if ( GetStoreValueOperand ( MI ) . isReg ( ) && GetStoreValueOperand ( MI ) . getReg ( ) != DepReg ) return false ; const <ISA_LIT>RegisterInfo * QRI = ( const <ISA_LIT>RegisterInfo * ) TM . getRegisterInfo ( ) ; const MCInstrDesc & MCID = PacketMI -> getDesc ( ) ; const TargetRegisterClass * PacketRC = QII -> getRegClass ( MCID , <NUM_LIT> , QRI , MF ) ; for ( std :: vector < MachineInstr * > :: iterator VI = CurrentPacketMIs . begin ( ) , VE = CurrentPacketMIs . end ( ) ; ( VI != VE ) ; ++ VI ) { SUnit * PacketSU = MIToSUnit [ * VI ] ; if ( PacketSU -> getInstr ( ) -> getDesc ( ) . mayStore ( ) || PacketSU -> getInstr ( ) -> getOpcode ( ) == <STR_LIT> || PacketSU -> getInstr ( ) -> getOpcode ( ) == <STR_LIT> ) return false ; } if ( PacketRC == & <STR_LIT> ) { return false ; } if ( QII -> isPostIncrement ( MI ) && MI -> getDesc ( ) . mayStore ( ) && GetPostIncrementOperand ( MI , QII ) . getReg ( ) == DepReg ) { return false ; } if ( QII -> isPostIncrement ( PacketMI ) && PacketMI -> getDesc ( ) . mayLoad ( ) && GetPostIncrementOperand ( PacketMI , QII ) . getReg ( ) == DepReg ) { return false ; } if ( QII -> isPredicated ( PacketMI ) ) { if ( ! QII -> isPredicated ( MI ) ) return false ; unsigned predRegNumSrc = <NUM_LIT> ; unsigned predRegNumDst = <NUM_LIT> ; const TargetRegisterClass * predRegClass = NULL ; for ( unsigned opNum = <NUM_LIT> ; opNum < PacketMI -> getNumOperands ( ) ; opNum ++ ) { if ( PacketMI -> getOperand ( opNum ) . isReg ( ) ) predRegNumSrc = PacketMI -> getOperand ( opNum ) . getReg ( ) ; predRegClass = QRI -> getMinimalPhysRegClass ( predRegNumSrc ) ; if ( predRegClass == & <STR_LIT> ) { break ; } } assert ( ( predRegClass == & <STR_LIT> ) && ( <STR_LIT> ) ) ; for ( unsigned opNum = <NUM_LIT> ; opNum < MI -> getNumOperands ( ) ; opNum ++ ) {

if ( MI -> getOperand ( opNum ) . isReg ( ) ) predRegNumDst = MI -> getOperand ( opNum ) . getReg ( ) ;


<s> MachineInstr * PseudoMI = MF -> CreateMachineInstr ( QII -> get ( <STR_LIT> ) , MI -> getDebugLoc ( ) ) ; bool CanReserve = ResourceTracker -> canReserveResources ( PseudoMI ) ; MF -> DeleteMachineInstr ( PseudoMI ) ; return CanReserve ;


<s> static bool DoesModifyCalleeSavedReg ( MachineInstr * MI , const TargetRegisterInfo * TRI ) { for ( const uint16_t * CSR = TRI -> getCalleeSavedRegs ( ) ; * CSR ; ++ CSR ) { unsigned CalleeSavedReg = * CSR ; if ( MI -> modifiesRegister ( CalleeSavedReg , TRI ) ) return true ; }


<s> if ( MI -> modifiesRegister ( CalleeSavedReg , TRI ) ) return true ; }


<s> AU . addRequired < MachineLoopInfo > ( ) ; AU . addPreserved < MachineLoopInfo > ( ) ;

MachineFunctionPass :: getAnalysisUsage ( AU ) ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const { AU . setPreservesCFG ( ) ; AU . addRequired < MachineDominatorTree > ( ) ;

AU . addRequired < MachineBranchProbabilityInfo > ( ) ;


<s> const char * getPassName ( ) const {


<s> <ISA_LIT>PacketizerList :: <ISA_LIT>PacketizerList ( MachineFunction & MF , MachineLoopInfo & MLI , MachineDominatorTree & MDT , const MachineBranchProbabilityInfo * MBPI ) : VLIWPacketizerList ( MF , MLI , MDT , true ) { this -> MBPI = MBPI ;


<s> } if ( QII -> isDeallocRet ( MI ) ) { if ( DepReg == QRI -> getFrameRegister ( ) || DepReg == QRI -> getStackRegister ( ) ) return true ; } const TargetRegisterClass * RC = QRI -> getMinimalPhysRegClass ( DepReg ) ; if ( RC == & <STR_LIT> ) { return true ; } if ( IsIndirectCall ( MI ) && ( DepType == SDep :: Data ) ) { MachineOperand MO = MI -> getOperand ( <NUM_LIT> ) ; if ( MO . isReg ( ) && MO . isUse ( ) && ( MO . getReg ( ) == DepReg ) ) {


<s> return ( MI -> getOpcode ( ) == <STR_LIT> ) ;


<s> static bool IsIndirectCall ( MachineInstr * MI ) { return ( ( MI -> getOpcode ( ) == <STR_LIT> ) || ( MI -> getOpcode ( ) == <STR_LIT> ) ) ;


<s> return ( ( MI -> getOpcode ( ) == <STR_LIT> ) || ( MI -> getOpcode ( ) == <STR_LIT> ) ) ;


<s> return ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) ;


<s> static bool IsSchedBarrier ( MachineInstr * MI ) { switch ( MI -> getOpcode ( ) ) { case <STR_LIT> : return true ;


<s> const <ISA_LIT>InstrInfo * QII = ( const <ISA_LIT>InstrInfo * ) TII ; MachineFunction * MF = MI -> getParent ( ) -> getParent ( ) ; MachineInstr * PseudoMI = MF -> CreateMachineInstr ( QII -> get ( <STR_LIT> ) , MI -> getDebugLoc ( ) ) ; if ( ResourceTracker -> canReserveResources ( PseudoMI ) ) { ResourceTracker -> reserveResources ( PseudoMI ) ; MI -> getParent ( ) -> getParent ( ) -> DeleteMachineInstr ( PseudoMI ) ;


<s> void <ISA_LIT>PacketizerList :: reserveResourcesForConstExt ( MachineInstr * MI ) { const <ISA_LIT>InstrInfo * QII = ( const <ISA_LIT>InstrInfo * ) TII ; MachineFunction * MF = MI -> getParent ( ) -> getParent ( ) ; MachineInstr * PseudoMI = MF -> CreateMachineInstr ( QII -> get ( <STR_LIT> ) , MI -> getDebugLoc ( ) ) ; if ( ResourceTracker -> canReserveResources ( PseudoMI ) ) { ResourceTracker -> reserveResources ( PseudoMI ) ; MI -> getParent ( ) -> getParent ( ) -> DeleteMachineInstr ( PseudoMI ) ;


<s> if ( ! QII -> isPredicated ( * VIN ) ) continue ; SUnit * PacketSU = MIToSUnit [ * VIN ] ; if ( PacketSU -> isSucc ( PacketSUDep ) ) { for ( unsigned i = <NUM_LIT> ; i < PacketSU -> Succs . size ( ) ; ++ i ) { if ( ( PacketSU -> Succs [ i ] . getSUnit ( ) == PacketSUDep ) && ( PacketSU -> Succs [ i ] . getKind ( ) == SDep :: Anti ) && ( PacketSU -> Succs [ i ] . getReg ( ) == DepReg ) ) { return true ;


<s> for ( unsigned i = <NUM_LIT> ; i < PacketSU -> Succs . size ( ) ; ++ i ) { if ( ( PacketSU -> Succs [ i ] . getSUnit ( ) == PacketSUDep ) && ( PacketSU -> Succs [ i ] . getKind ( ) == SDep :: Anti ) && ( PacketSU -> Succs [ i ] . getReg ( ) == DepReg ) ) { return true ; } } }


<s> End = MBB -> end ( ) ; continue ; } ++ MI ; } } for ( MachineFunction :: iterator MBB = Fn . begin ( ) , MBBe = Fn . end ( ) ; MBB != MBBe ; ++ MBB ) { unsigned RemainingCount = MBB -> size ( ) ; for ( MachineBasicBlock :: iterator RegionEnd = MBB -> end ( ) ; RegionEnd != MBB -> begin ( ) ; ) { MachineBasicBlock :: iterator I = RegionEnd ; for ( ; I != MBB -> begin ( ) ; -- I , -- RemainingCount ) { if ( TII -> isSchedulingBoundary ( std :: prev ( I ) , MBB , Fn ) ) break ; } I = MBB -> begin ( ) ; if ( I == RegionEnd ) { RegionEnd = std :: prev ( RegionEnd ) ; -- RemainingCount ; continue ; } if ( I == std :: prev ( RegionEnd ) ) { RegionEnd = std :: prev ( RegionEnd ) ; continue ; } Packetizer . PacketizeMIs ( MBB , I , RegionEnd ) ; RegionEnd = I ; } }


<s> ResourceTracker -> reserveResources ( PseudoMI ) ; MI -> getParent ( ) -> getParent ( ) -> DeleteMachineInstr ( PseudoMI ) ; return true ; } else { MI -> getParent ( ) -> getParent ( ) -> DeleteMachineInstr ( PseudoMI ) ; return false ; }


<s> MachineInstr * PseudoMI = MF -> CreateMachineInstr ( QII -> get ( <STR_LIT> ) , MI -> getDebugLoc ( ) ) ; if ( ResourceTracker -> canReserveResources ( PseudoMI ) ) {


<s> addMutation ( std :: make_unique < <ISA_LIT>Subtarget :: UsrOverflowMutation > ( ) ) ; addMutation ( std :: make_unique < <ISA_LIT>Subtarget :: HVXMemLatencyMutation > ( ) ) ; addMutation ( std :: make_unique < <ISA_LIT>Subtarget :: BankConflictMutation > ( ) ) ;


<s> assert ( ResourceTracker -> canReserveResources ( MI ) ) ; ResourceTracker -> reserveResources ( MI ) ; if ( QII -> isExtended ( MI ) && ! tryAllocateResourcesForConstExt ( MI ) ) { endPacket ( MBB , MI ) ; ResourceTracker -> reserveResources ( MI ) ; assert ( canReserveResourcesForConstExt ( MI ) && <STR_LIT> ) ; reserveResourcesForConstExt ( MI ) ; assert ( canReserveResourcesForConstExt ( MI ) && <STR_LIT> ) ; reserveResourcesForConstExt ( nvjMI ) ; assert ( ResourceTracker -> canReserveResources ( nvjMI ) && <STR_LIT> ) ; } else if ( ( QII -> isExtended ( nvjMI ) && ( ! tryAllocateResourcesForConstExt ( nvjMI ) || ! ResourceTracker -> canReserveResources ( nvjMI ) ) ) || ( ! QII -> isExtended ( nvjMI ) && ! ResourceTracker -> canReserveResources ( nvjMI ) ) ) { endPacket ( MBB , MI ) ; ResourceTracker -> reserveResources ( MI ) ; if ( QII -> isExtended ( nvjMI ) ) reserveResourcesForConstExt ( nvjMI ) ; } ResourceTracker -> reserveResources ( nvjMI ) ; CurrentPacketMIs . push_back ( MI ) ; CurrentPacketMIs . push_back ( nvjMI ) ;


<s> else { const <ISA_LIT>InstrInfo * QII = ( const <ISA_LIT>InstrInfo * ) TII ; int NewOpcode = GetDotNewOp ( MI -> getOpcode ( ) ) ; const MCInstrDesc & desc = QII -> get ( NewOpcode ) ; DebugLoc dl ; MachineInstr * NewMI = MI -> getParent ( ) -> getParent ( ) -> CreateMachineInstr ( desc , dl ) ; bool ResourcesAvailable = ResourceTracker -> canReserveResources ( NewMI ) ; MI -> getParent ( ) -> getParent ( ) -> DeleteMachineInstr ( NewMI ) ; if ( ! ResourcesAvailable ) return false ; if ( ! CanPromoteToNewValue ( MI , PacketSU , DepReg , MIToSUnit , MII ) ) {


<s> if ( ! QRI -> Subtarget . hasV4TOps ( ) || ! IsNewifyStore ( MI ) ) return false ; MachineInstr * PacketMI = PacketSU -> getInstr ( ) ; if ( CanPromoteToNewValueStore ( MI , PacketMI , DepReg , MIToSUnit ) ) return true ; return false ;


<s> bool <ISA_LIT>PacketizerList :: CanPromoteToNewValue ( MachineInstr * MI , SUnit * PacketSU , unsigned DepReg , std :: map < MachineInstr * , SUnit * > MIToSUnit , MachineBasicBlock :: iterator & MII ) { const <ISA_LIT>RegisterInfo * QRI = ( const <ISA_LIT>RegisterInfo * ) TM . getRegisterInfo ( ) ; if ( ! QRI -> Subtarget . hasV4TOps ( ) || ! IsNewifyStore ( MI ) ) return false ; MachineInstr * PacketMI = PacketSU -> getInstr ( ) ; if ( CanPromoteToNewValueStore ( MI , PacketMI , DepReg , MIToSUnit ) ) return true ;


<s> if ( QII -> isPostIncrement ( PacketMI ) && PacketMI -> getDesc ( ) . mayLoad ( ) && GetPostIncrementOperand ( PacketMI , QII ) . getReg ( ) == DepReg ) { return false ; } if ( QII -> isPredicated ( PacketMI ) ) { if ( ! QII -> isPredicated ( MI ) ) return false ; unsigned predRegNumSrc = <NUM_LIT> ; unsigned predRegNumDst = <NUM_LIT> ; const TargetRegisterClass * predRegClass = NULL ; for ( unsigned opNum = <NUM_LIT> ; opNum < PacketMI -> getNumOperands ( ) ; opNum ++ ) { if ( PacketMI -> getOperand ( opNum ) . isReg ( ) ) predRegNumSrc = PacketMI -> getOperand ( opNum ) . getReg ( ) ; predRegClass = QRI -> getMinimalPhysRegClass ( predRegNumSrc ) ; if ( predRegClass == & <STR_LIT> ) { break ; } } assert ( ( predRegClass == & <STR_LIT> ) && ( <STR_LIT> ) ) ; for ( unsigned opNum = <NUM_LIT> ; opNum < MI -> getNumOperands ( ) ; opNum ++ ) { if ( MI -> getOperand ( opNum ) . isReg ( ) ) predRegNumDst = MI -> getOperand ( opNum ) . getReg ( ) ; predRegClass = QRI -> getMinimalPhysRegClass ( predRegNumDst ) ; if ( predRegClass == & <STR_LIT> ) { break ; } } assert ( ( predRegClass == & <STR_LIT> ) && ( <STR_LIT> ) ) ; if ( ( predRegNumDst != predRegNumSrc ) || isDotNewInst ( PacketMI ) != isDotNewInst ( MI ) || GetPredicateSense ( MI , QII ) != GetPredicateSense ( PacketMI , QII ) ) { return false ; } } std :: vector < MachineInstr * > :: iterator VI ; std :: vector < MachineInstr * > :: iterator VE ; unsigned StartCheck = <NUM_LIT> ; for ( VI = CurrentPacketMIs . begin ( ) , VE = CurrentPacketMIs . end ( ) ; ( VI != VE ) ; ++ VI ) { SUnit * TempSU = MIToSUnit [ * VI ] ; MachineInstr * TempMI = TempSU -> getInstr ( ) ; if ( TempMI != PacketMI && ! StartCheck ) continue ; StartCheck = <NUM_LIT> ; if ( TempMI == PacketMI ) continue ; for ( unsigned opNum = <NUM_LIT> ; opNum < MI -> getNumOperands ( ) ; opNum ++ ) { if ( MI -> getOperand ( opNum ) . isReg ( ) && TempSU -> getInstr ( ) -> modifiesRegister ( MI -> getOperand ( opNum ) . getReg ( ) , QRI ) ) return false ; }


<s> bool <ISA_LIT>PacketizerList :: canReserveResourcesForConstExt ( MachineInstr * MI ) { const <ISA_LIT>InstrInfo * QII = ( const <ISA_LIT>InstrInfo * ) TII ; assert ( QII -> isExtended ( MI ) && <STR_LIT> ) ; MachineFunction * MF = MI -> getParent ( ) -> getParent ( ) ; MachineInstr * PseudoMI = MF -> CreateMachineInstr ( QII -> get ( <STR_LIT> ) , MI -> getDebugLoc ( ) ) ; bool CanReserve = ResourceTracker -> canReserveResources ( PseudoMI ) ; MF -> DeleteMachineInstr ( PseudoMI ) ;


<s> MachineFunction * MF = MI -> getParent ( ) -> getParent ( ) ;

MachineInstr * PseudoMI = MF -> CreateMachineInstr ( QII -> get ( <STR_LIT> ) , MI -> getDebugLoc ( ) ) ;


<s> bool <ISA_LIT>PacketizerList :: DemoteToDotOld ( MachineInstr * MI ) { const <ISA_LIT>InstrInfo * QII = ( const <ISA_LIT>InstrInfo * ) TII ;

int NewOpcode = GetDotOldOp ( MI -> getOpcode ( ) ) ;


<s> AU . addRequired < MachineDominatorTree > ( ) ; AU . addPreserved < MachineDominatorTree > ( ) ;

AU . addRequired < MachineLoopInfo > ( ) ;


<s> AU . addRequired < MachineDominatorTree > ( ) ; AU . addPreserved < MachineDominatorTree > ( ) ; AU . addRequired < MachineLoopInfo > ( ) ; AU . addPreserved < MachineLoopInfo > ( ) ;


<s> return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ;


<s> <ISA_LIT>PacketizerList :: <ISA_LIT>PacketizerList ( MachineFunction & MF , MachineLoopInfo & MLI , MachineDominatorTree & MDT ) : VLIWPacketizerList ( MF , MLI , MDT , true ) {


<s> Dependence = true ; return false ; } if ( ( QII -> isMemOp ( J ) && MCIDI . mayStore ( ) ) || ( MCIDJ . mayStore ( ) && QII -> isMemOp ( I ) ) || ( QII -> isMemOp ( J ) && QII -> isMemOp ( I ) ) ) { Dependence = true ; return false ; } if ( MCIDJ . mayStore ( ) && QII -> isDeallocRet ( I ) ) { Dependence = true ; return false ; } MachineBasicBlock :: iterator NextMII = I ; ++ NextMII ; MachineInstr * NextMI = NextMII ; if ( QII -> isNewValueJump ( NextMI ) ) { bool secondRegMatch = false ; bool maintainNewValueJump = false ; if ( NextMI -> getOperand ( <NUM_LIT> ) . isReg ( ) && I -> getOperand ( <NUM_LIT> ) . getReg ( ) == NextMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) { secondRegMatch = true ; maintainNewValueJump = true ; } if ( ! secondRegMatch && I -> getOperand ( <NUM_LIT> ) . getReg ( ) == NextMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) { maintainNewValueJump = true ; } for ( std :: vector < MachineInstr * > :: iterator VI = CurrentPacketMIs . begin ( ) , VE = CurrentPacketMIs . end ( ) ; ( VI != VE && maintainNewValueJump ) ; ++ VI ) { SUnit * PacketSU = MIToSUnit [ * VI ] ; if ( PacketSU -> getInstr ( ) -> getDesc ( ) . isCall ( ) ) { Dependence = true ; break ; } if ( PacketSU -> getInstr ( ) -> getDesc ( ) . mayStore ( ) || PacketSU -> getInstr ( ) -> getOpcode ( ) == <STR_LIT> || ( ! secondRegMatch && NextMI -> getOperand ( <NUM_LIT> ) . isReg ( ) && PacketSU -> getInstr ( ) -> modifiesRegister ( NextMI -> getOperand ( <NUM_LIT> ) . getReg ( ) , QRI ) ) || ( secondRegMatch && PacketSU -> getInstr ( ) -> modifiesRegister ( NextMI -> getOperand ( <NUM_LIT> ) . getReg ( ) , QRI ) ) ) { Dependence = true ; break ; } } if ( ! Dependence ) GlueToNewValueJump = true ; else return false ; } } if ( SUJ -> isSucc ( SUI ) ) { for ( unsigned i = <NUM_LIT> ; ( i < SUJ -> Succs . size ( ) ) && ! FoundSequentialDependence ;


<s> bool <ISA_LIT>PacketizerList :: isNewifiable ( MachineInstr * MI ) {

if ( isCondInst ( MI ) || IsNewifyStore ( MI ) ) return true ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return QRI -> Subtarget . hasV4TOps ( ) ; } return false ;


<s> if ( QII -> isNewValueJump ( MI ) ) return true ;

if ( QII -> isNewValueStore ( MI ) ) return true ;


<s> if ( QII -> isNewValueJump ( MI ) ) return true ; if ( QII -> isNewValueStore ( MI ) ) return true ; return false ;


<s> if ( RC == & <STR_LIT> ) NewOpcode = GetDotNewPredOp ( MI -> getOpcode ( ) ) ; else NewOpcode = GetDotNewOp ( MI -> getOpcode ( ) ) ;

MI -> setDesc ( QII -> get ( NewOpcode ) ) ;


<s> if ( ResourceTracker -> canReserveResources ( PseudoMI ) ) { ResourceTracker -> reserveResources ( PseudoMI ) ; MI -> getParent ( ) -> getParent ( ) -> DeleteMachineInstr ( PseudoMI ) ;


<s> while ( MI != End ) { if ( MI -> isKill ( ) ) { MachineBasicBlock :: iterator DeleteMI = MI ; ++ MI ; MBB -> erase ( DeleteMI ) ; End = MBB -> end ( ) ; continue ; } ++ MI ; } } for ( MachineFunction :: iterator MBB = Fn . begin ( ) , MBBe = Fn . end ( ) ; MBB != MBBe ; ++ MBB ) { unsigned RemainingCount = MBB -> size ( ) ; for ( MachineBasicBlock :: iterator RegionEnd = MBB -> end ( ) ; RegionEnd != MBB -> begin ( ) ; ) { MachineBasicBlock :: iterator I = RegionEnd ; for ( ; I != MBB -> begin ( ) ; -- I , -- RemainingCount ) { if ( TII -> isSchedulingBoundary ( llvm :: prior ( I ) , MBB , Fn ) ) break ; } I = MBB -> begin ( ) ;


<s> MBB != MBBe ; ++ MBB ) { unsigned RemainingCount = MBB -> size ( ) ; for ( MachineBasicBlock :: iterator RegionEnd = MBB -> end ( ) ; RegionEnd != MBB -> begin ( ) ; ) { MachineBasicBlock :: iterator I = RegionEnd ; for ( ; I != MBB -> begin ( ) ; -- I , -- RemainingCount ) { if ( TII -> isSchedulingBoundary ( llvm :: prior ( I ) , MBB , Fn ) ) break ; } I = MBB -> begin ( ) ; if ( I == RegionEnd ) { RegionEnd = llvm :: prior ( RegionEnd ) ; -- RemainingCount ; continue ; } if ( I == llvm :: prior ( RegionEnd ) ) { RegionEnd = llvm :: prior ( RegionEnd ) ; continue ; } Packetizer . PacketizeMIs ( MBB , I , RegionEnd ) ; RegionEnd = I ;


<s> if ( ResourceTracker -> canReserveResources ( PseudoMI ) ) { ResourceTracker -> reserveResources ( PseudoMI ) ; MI -> getParent ( ) -> getParent ( ) -> DeleteMachineInstr ( PseudoMI ) ; return true ; } else { MI -> getParent ( ) -> getParent ( ) -> DeleteMachineInstr ( PseudoMI ) ;


<s> } assert ( ( predRegClass == & <STR_LIT> ) && ( <STR_LIT> ) ) ; if ( ( predRegNumDst != predRegNumSrc ) || QII -> isDotNewInst ( PacketMI ) != QII -> isDotNewInst ( MI ) || getPredicateSense ( MI , QII ) != getPredicateSense ( PacketMI , QII ) ) { return false ; } } std :: vector < MachineInstr * > :: iterator VI ; std :: vector < MachineInstr * > :: iterator VE ; unsigned StartCheck = <NUM_LIT> ; for ( VI = CurrentPacketMIs . begin ( ) , VE = CurrentPacketMIs . end ( ) ; ( VI != VE ) ; ++ VI ) { SUnit * TempSU = MIToSUnit . find ( * VI ) -> second ; MachineInstr * TempMI = TempSU -> getInstr ( ) ; if ( TempMI != PacketMI && ! StartCheck ) continue ; StartCheck = <NUM_LIT> ; if ( TempMI == PacketMI ) continue ; for ( unsigned opNum = <NUM_LIT> ; opNum < MI -> getNumOperands ( ) ; opNum ++ ) { if ( MI -> getOperand ( opNum ) . isReg ( ) && TempSU -> getInstr ( ) -> modifiesRegister ( MI -> getOperand ( opNum ) . getReg ( ) , QRI ) ) return false ; } } if ( ! QII -> isPostIncrement ( MI ) && GetStoreValueOperand ( MI ) . isReg ( ) && GetStoreValueOperand ( MI ) . getReg ( ) == DepReg ) { for ( unsigned opNum = <NUM_LIT> ; opNum < MI -> getNumOperands ( ) - <NUM_LIT> ; opNum ++ ) { if ( MI -> getOperand ( opNum ) . isReg ( ) && MI -> getOperand ( opNum ) . getReg ( ) == DepReg ) { return false ; } }

for ( unsigned opNum = <NUM_LIT> ; opNum < PacketMI -> getNumOperands ( ) ; opNum ++ ) {


<s> for ( unsigned opNum = <NUM_LIT> ; opNum < PacketMI -> getNumOperands ( ) ; opNum ++ ) { if ( PacketMI -> getOperand ( opNum ) . isReg ( ) ) predRegNumSrc = PacketMI -> getOperand ( opNum ) . getReg ( ) ; predRegClass = QRI -> getMinimalPhysRegClass ( predRegNumSrc ) ; if ( predRegClass == & <STR_LIT> ) { break ; } } assert ( ( predRegClass == & <STR_LIT> ) && ( <STR_LIT> ) ) ; for ( unsigned opNum = <NUM_LIT> ; opNum < MI -> getNumOperands ( ) ; opNum ++ ) { if ( MI -> getOperand ( opNum ) . isReg ( ) ) predRegNumDst = MI -> getOperand ( opNum ) . getReg ( ) ; predRegClass = QRI -> getMinimalPhysRegClass ( predRegNumDst ) ; if ( predRegClass == & <STR_LIT> ) { break ; } } assert ( ( predRegClass == & <STR_LIT> ) && ( <STR_LIT> ) ) ; if ( ( predRegNumDst != predRegNumSrc ) || QII -> isDotNewInst ( PacketMI ) != QII -> isDotNewInst ( MI ) || getPredicateSense ( MI , QII ) != getPredicateSense ( PacketMI , QII ) ) { return false ; } } std :: vector < MachineInstr * > :: iterator VI ; std :: vector < MachineInstr * > :: iterator VE ; unsigned StartCheck = <NUM_LIT> ; for ( VI = CurrentPacketMIs . begin ( ) , VE = CurrentPacketMIs . end ( ) ; ( VI != VE ) ; ++ VI ) { SUnit * TempSU = MIToSUnit . find ( * VI ) -> second ; MachineInstr * TempMI = TempSU -> getInstr ( ) ; if ( TempMI != PacketMI && ! StartCheck ) continue ; StartCheck = <NUM_LIT> ; if ( TempMI == PacketMI ) continue ;


<s> MachineInstr & NvjMI = * ++ MII ; ResourceTracker -> reserveResources ( MI ) ; if ( ExtMI ) Good = tryAllocateResourcesForConstExt ( true ) ; bool ExtNvjMI = HII -> isExtended ( NvjMI ) || HII -> isConstExtended ( NvjMI ) ; if ( Good ) { if ( ResourceTracker -> canReserveResources ( NvjMI ) ) ResourceTracker -> reserveResources ( NvjMI ) ; else Good = false ; } if ( Good && ExtNvjMI ) Good = tryAllocateResourcesForConstExt ( true ) ; if ( ! Good ) { endPacket ( MBB , MI ) ; assert ( ResourceTracker -> canReserveResources ( MI ) ) ; ResourceTracker -> reserveResources ( MI ) ; if ( ExtMI ) { assert ( canReserveResourcesForConstExt ( ) ) ; tryAllocateResourcesForConstExt ( true ) ; } assert ( ResourceTracker -> canReserveResources ( NvjMI ) ) ; ResourceTracker -> reserveResources ( NvjMI ) ; if ( ExtNvjMI ) { assert ( canReserveResourcesForConstExt ( ) ) ; reserveResourcesForConstExt ( ) ; } } CurrentPacketMIs . push_back ( & MI ) ; CurrentPacketMIs . push_back ( & NvjMI ) ; return MII ; } ResourceTracker -> reserveResources ( MI ) ; if ( ExtMI && ! tryAllocateResourcesForConstExt ( true ) ) { endPacket ( MBB , MI ) ; if ( PromotedToDotNew ) demoteToDotOld ( MI ) ; if ( GlueAllocframeStore ) { useCalleesSP ( MI ) ; GlueAllocframeStore = false ; } ResourceTracker -> reserveResources ( MI ) ; reserveResourcesForConstExt ( ) ; } CurrentPacketMIs . push_back ( & MI ) ; return MII ;


<s> for ( auto & Pred : SUI -> Preds ) if ( Pred . getSUnit ( ) == SUJ ) if ( ( Pred . getLatency ( ) == <NUM_LIT> && Pred . isAssignedRegDep ( ) ) || HII -> isNewValueJump ( I ) || HII -> isToBeScheduledASAP ( * J , I ) ) return <NUM_LIT> ; } for ( auto J : OldPacketMIs ) { SUnit * SUJ = MIToSUnit [ J ] ;

for ( auto & Pred : SUI -> Preds ) if ( Pred . getSUnit ( ) == SUJ && Pred . getLatency ( ) > <NUM_LIT> ) return Pred . getLatency ( ) ;


<s> if ( MI . isInlineAsm ( ) ) return MJ . isInlineAsm ( ) || MJ . isBranch ( ) || MJ . isBarrier ( ) || MJ . isCall ( ) || MJ . isTerminator ( ) ; if ( HII . isNewValueStore ( MI ) && MJ . mayStore ( ) ) return true ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { unsigned TJ = HII . getType ( MJ ) ; if ( TJ != <STR_LIT> && TJ != <STR_LIT> && TJ != <STR_LIT> ) return true ; break ; }


<s> bool <ISA_LIT>PacketizerList :: canPromoteToDotNew ( const MachineInstr & MI , const SUnit * PacketSU , unsigned DepReg , MachineBasicBlock :: iterator & MII , const TargetRegisterClass * RC ) { if ( HII -> isDotNewInst ( MI ) && ! HII -> mayBeNewStore ( MI ) ) return false ; if ( ! isNewifiable ( MI , RC ) ) return false ; const MachineInstr & PI = * PacketSU -> getInstr ( ) ; if ( PI . isInlineAsm ( ) ) return false ; if ( PI . isImplicitDef ( ) ) return false ; if ( isImplicitDependency ( PI , true , DepReg ) || isImplicitDependency ( MI , false , DepReg ) ) return false ; const MCInstrDesc & MCID = PI . getDesc ( ) ; const TargetRegisterClass * VecRC = HII -> getRegClass ( MCID , <NUM_LIT> , HRI , MF ) ; if ( DisableVecDblNVStores && VecRC == & <STR_LIT> ) return false ; if ( RC == & <STR_LIT> ) return HII -> predCanBeUsedAsDotNew ( PI , DepReg ) ; if ( RC != & <STR_LIT> && ! HII -> mayBeNewStore ( MI ) ) return false ; int NewOpcode = ( RC != & <STR_LIT> ) ? HII -> getDotNewOp ( MI ) : HII -> getDotNewPredOp ( MI , MBPI ) ;

const MCInstrDesc & D = HII -> get ( NewOpcode ) ;


<s> if ( MI . isInlineAsm ( ) && ! ScheduleInlineAsm ) return true ; if ( isSchedBarrier ( MI ) ) return true ; if ( HII -> isSolo ( MI ) ) return true ; if ( MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ) return true ; if ( MI . getOpcode ( ) == <STR_LIT> ) return true ;


<s> if ( Latency == <NUM_LIT> ) return false ; if ( PacketStalls ) return Latency > PacketStallCycles ;


<s> if ( PacketStalls ) return Latency > PacketStallCycles ; return true ;


<s> if ( Minimal ) return false ; if ( producesStall ( MI ) ) return false ; auto & HST = MI . getParent ( ) -> getParent ( ) -> getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( HST . isTinyCoreWithDuplex ( ) && CurrentPacketMIs . size ( ) > <NUM_LIT> && ! PacketHasDuplex ) { for ( auto & MJ : CurrentPacketMIs ) PacketHasSLOT0OnlyInsn |= HII -> isPureSlot0 ( * MJ ) ; int Opcode = HII -> getDuplexOpcode ( MI , false ) ;


<s> for ( auto & MJ : CurrentPacketMIs ) PacketHasSLOT0OnlyInsn |= HII -> isPureSlot0 ( * MJ ) ; int Opcode = HII -> getDuplexOpcode ( MI , false ) ; if ( Opcode >= <NUM_LIT> ) { for ( auto & MJ : CurrentPacketMIs ) {

if ( HII -> isDuplexPair ( MI , * MJ ) && ! PacketHasSLOT0OnlyInsn ) {


<s> bool <ISA_LIT>PacketizerList :: tryAllocateResourcesForConstExt ( bool Reserve ) {

auto * ExtMI = MF . CreateMachineInstr ( HII -> get ( <STR_LIT> ) , DebugLoc ( ) ) ;


<s> } assert ( ( predRegClass == & <STR_LIT> ) && <STR_LIT> ) ; for ( auto & MO : MI . operands ( ) ) { if ( ! MO . isReg ( ) ) continue ; predRegNumDst = MO . getReg ( ) ; predRegClass = HRI -> getMinimalPhysRegClass ( predRegNumDst ) ; if ( predRegClass == & <STR_LIT> ) break ; } assert ( ( predRegClass == & <STR_LIT> ) && <STR_LIT> ) ; if ( predRegNumDst != predRegNumSrc || HII -> isDotNewInst ( PacketMI ) != HII -> isDotNewInst ( MI ) || getPredicateSense ( MI , HII ) != getPredicateSense ( PacketMI , HII ) ) return false ; } unsigned StartCheck = <NUM_LIT> ; for ( auto I : CurrentPacketMIs ) { SUnit * TempSU = MIToSUnit . find ( I ) -> second ; MachineInstr & TempMI = * TempSU -> getInstr ( ) ; if ( & TempMI != & PacketMI && ! StartCheck ) continue ; StartCheck = <NUM_LIT> ; if ( & TempMI == & PacketMI ) continue ; for ( auto & MO : MI . operands ( ) ) if ( MO . isReg ( ) && TempSU -> getInstr ( ) -> modifiesRegister ( MO . getReg ( ) , HRI ) ) return false ; } if ( ! HII -> isPostIncrement ( MI ) ) { for ( unsigned opNum = <NUM_LIT> ; opNum < MI . getNumOperands ( ) - <NUM_LIT> ; opNum ++ ) { const MachineOperand & MO = MI . getOperand ( opNum ) ; if ( MO . isReg ( ) && MO . getReg ( ) == DepReg ) return false ; } } for ( auto & MO : PacketMI . operands ( ) ) { if ( MO . isRegMask ( ) && MO . clobbersPhysReg ( DepReg ) ) return false ; if ( ! MO . isReg ( ) || ! MO . isDef ( ) || ! MO . isImplicit ( ) ) continue ;


<s> LLVM_DEBUG ( { if ( ! CurrentPacketMIs . empty ( ) ) { dbgs ( ) << <STR_LIT> ;


<s> unsigned R = ResourceTracker -> getUsedResources ( Idx ++ ) ;

dbgs ( ) << <STR_LIT> << utohexstr ( R ) << <STR_LIT> << * MI ;


<s> for ( auto & MO : J . operands ( ) ) { if ( ! MO . isReg ( ) || ! MO . isDef ( ) || ! MO . isDead ( ) ) continue ; Register R = MO . getReg ( ) ; if ( R != <STR_LIT> && DeadDefs [ R ] ) return true ;


<s> if ( ! MO . isReg ( ) || ! MO . isDef ( ) || ! MO . isDead ( ) ) continue ; DeadDefs [ MO . getReg ( ) ] = true ; } for ( auto & MO : J . operands ( ) ) { if ( ! MO . isReg ( ) || ! MO . isDef ( ) || ! MO . isDead ( ) ) continue ; Register R = MO . getReg ( ) ; if ( R != <STR_LIT> && DeadDefs [ R ] ) return true ;


<s> if ( ! MO . isReg ( ) || ! MO . isDef ( ) ) continue ; Register R = MO . getReg ( ) ; if ( SecondI . readsRegister ( R , TRI ) ) return true ; } return false ;


<s> Register R = MO . getReg ( ) ; if ( SecondI . readsRegister ( R , TRI ) ) return true ; } return false ;


<s> HRI = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getRegisterInfo ( ) ; addMutation ( std :: make_unique < <ISA_LIT>Subtarget :: UsrOverflowMutation > ( ) ) ; addMutation ( std :: make_unique < <ISA_LIT>Subtarget :: HVXMemLatencyMutation > ( ) ) ; addMutation ( std :: make_unique < <ISA_LIT>Subtarget :: BankConflictMutation > ( ) ) ;


<s> bool <ISA_LIT>PacketizerList :: ignorePseudoInstruction ( const MachineInstr & MI , const MachineBasicBlock * ) { if ( MI . isDebugInstr ( ) ) return true ; if ( MI . isCFIInstruction ( ) ) return false ; if ( MI . isInlineAsm ( ) ) return false ;

if ( MI . isImplicitDef ( ) ) return false ;


<s> bool <ISA_LIT>PacketizerList :: ignorePseudoInstruction ( const MachineInstr & MI , const MachineBasicBlock * ) { if ( MI . isDebugInstr ( ) ) return true ; if ( MI . isCFIInstruction ( ) ) return false ; if ( MI . isInlineAsm ( ) ) return false ; if ( MI . isImplicitDef ( ) ) return false ; const MCInstrDesc & TID = MI . getDesc ( ) ; auto * IS = ResourceTracker -> getInstrItins ( ) -> beginStage ( TID . getSchedClass ( ) ) ;


<s> if ( HII -> isDeallocRet ( MI ) ) if ( DepReg == HRI -> getFrameRegister ( ) || DepReg == HRI -> getStackRegister ( ) ) return true ; if ( DepType == SDep :: Data ) { for ( const MachineOperand & MO : MI . operands ( ) ) if ( MO . isReg ( ) && MO . getReg ( ) == DepReg && ! MO . isImplicit ( ) ) return true ; }


<s> bool <ISA_LIT>PacketizerList :: isCallDependent ( const MachineInstr & MI , SDep :: Kind DepType , unsigned DepReg ) { if ( DepReg == HRI -> getRARegister ( ) ) return true ; if ( HII -> isDeallocRet ( MI ) ) if ( DepReg == HRI -> getFrameRegister ( ) || DepReg == HRI -> getStackRegister ( ) ) return true ; if ( DepType == SDep :: Data ) { for ( const MachineOperand & MO : MI . operands ( ) ) if ( MO . isReg ( ) && MO . getReg ( ) == DepReg && ! MO . isImplicit ( ) ) return true ;


<s> for ( auto J : CurrentPacketMIs ) { noSlot1Store |= HII -> isRestrictNoSlot1Store ( * J ) ; isSlot0Only |= HII -> isPureSlot0 ( * J ) ;


<s> if ( MI . mayStore ( ) && isPureSlot0InsnWithNoSlot1Store ( MI ) ) return false ; if ( producesStall ( MI ) ) return false ; auto & HST = MI . getParent ( ) -> getParent ( ) -> getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( HST . isTinyCoreWithDuplex ( ) && CurrentPacketMIs . size ( ) > <NUM_LIT> && ! PacketHasDuplex ) { for ( auto & MJ : CurrentPacketMIs ) PacketHasSLOT0OnlyInsn |= HII -> isPureSlot0 ( * MJ ) ; int Opcode = HII -> getDuplexOpcode ( MI , false ) ; if ( Opcode >= <NUM_LIT> ) {


<s> if ( Reg != MJ . getOperand ( BPJ ) . getReg ( ) ) return false ; for ( const auto & PI : SUI -> Preds ) if ( PI . getKind ( ) != SDep :: Anti && ( PI . getKind ( ) != SDep :: Data || PI . getReg ( ) != Reg ) ) return false ; int Incr ;

if ( ! HII -> getIncrementValue ( MJ , Incr ) ) return false ;


<s> assert ( ( predRegClass == & <STR_LIT> ) && ( <STR_LIT> ) ) ; for ( unsigned opNum = <NUM_LIT> ; opNum < MI -> getNumOperands ( ) ; opNum ++ ) { if ( MI -> getOperand ( opNum ) . isReg ( ) ) predRegNumDst = MI -> getOperand ( opNum ) . getReg ( ) ; predRegClass = QRI -> getMinimalPhysRegClass ( predRegNumDst ) ; if ( predRegClass == & <STR_LIT> ) { break ; } } assert ( ( predRegClass == & <STR_LIT> ) && ( <STR_LIT> ) ) ; if ( ( predRegNumDst != predRegNumSrc ) || QII -> isDotNewInst ( PacketMI ) != QII -> isDotNewInst ( MI ) || getPredicateSense ( MI , QII ) != getPredicateSense ( PacketMI , QII ) ) { return false ; } } std :: vector < MachineInstr * > :: iterator VI ; std :: vector < MachineInstr * > :: iterator VE ; unsigned StartCheck = <NUM_LIT> ; for ( VI = CurrentPacketMIs . begin ( ) , VE = CurrentPacketMIs . end ( ) ; ( VI != VE ) ; ++ VI ) { SUnit * TempSU = MIToSUnit [ * VI ] ; MachineInstr * TempMI = TempSU -> getInstr ( ) ; if ( TempMI != PacketMI && ! StartCheck ) continue ; StartCheck = <NUM_LIT> ; if ( TempMI == PacketMI ) continue ; for ( unsigned opNum = <NUM_LIT> ; opNum < MI -> getNumOperands ( ) ; opNum ++ ) { if ( MI -> getOperand ( opNum ) . isReg ( ) && TempSU -> getInstr ( ) -> modifiesRegister ( MI -> getOperand ( opNum ) . getReg ( ) , QRI ) ) return false ; } } if ( ! QII -> isPostIncrement ( MI ) && GetStoreValueOperand ( MI ) . isReg ( ) && GetStoreValueOperand ( MI ) . getReg ( ) == DepReg ) { for ( unsigned opNum = <NUM_LIT> ; opNum < MI -> getNumOperands ( ) - <NUM_LIT> ; opNum ++ ) { if ( MI -> getOperand ( opNum ) . isReg ( ) && MI -> getOperand ( opNum ) . getReg ( ) == DepReg ) { return false ; } } for ( unsigned opNum = <NUM_LIT> ; opNum < PacketMI -> getNumOperands ( ) ; opNum ++ ) { if ( PacketMI -> getOperand ( opNum ) . isReg ( ) && PacketMI -> getOperand ( opNum ) . getReg ( ) == DepReg && PacketMI -> getOperand ( opNum ) . isDef ( ) && PacketMI -> getOperand ( opNum ) . isImplicit ( ) ) {


<s> bool MinOnly = Minimal || DisablePacketizer || ! HST . usePackets ( ) || skipFunction ( MF . getFunction ( ) ) ; <ISA_LIT>PacketizerList Packetizer ( MF , MLI , AA , MBPI , MinOnly ) ; assert ( Packetizer . getResourceTracker ( ) && <STR_LIT> ) ; for ( MachineBasicBlock & MB : MF ) { auto End = MB . end ( ) ; auto MI = MB . begin ( ) ; while ( MI != End ) { auto NextI = std :: next ( MI ) ; if ( MI -> isKill ( ) ) { MB . erase ( MI ) ; End = MB . end ( ) ; } MI = NextI ; } } if ( HST . isTinyCoreWithDuplex ( ) ) HII -> translateInstrsForDup ( MF , true ) ; for ( auto & MB : MF ) { auto Begin = MB . begin ( ) , End = MB . end ( ) ; while ( Begin != End ) { MachineBasicBlock :: iterator RB = Begin ;

while ( RB != End && HII -> isSchedulingBoundary ( * RB , & MB , MF ) ) ++ RB ;


<s> MachineBasicBlock * MBB = MI . getParent ( ) ; if ( MI . isImplicitDef ( ) ) { unsigned R = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( <STR_LIT> . contains ( R ) ) { MCSuperRegIterator S ( R , HRI , false ) ; MI . addOperand ( MachineOperand :: CreateReg ( * S , true , true ) ) ; } return MII ; } assert ( ResourceTracker -> canReserveResources ( MI ) ) ; bool ExtMI = HII -> isExtended ( MI ) || HII -> isConstExtended ( MI ) ; bool Good = true ; if ( GlueToNewValueJump ) { MachineInstr & NvjMI = * ++ MII ; ResourceTracker -> reserveResources ( MI ) ; if ( ExtMI ) Good = tryAllocateResourcesForConstExt ( true ) ; bool ExtNvjMI = HII -> isExtended ( NvjMI ) || HII -> isConstExtended ( NvjMI ) ; if ( Good ) { if ( ResourceTracker -> canReserveResources ( NvjMI ) ) ResourceTracker -> reserveResources ( NvjMI ) ; else Good = false ; } if ( Good && ExtNvjMI ) Good = tryAllocateResourcesForConstExt ( true ) ; if ( ! Good ) { endPacket ( MBB , MI ) ; assert ( ResourceTracker -> canReserveResources ( MI ) ) ; ResourceTracker -> reserveResources ( MI ) ; if ( ExtMI ) { assert ( canReserveResourcesForConstExt ( ) ) ; tryAllocateResourcesForConstExt ( true ) ; } assert ( ResourceTracker -> canReserveResources ( NvjMI ) ) ; ResourceTracker -> reserveResources ( NvjMI ) ;


<s> if ( MI . isInlineAsm ( ) ) return MJ . isInlineAsm ( ) || MJ . isBranch ( ) || MJ . isBarrier ( ) || MJ . isCall ( ) || MJ . isTerminator ( ) ; return false ;


<s> const MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; if ( MF -> getSubtarget < <ISA_LIT>Subtarget > ( ) . hasV60TOpsOnly ( ) && HII . isHVXMemWithAIndirect ( MI , MJ ) ) return true ; if ( MI . isInlineAsm ( ) ) return MJ . isInlineAsm ( ) || MJ . isBranch ( ) || MJ . isBarrier ( ) || MJ . isCall ( ) || MJ . isTerminator ( ) ;


<s> if ( HII -> isDotNewInst ( MI ) && ! HII -> mayBeNewStore ( MI ) ) return false ; if ( ! isNewifiable ( MI , RC ) ) return false ; const MachineInstr & PI = * PacketSU -> getInstr ( ) ; if ( PI . isInlineAsm ( ) ) return false ; if ( PI . isImplicitDef ( ) ) return false ; if ( isImplicitDependency ( PI , DepReg ) ) return false ; const MCInstrDesc & MCID = PI . getDesc ( ) ; const TargetRegisterClass * VecRC = HII -> getRegClass ( MCID , <NUM_LIT> , HRI , MF ) ; if ( DisableVecDblNVStores && VecRC == & <STR_LIT> ) return false ; if ( RC == & <STR_LIT> ) if ( HII -> isCondInst ( MI ) || MI . isReturn ( ) ) return HII -> predCanBeUsedAsDotNew ( PI , DepReg ) ; if ( RC != & <STR_LIT> && ! HII -> mayBeNewStore ( MI ) ) return false ; int NewOpcode = HII -> getDotNewOp ( MI ) ; const MCInstrDesc & D = HII -> get ( NewOpcode ) ; MachineInstr * NewMI = MF . CreateMachineInstr ( D , DebugLoc ( ) ) ;

bool ResourcesAvailable = ResourceTracker -> canReserveResources ( * NewMI ) ;


<s> if ( MI ) { for ( auto & MO : BI -> operands ( ) ) if ( MO . isReg ( ) && MO . getReg ( ) == MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) return ;