<s> TargetRegistry :: RegisterMCSubtargetInfo ( * T , <ISA_LIT>_MC :: create<ISA_LIT>MCSubtargetInfo ) ; TargetRegistry :: RegisterMCInstrAnalysis ( * T , create<ISA_LIT>MCInstrAnalysis ) ; TargetRegistry :: RegisterELFStreamer ( * T , createELFStreamer ) ; TargetRegistry :: RegisterCOFFStreamer ( * T , create<ISA_LIT>WinCOFFStreamer ) ; TargetRegistry :: RegisterMachOStreamer ( * T , create<ISA_LIT>MachOStreamer ) ; TargetRegistry :: RegisterObjectTargetStreamer ( * T , create<ISA_LIT>ObjectTargetStreamer ) ; TargetRegistry :: RegisterAsmTargetStreamer ( * T , create<ISA_LIT>TargetAsmStreamer ) ; TargetRegistry :: RegisterNullTargetStreamer ( * T , create<ISA_LIT>NullTargetStreamer ) ; TargetRegistry :: RegisterMCInstPrinter ( * T , create<ISA_LIT>MCInstPrinter ) ; TargetRegistry :: RegisterMCRelocationInfo ( * T , create<ISA_LIT>MCRelocationInfo ) ; } for ( Target * T : { & getThe<ISA_LIT>LETarget ( ) , & getTheThumbLETarget ( ) } ) TargetRegistry :: RegisterMCCodeEmitter ( * T , create<ISA_LIT>LEMCCodeEmitter ) ; for ( Target * T : { & getThe<ISA_LIT>BETarget ( ) , & getTheThumbBETarget ( ) } ) TargetRegistry :: RegisterMCCodeEmitter ( * T , create<ISA_LIT>BEMCCodeEmitter ) ; TargetRegistry :: RegisterMCAsmBackend ( getThe<ISA_LIT>LETarget ( ) , create<ISA_LIT>LEAsmBackend ) ; TargetRegistry :: RegisterMCAsmBackend ( getThe<ISA_LIT>BETarget ( ) , create<ISA_LIT>BEAsmBackend ) ; TargetRegistry :: RegisterMCAsmBackend ( getTheThumbLETarget ( ) ,


<s> TargetRegistry :: RegisterCOFFStreamer ( * T , create<ISA_LIT>WinCOFFStreamer ) ; TargetRegistry :: RegisterMachOStreamer ( * T , create<ISA_LIT>MachOStreamer ) ; TargetRegistry :: RegisterObjectTargetStreamer ( * T , create<ISA_LIT>ObjectTargetStreamer ) ; TargetRegistry :: RegisterAsmTargetStreamer ( * T , create<ISA_LIT>TargetAsmStreamer ) ; TargetRegistry :: RegisterNullTargetStreamer ( * T , create<ISA_LIT>NullTargetStreamer ) ; TargetRegistry :: RegisterMCInstPrinter ( * T , create<ISA_LIT>MCInstPrinter ) ; TargetRegistry :: RegisterMCRelocationInfo ( * T , create<ISA_LIT>MCRelocationInfo ) ; } for ( Target * T : { & getThe<ISA_LIT>LETarget ( ) , & getTheThumbLETarget ( ) } ) TargetRegistry :: RegisterMCCodeEmitter ( * T , create<ISA_LIT>LEMCCodeEmitter ) ; for ( Target * T : { & getThe<ISA_LIT>BETarget ( ) , & getTheThumbBETarget ( ) } ) TargetRegistry :: RegisterMCCodeEmitter ( * T , create<ISA_LIT>BEMCCodeEmitter ) ; TargetRegistry :: RegisterMCAsmBackend ( getThe<ISA_LIT>LETarget ( ) , create<ISA_LIT>LEAsmBackend ) ; TargetRegistry :: RegisterMCAsmBackend ( getThe<ISA_LIT>BETarget ( ) , create<ISA_LIT>BEAsmBackend ) ; TargetRegistry :: RegisterMCAsmBackend ( getTheThumbLETarget


<s> else MAI = new <ISA_LIT>ELFMCAsmInfo ( TheTriple ) ; unsigned Reg = MRI . getDwarfRegNum ( <STR_LIT> , true ) ; MAI -> addInitialFrameState ( MCCFIInstruction :: createDefCfa ( nullptr , Reg


<s> MCAsmInfo * MAI ; if ( TheTriple . isOSDarwin ( ) || TheTriple . isOSBinFormatMachO ( ) ) MAI = new <ISA_LIT>MCAsmInfoDarwin ( TheTriple ) ; else if ( TheTriple . isWindowsItaniumEnvironment ( ) ) MAI = new <ISA_LIT>COFFMCAsmInfoGNU ( ) ; else if ( TheTriple . isWindowsMSVCEnvironment ( ) ) MAI = new <ISA_LIT>COFFMCAsmInfoMicrosoft ( ) ; else MAI = new <ISA_LIT>ELFMCAsmInfo ( TheTriple ) ; unsigned Reg = MRI . getDwarfRegNum


<s> static MCCodeGenInfo * create<ISA_LIT>MCCodeGenInfo ( StringRef TT , Reloc :: Model RM , CodeModel :: Model CM , CodeGenOpt :: Level OL ) { MCCodeGenInfo * X = new MCCodeGenInfo ( ) ; if ( RM == Reloc :: Default ) { Triple TheTriple ( TT ) ; RM = TheTriple . isOSDarwin ( ) ? Reloc :: PIC_ : Reloc :: DynamicNoPIC ; } X -> initMCCodeGenInfo ( RM ,


<s> assert ( ! STI . getFeatureBits ( ) [ llvm :: <STR_LIT> ] && <STR_LIT> ) ; assert ( MI . getNumOperands ( ) >= <NUM_LIT> && <STR_LIT> ) ; bool ListContainsPC = false , ListContainsLR = false ; for ( unsigned OI = <NUM_LIT> , OE = MI . getNumOperands ( ) ; OI < OE ; ++ OI ) { assert ( MI . getOperand ( OI ) . isReg (


<s> static bool get<ISA_LIT>LoadDeprecationInfo ( MCInst & MI , MCSubtargetInfo & STI , std :: string & Info ) { assert ( ! STI . getFeatureBits ( ) [ llvm :: <STR_LIT> ] && <STR_LIT> ) ; assert ( MI . getNumOperands ( ) >= <NUM_LIT> && <STR_LIT> ) ; bool ListContainsPC = false , ListContainsLR = false ; for ( unsigned OI = <NUM_LIT> , OE = MI . getNumOperands ( ) ; OI < OE ; ++ OI ) { assert ( MI . getOperand ( OI )


<s> static bool getMCRDeprecationInfo ( MCInst & MI , MCSubtargetInfo & STI , std :: string & Info ) { if ( STI . getFeatureBits ( ) [ llvm :: <STR_LIT> ] && ( MI . getOperand ( <NUM_LIT> ) . isImm ( ) && MI . getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> ) && ( MI . getOperand ( <NUM_LIT> ) . isImm ( ) && MI . getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> ) && ( MI . getOperand ( <NUM_LIT> ) . isImm ( ) && MI . getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> ) ) { if ( ( MI . getOperand ( <NUM_LIT> ) . isImm ( ) && MI . getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> ) ) { if ( MI . getOperand ( <NUM_LIT> ) . isImm ( ) && MI . getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> ) { Info = <STR_LIT> ; return true ; } if ( MI . getOperand ( <NUM_LIT> ) . isImm ( ) && MI . getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> ) { Info = <STR_LIT> ; return true ; } } if ( MI . getOperand ( <NUM_LIT> ) . isImm ( ) && MI . getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> && ( MI . getOperand ( <NUM_LIT> ) . isImm ( ) && MI . getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> ) ) { Info =


<s> Triple triple ( TT ) ; bool isThumb = triple . getArch ( ) == Triple :: thumb || triple . getArch ( ) == Triple :: thumbeb ; bool NoCPU = CPU == <STR_LIT> || CPU . empty ( ) ; std :: string <ISA_LIT>ArchFeature ; switch ( triple . getSubArch ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case Triple :: <ISA_LIT>SubArch_v8 : if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v8_1a : if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v7m : isThumb = true ; if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v7em : if ( NoCPU )


<s> else <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v8_1a : if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v7m : isThumb = true ; if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v7em : if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v7s : if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v7 : if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v6t2 : <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v6k : <ISA_LIT>ArchFeature =


<s> TargetRegistry :: RegisterMCSubtargetInfo ( * T , <ISA_LIT>_MC :: create<ISA_LIT>MCSubtargetInfo ) ; TargetRegistry :: RegisterMCInstrAnalysis ( * T , create<ISA_LIT>MCInstrAnalysis ) ; TargetRegistry :: RegisterELFStreamer ( * T , createELFStreamer ) ; TargetRegistry :: RegisterCOFFStreamer ( * T , create<ISA_LIT>WinCOFFStreamer ) ; TargetRegistry :: RegisterMachOStreamer ( * T , create<ISA_LIT>MachOStreamer ) ; TargetRegistry :: RegisterObjectTargetStreamer ( * T , create<ISA_LIT>ObjectTargetStreamer ) ; TargetRegistry :: RegisterAsmTargetStreamer ( * T , create<ISA_LIT>TargetAsmStreamer ) ; TargetRegistry :: RegisterNullTargetStreamer ( * T , create<ISA_LIT>NullTargetStreamer ) ; TargetRegistry :: RegisterMCInstPrinter ( * T


<s> TargetRegistry :: RegisterMCInstPrinter ( * T , create<ISA_LIT>MCInstPrinter ) ; TargetRegistry :: RegisterMCRelocationInfo ( * T , create<ISA_LIT>MCRelocationInfo ) ; } for ( Target * T : { & getThe<ISA_LIT>LETarget ( ) , & getTheThumbLETarget ( ) } ) TargetRegistry :: RegisterMCCodeEmitter ( * T , create<ISA_LIT>LEMCCodeEmitter ) ; for ( Target * T : { & getThe<ISA_LIT>BETarget ( ) , & getTheThumbBETarget ( ) } ) TargetRegistry :: RegisterMCCodeEmitter ( * T , create<ISA_LIT>BEMCCodeEmitter ) ; TargetRegistry :: RegisterMCAsmBackend ( getThe<ISA_LIT>LETarget ( ) , create<ISA_LIT>LEAsmBackend ) ; TargetRegistry :: RegisterMCAsmBackend ( getThe<ISA_LIT>BETarget ( ) , create<ISA_LIT>BEAsmBackend ) ; TargetRegistry :: RegisterMCAsmBackend ( getTheThumbLETarget ( ) ,


<s> void LLVMInitialize<ISA_LIT>TargetMC ( ) { for ( Target * T : { & The<ISA_LIT>LETarget , & The<ISA_LIT>BETarget , & TheThumbLETarget , & TheThumbBETarget } ) { RegisterMCAsmInfoFn X ( * T , create<ISA_LIT>MCAsmInfo ) ; TargetRegistry :: RegisterMCInstrInfo ( * T , create<ISA_LIT>MCInstrInfo ) ; TargetRegistry :: RegisterMCRegInfo ( * T , create<ISA_LIT>MCRegisterInfo ) ; TargetRegistry :: RegisterMCSubtargetInfo ( * T , <ISA_LIT>_MC :: create<ISA_LIT>MCSubtargetInfo ) ; TargetRegistry :: RegisterMCInstrAnalysis ( * T , create<ISA_LIT>MCInstrAnalysis ) ; TargetRegistry :: RegisterELFStreamer ( * T , createELFStreamer ) ; TargetRegistry :: RegisterCOFFStreamer ( * T , create<ISA_LIT>WinCOFFStreamer ) ; TargetRegistry :: RegisterMachOStreamer ( * T , create<ISA_LIT>MachOStreamer ) ; TargetRegistry :: RegisterObjectTargetStreamer ( * T , create<ISA_LIT>ObjectTargetStreamer ) ; TargetRegistry :: RegisterAsmTargetStreamer ( * T , create<ISA_LIT>TargetAsmStreamer ) ; TargetRegistry :: RegisterNullTargetStreamer ( * T , create<ISA_LIT>NullTargetStreamer ) ; TargetRegistry :: RegisterMCInstPrinter ( * T , create<ISA_LIT>MCInstPrinter ) ; TargetRegistry :: RegisterMCRelocationInfo ( * T , create<ISA_LIT>MCRelocationInfo ) ; } for ( Target * T : { & The<ISA_LIT>LETarget , & TheThumbLETarget } ) TargetRegistry :: RegisterMCCodeEmitter ( * T , create<ISA_LIT>LEMCCodeEmitter ) ; for ( Target * T : { & The<ISA_LIT>BETarget , & TheThumbBETarget } ) TargetRegistry :: RegisterMCCodeEmitter ( * T


<s> Triple triple ( TT ) ; bool isThumb = triple . getArch ( ) == Triple :: thumb || triple . getArch ( ) == Triple :: thumbeb ; bool NoCPU = CPU == <STR_LIT> || CPU . empty ( ) ; std :: string <ISA_LIT>ArchFeature ; switch ( triple . getSubArch ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case Triple :: <ISA_LIT>SubArch_v8 : if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v7m : isThumb = true ; if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v7em : if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v7s : if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v7 : if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v6t2 : <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v6m : isThumb = true ; if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v6 : <ISA_LIT>ArchFeature = <STR_LIT>


<s> } bool NoCPU = CPU == <STR_LIT> || CPU . empty ( ) ; std :: string <ISA_LIT>ArchFeature ; if ( Idx ) { unsigned SubVer = TT [ Idx ] ; if ( SubVer == '8' ) { if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; } else if ( SubVer == '7' ) { if ( Len >= Idx + <NUM_LIT> && TT [ Idx + <NUM_LIT> ] == 'm' ) { isThumb = true ; if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; } else if ( Len >= Idx + <NUM_LIT> && TT [ Idx + <NUM_LIT> ] == 'e' && TT [ Idx + <NUM_LIT> ] == 'm' ) { if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; } else if ( Len >= Idx + <NUM_LIT> && TT [ Idx + <NUM_LIT> ] == 's' ) { if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; }


<s> switch ( TheTriple . getObjectFormat ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case Triple :: MachO : { MCStreamer * S = createMachOStreamer ( Ctx , MAB , OS , Emitter , false ) ; new <ISA_LIT>TargetStreamer ( * S ) ; return S ; } case Triple :: COFF : assert ( TheTriple . isOSWindows ( ) && <STR_LIT> ) ; return create<ISA_LIT>WinCOFFStreamer ( Ctx , MAB , * Emitter , OS ) ; case Triple :: ELF : return create<ISA_LIT>ELFStreamer ( Ctx , MAB , OS , Emitter , false , TheTriple . getArch ( ) ==


<s> if ( TheTriple . isOSBinFormatMachO ( ) ) MAI = new <ISA_LIT>MCAsmInfoDarwin ( ) ; else MAI = new <ISA_LIT>ELFMCAsmInfo ( ) ; unsigned Reg = MRI . getDwarfRegNum (


<s> RegisterMCAsmInfoFn A ( The<ISA_LIT>Target , create<ISA_LIT>MCAsmInfo ) ; RegisterMCAsmInfoFn B ( TheThumbTarget , create<ISA_LIT>MCAsmInfo ) ; TargetRegistry :: RegisterMCCodeGenInfo ( The<ISA_LIT>Target , create<ISA_LIT>MCCodeGenInfo ) ; TargetRegistry :: RegisterMCCodeGenInfo ( TheThumbTarget , create<ISA_LIT>MCCodeGenInfo ) ; TargetRegistry :: RegisterMCInstrInfo ( The<ISA_LIT>Target , create<ISA_LIT>MCInstrInfo ) ; TargetRegistry :: RegisterMCInstrInfo ( TheThumbTarget , create<ISA_LIT>MCInstrInfo ) ; TargetRegistry :: RegisterMCRegInfo ( The<ISA_LIT>Target , create<ISA_LIT>MCRegisterInfo ) ; TargetRegistry :: RegisterMCRegInfo ( TheThumbTarget , create<ISA_LIT>MCRegisterInfo ) ; TargetRegistry :: RegisterMCSubtargetInfo ( The<ISA_LIT>Target , <ISA_LIT>_MC :: create<ISA_LIT>MCSubtargetInfo ) ; TargetRegistry :: RegisterMCSubtargetInfo ( TheThumbTarget , <ISA_LIT>_MC :: create<ISA_LIT>MCSubtargetInfo ) ; TargetRegistry :: RegisterMCInstrAnalysis ( The<ISA_LIT>Target , create<ISA_LIT>MCInstrAnalysis ) ; TargetRegistry :: RegisterMCInstrAnalysis ( TheThumbTarget


<s> if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; } else { if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; } } else if ( SubVer == '6' ) { if ( Len >= Idx + <NUM_LIT> && TT [ Idx + <NUM_LIT> ] == 't' && TT [ Idx + <NUM_LIT> ] == '2' ) <ISA_LIT>ArchFeature = <STR_LIT> ; else if ( Len >= Idx + <NUM_LIT> && TT [ Idx + <NUM_LIT> ] == 'm' ) { isThumb = true ; if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; } else <ISA_LIT>ArchFeature = <STR_LIT> ; } else if ( SubVer == '5' ) { if ( Len >= Idx + <NUM_LIT> && TT [ Idx + <NUM_LIT> ] == 't' && TT [ Idx + <NUM_LIT> ] == 'e' ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; } else if ( SubVer == '4' && Len >= Idx + <NUM_LIT> && TT [ Idx + <NUM_LIT> ] == 't' ) <ISA_LIT>ArchFeature = <STR_LIT> ; } if ( isThumb ) { if ( <ISA_LIT>ArchFeature . empty ( ) ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature += <STR_LIT> ; } if ( triple . isOSNaCl ( ) ) { if ( <ISA_LIT>ArchFeature . empty ( ) )


<s> if ( TheTriple . isOSDarwin ( ) ) return new <ISA_LIT>MCAsmInfoDarwin ( ) ; <ISA_LIT>ELFMCAsmInfo * MAI = new <ISA_LIT>ELFMCAsmInfo ( ) ; if ( TheTriple . isOSNaCl


<s> auto * Exp = new <STR_LIT> ( S . getContext ( ) , std :: move ( RegInfo ) , std :: move ( InstInfo ) ) ; S . setNaClExpander


<s> TargetRegistry :: RegisterMCRelocationInfo ( * T , create<ISA_LIT>MCRelocationInfo ) ; } for ( Target * T : { & The<ISA_LIT>LETarget , & TheThumbLETarget } ) TargetRegistry :: RegisterMCCodeEmitter ( * T , create<ISA_LIT>LEMCCodeEmitter ) ; for ( Target * T : { & The<ISA_LIT>BETarget , & TheThumbBETarget } ) TargetRegistry :: RegisterMCCodeEmitter ( * T , create<ISA_LIT>BEMCCodeEmitter ) ; TargetRegistry :: RegisterMCAsmBackend ( The<ISA_LIT>LETarget , create<ISA_LIT>LEAsmBackend ) ; TargetRegistry :: RegisterMCAsmBackend ( The<ISA_LIT>BETarget , create<ISA_LIT>BEAsmBackend ) ; TargetRegistry :: RegisterMCAsmBackend ( TheThumbLETarget ,


<s> TargetRegistry :: RegisterMCSubtargetInfo ( * T , <ISA_LIT>_MC :: create<ISA_LIT>MCSubtargetInfo ) ; TargetRegistry :: RegisterMCInstrAnalysis ( * T , create<ISA_LIT>MCInstrAnalysis ) ; TargetRegistry :: RegisterELFStreamer ( * T , createELFStreamer ) ; TargetRegistry :: RegisterCOFFStreamer ( * T , create<ISA_LIT>WinCOFFStreamer ) ; TargetRegistry :: RegisterMachOStreamer ( * T , create<ISA_LIT>MachOStreamer ) ; TargetRegistry :: RegisterMCNaClExpander ( * T , create<ISA_LIT>MCNaClExpander ) ; TargetRegistry :: RegisterObjectTargetStreamer ( * T , create<ISA_LIT>ObjectTargetStreamer ) ; TargetRegistry :: RegisterAsmTargetStreamer ( * T , create<ISA_LIT>TargetAsmStreamer ) ; TargetRegistry :: RegisterNullTargetStreamer ( * T , create<ISA_LIT>NullTargetStreamer ) ; TargetRegistry :: RegisterMCInstPrinter ( * T , create<ISA_LIT>MCInstPrinter ) ; TargetRegistry :: RegisterMCRelocationInfo ( * T , create<ISA_LIT>MCRelocationInfo ) ; } for ( Target * T : { & The<ISA_LIT>LETarget , & TheThumbLETarget } ) TargetRegistry :: RegisterMCCodeEmitter ( * T , create<ISA_LIT>LEMCCodeEmitter ) ; for ( Target * T : { & The<ISA_LIT>BETarget , & TheThumbBETarget } ) TargetRegistry :: RegisterMCCodeEmitter ( * T , create<ISA_LIT>BEMCCodeEmitter ) ; TargetRegistry :: RegisterMCAsmBackend ( The<ISA_LIT>LETarget , create<ISA_LIT>LEAsmBackend ) ; TargetRegistry :: RegisterMCAsmBackend ( The<ISA_LIT>BETarget


<s> if ( ArchID != <STR_LIT> && ( CPU . empty ( ) || CPU == <STR_LIT> ) ) <ISA_LIT>ArchFeature = ( <ISA_LIT>ArchFeature + <STR_LIT> + <STR_LIT> ( ArchID ) ) . str ( ) ; if ( isThumb ) { if ( <ISA_LIT>ArchFeature . empty ( ) ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature += <STR_LIT> ; } if ( TT . isOSNaCl ( ) ) { if ( <ISA_LIT>ArchFeature . empty ( ) ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature += <STR_LIT>


<s> static MCCodeGenInfo * create<ISA_LIT>MCCodeGenInfo ( const Triple & TT , Reloc :: Model RM , CodeModel :: Model CM , CodeGenOpt :: Level OL ) { MCCodeGenInfo * X = new MCCodeGenInfo ( ) ; X -> initMCCodeGenInfo ( RM , CM , OL


<s> static MCInstPrinter * create<ISA_LIT>MCInstPrinter ( const Target & T , unsigned SyntaxVariant , const MCAsmInfo & MAI , const MCRegisterInfo & MRI , const MCSubtargetInfo & STI ) { if ( SyntaxVariant == <NUM_LIT> ) return new <ISA_LIT>InstPrinter ( MAI , MRI , STI ) ; return


<s> static MCInstPrinter * create<ISA_LIT>MCInstPrinter ( const Target & T , unsigned SyntaxVariant , const MCAsmInfo & MAI , const MCRegisterInfo & MRI , const


<s> else <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v7s : if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v7 : if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v6t2 : <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v6k : <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v6m : isThumb = true ; if ( NoCPU ) <ISA_LIT>ArchFeature = <STR_LIT> ; else <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v6 : <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v5te : <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v5 : <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: <ISA_LIT>SubArch_v4t : <ISA_LIT>ArchFeature = <STR_LIT> ; break ; case Triple :: NoSubArch : break ; } if ( isThumb ) { if ( <ISA_LIT>ArchFeature . empty ( ) ) <ISA_LIT>ArchFeature =


<s> virtual const char * getPassName ( )


<s> switch ( MI . getOpcode ( ) ) { default : return


<s> StringRef getPassName ( ) const override { return <STR_LIT>


<s> if ( skipFunction ( MF . getFunction ( ) ) ) return false ; std :: vector < MachineInstr * > ToRemove ; int64_t DMBType = - <NUM_LIT> ; for ( auto & MBB : MF ) { bool IsRemovableNextDMB = false ; for ( auto & MI : MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) { if ( IsRemovableNextDMB ) { if ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) == DMBType ) { ToRemove . push_back ( & MI ) ; } else


<s> std :: vector < MachineInstr * > ToRemove ; int64_t DMBType = - <NUM_LIT> ; for ( auto & MBB : MF ) { bool IsRemovableNextDMB = false ; for ( auto & MI : MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) { if ( IsRemovableNextDMB ) { if ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) == DMBType ) { ToRemove . push_back ( & MI ) ; } else { DMBType = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; } } else


<s> return ! ( MI -> mayLoad ( ) || MI -> mayStore ( ) || MI -> hasUnmodeledSideEffects ( ) ||

MI -> isCall ( ) || MI -> isReturn ( ) ) ;


<s> const char * getPassName ( ) const override { return <STR_LIT>


<s> explicit <ISA_LIT>PagerandoOptimizer ( ) : MachineFunctionPass


<s> void <ISA_LIT>PagerandoOptimizer :: changeToPCRelativeCall ( MachineInstr * MI , const Function * Callee ) { auto & MBB = * MI -> getParent ( ) ; auto & MF = * MBB . getParent ( ) ; auto & C = MF . getFunction ( ) . getContext ( ) ; auto & AFI = * MF . getInfo < <ISA_LIT>FunctionInfo > ( ) ; auto & TII = * MF . getSubtarget ( ) . getInstrInfo ( ) ; auto & TLI = * MF . getSubtarget ( ) . getTargetLowering ( ) ; auto & DL = MF . getDataLayout ( ) ; auto & MRI = MF . getRegInfo ( ) ; auto isThumb = AFI . isThumbFunction ( ) ; auto Label = AFI . createPICLabelUId ( ) ; auto PCAdj = isThumb ? <NUM_LIT> : <NUM_LIT> ; auto * CPV = <ISA_LIT>ConstantPoolConstant :: Create ( Callee , Label , <STR_LIT> , PCAdj , <STR_LIT> , false ) ; auto Alignment = DL . getPrefTypeAlignment ( Type :: getInt32PtrTy ( C ) ) ; auto Index = MF . getConstantPool ( ) -> getConstantPoolIndex ( CPV , Alignment ) ; auto Opc = AFI . isThumb2Function ( ) ? <STR_LIT> : <STR_LIT> ; auto OffsetReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; auto MIB = BuildMI ( MBB , * MI , MI -> getDebugLoc ( ) , TII . get ( Opc ) , OffsetReg ) . addConstantPoolIndex ( Index ) ; if ( Opc == <STR_LIT> ) MIB . addImm ( <NUM_LIT> ) ; MIB . add ( predOps (


create<ISA_LIT>PagerandoOptimizerPass ( ) {


<s> static const Function * getCallee ( const MachineConstantPoolEntry & E ) { auto * CPC = static_cast < <ISA_LIT>ConstantPoolConstant * > ( E


<s> if ( MI . mayLoad ( ) && MI . getNumOperands ( ) > <NUM_LIT> && MI . getOperand ( <NUM_LIT> ) . isCPI ( ) ) return MI . getOperand ( <NUM_LIT> ) . getIndex ( ) ; return -


<s> if ( ! CPC ) return false ; auto M = CPC -> getModifier ( ) ; auto * F = dyn_cast_or_null < Function > ( CPC -> getGV ( ) ) ; return ( M == <STR_LIT> || M == <STR_LIT> )

&& F && F -> getSectionPrefix ( ) == BinPrefix ;


<s> auto * CPV = static_cast < <ISA_LIT>ConstantPoolValue * > ( E . Val . MachineCPVal ) ; auto * CPC = dyn_cast < <ISA_LIT>ConstantPoolConstant > ( CPV ) ; if ( ! CPC


<s> void <ISA_LIT>PagerandoOptimizer :: optimizeCalls ( MachineInstr * MI , const Function * Callee ) { auto & MRI = MI -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; SmallVector < MachineInstr * , <NUM_LIT> > Queue { MI } ; while ( ! Queue . empty ( ) ) { MI = Queue . pop_back_val ( ) ; if ( ! MI -> isCall ( ) ) { for ( auto & Op : MI -> defs ( ) ) { for ( auto & User : MRI . use_instructions ( Op . getReg ( ) ) ) Queue . push_back ( & User ) ; } MI -> eraseFromParent ( ) ; } else if ( isBXCall ( MI -> getOpcode ( ) ) ) { changeToPCRelativeCall ( MI , Callee ) ; } else


<s> auto & TII = * MBB . getParent ( ) -> getSubtarget ( ) . getInstrInfo ( ) ; auto Opc = toDirectCall ( MI -> getOpcode ( ) ) ; auto MIB = BuildMI ( MBB , * MI , MI -> getDebugLoc ( ) , TII . get ( Opc ) ) ; int SkipOps = <NUM_LIT> ; if ( MI -> getOpcode ( ) == <STR_LIT> ) { auto CondOp = predOps ( <STR_LIT> ) ; MIB . add ( CondOp ) ; SkipOps += CondOp . size (


<s> return false ; } auto BinPrefix = F . getSectionPrefix ( ) . getValue ( ) ; auto & CPEntries = MF . getConstantPool ( ) -> getConstants ( ) ; SmallSet < int , <NUM_LIT> > CPIndices ; int Index = <NUM_LIT> ; for ( auto & E : CPEntries ) { if ( isIntraBin ( E , BinPrefix ) ) CPIndices . insert ( Index ) ; Index ++ ; } if ( CPIndices . empty ( ) ) return false ; std :: vector < MachineInstr * > Uses ; for ( auto & BB : MF ) { for ( auto & MI : BB


<s> if ( ! F . isPagerando ( ) || skipFunction ( F ) ) { return false ; } auto BinPrefix = F . getSectionPrefix ( ) . getValue ( ) ; auto & CPEntries = MF . getConstantPool ( ) -> getConstants ( ) ; SmallSet < int , <NUM_LIT> > CPIndices ; int Index = <NUM_LIT> ; for ( auto & E : CPEntries ) { if ( isIntraBin ( E , BinPrefix ) ) CPIndices . insert ( Index ) ; Index ++ ; } if ( CPIndices . empty ( ) ) return false ; std :: vector < MachineInstr * > Uses ; for ( auto & BB : MF ) { for ( auto & MI : BB ) { int Index = getCPIndex ( MI ) ; if ( CPIndices . count ( Index ) ) Uses . push_back ( & MI ) ; } } for ( auto * MI : Uses ) { int Index = getCPIndex ( * MI ) ; auto * Callee = getCallee ( CPEntries [ Index ] ) ; optimizeCalls ( MI ,


<s> static unsigned toDirectCall ( unsigned Opc ) { switch ( Opc ) { case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> :


<s> } } else if ( AreSequentialLoads ( Ld1 , Ld0 , PMul0 -> VecLd ) && AreSequentialLoads ( Ld2 , Ld3 , PMul1 -> VecLd ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; R . AddMulPair ( PMul1 , PMul0 , true ) ; return true ; } return false ; } ; MulCandList & Muls = R . getMuls ( ) ; const unsigned Elems = Muls . size ( ) ; for ( unsigned i = <NUM_LIT> ; i < Elems ; ++ i ) { MulCandidate * PMul0 = static_cast < MulCandidate * > ( Muls [ i ] . get ( ) ) ; if ( PMul0 -> Paired ) continue ; for ( unsigned j = <NUM_LIT> ; j < Elems ; ++ j ) { if ( i == j ) continue ; MulCandidate * PMul1 = static_cast < MulCandidate * > ( Muls [ j ] . get ( ) ) ; if ( PMul1 -> Paired ) continue ; const Instruction * Mul0 = PMul0 -> Root ; const Instruction * Mul1 = PMul1 -> Root ; if ( Mul0 == Mul1


<s> LoadInst * <ISA_LIT>ParallelDSP :: CreateWideLoad ( MemInstList & Loads , IntegerType * LoadTy ) { assert ( Loads . size ( ) == <NUM_LIT> && <STR_LIT> ) ; LoadInst * Base = Loads [ <NUM_LIT> ] ; LoadInst * Offset = Loads [ <NUM_LIT> ] ; Instruction * BaseSExt = dyn_cast < SExtInst > ( Base -> user_back ( ) ) ; Instruction * OffsetSExt = dyn_cast < SExtInst > ( Offset -> user_back ( ) ) ; assert ( ( BaseSExt && OffsetSExt ) && <STR_LIT> ) ; std :: function < void ( Value * , Value * ) > MoveBefore = [ & ] ( Value * A , Value * B ) -> void { if ( ! isa < Instruction > ( A ) || ! isa < Instruction > ( B ) ) return ; auto * Source = cast < Instruction > ( A ) ; auto * Sink = cast < Instruction > ( B ) ; if ( DT -> dominates ( Source , Sink ) || Source -> getParent ( ) != Sink -> getParent ( ) || isa < PHINode > ( Source ) || isa < PHINode > ( Sink ) ) return ; Source -> moveBefore ( Sink ) ; for ( auto & Op : Source -> operands ( ) ) MoveBefore ( Op , Source ) ; } ; LoadInst * DomLoad = DT -> dominates ( Base , Offset ) ? Base : Offset ; IRBuilder < NoFolder > IRB ( DomLoad -> getParent ( ) , ++ BasicBlock :: iterator ( DomLoad ) ) ; const unsigned AddrSpace = DomLoad ->


<s> } ; LoadInst * DomLoad = DT -> dominates ( Base , Offset ) ? Base : Offset ; IRBuilder < NoFolder > IRB ( DomLoad -> getParent ( ) , ++ BasicBlock :: iterator ( DomLoad ) ) ; const unsigned AddrSpace = DomLoad -> getPointerAddressSpace ( ) ; Value * VecPtr = IRB . CreateBitCast ( Base -> getPointerOperand ( ) , LoadTy -> getPointerTo ( AddrSpace ) ) ; LoadInst * WideLoad = IRB . CreateAlignedLoad ( LoadTy , VecPtr , Base -> getAlign ( ) ) ; MoveBefore ( Base -> getPointerOperand ( ) , VecPtr ) ; MoveBefore ( VecPtr , WideLoad ) ; Value * Bottom = IRB . CreateTrunc ( WideLoad , Base -> getType ( ) ) ; Value * NewBaseSExt = IRB . CreateSExt ( Bottom , BaseSExt -> getType ( ) ) ; BaseSExt -> replaceAllUsesWith ( NewBaseSExt ) ; IntegerType * OffsetTy = cast < IntegerType > ( Offset -> getType ( ) ) ; Value * ShiftVal = ConstantInt :: get ( LoadTy , OffsetTy -> getBitWidth (


<s> LoadInst * getBaseLoad ( ) const { return VecLd . front ( )


<s> } else if ( auto * I = dyn_cast < Instruction > ( V ) ) { if ( I -> getOpcode ( ) == Instruction :: Mul ) return I ; } return nullptr ; } ; auto InsertMul = [ this ] ( Instruction * I ) { Value * LHS = cast < Instruction > ( I -> getOperand ( <NUM_LIT> ) ) -> getOperand ( <NUM_LIT> ) ; Value * RHS = cast < Instruction > ( I -> getOperand ( <NUM_LIT> ) ) -> getOperand ( <NUM_LIT> ) ; Muls . push_back ( std :: make_unique < MulCandidate > ( I , LHS , RHS ) ) ; } ; for ( auto * Add : Adds ) { if ( Add == Acc ) continue ; if ( auto * Mul = GetMulOperand ( Add -> getOperand ( <NUM_LIT> ) ) ) InsertMul ( Mul ) ; if ( auto * Mul = GetMulOperand ( Add -> getOperand ( <NUM_LIT>


<s> bool is64Bit ( ) const { return Root -> getType ( ) -> isIntegerTy


<s> bool <ISA_LIT>ParallelDSP :: IsNarrowSequence ( Value * V ) { if ( auto * SExt = dyn_cast < SExtInst > ( V ) ) { if ( SExt -> getSrcTy ( ) -> getIntegerBitWidth ( ) != MaxBitWidth ) return false ; if ( auto * Ld = dyn_cast < LoadInst > ( SExt -> getOperand ( <NUM_LIT> ) ) ) { return LoadPairs . count ( Ld ) || OffsetLoads . count ( Ld ) ; } } return false


<s> SmallPtrSet < Instruction * , <NUM_LIT> > AllAdds ; if ( ! RecordMemoryOps ( & BB ) ) continue ; for ( Instruction & I : reverse ( BB ) ) { if ( I . getOpcode ( ) != Instruction :: Add ) continue ; if ( AllAdds . count ( & I ) ) continue ; const auto * Ty = I . getType ( ) ; if ( ! Ty -> isIntegerTy ( <NUM_LIT> ) && ! Ty -> isIntegerTy ( <NUM_LIT> ) ) continue ; Reduction R ( & I ) ; if ( ! Search ( & I , & BB , R ) ) continue ; R . InsertMuls


<s> if ( ! isModOrRefSet ( intersectModRef ( AA -> getModRefInfo ( Write , ReadLoc ) , ModRefInfo :: ModRef ) ) ) continue ; if ( Write -> comesBefore ( Read ) ) RAWDeps [ Read ] . insert ( Write ) ; } } auto SafeToPair = [ & ] ( LoadInst * Base , LoadInst * Offset ) { bool BaseFirst = Base -> comesBefore ( Offset ) ; LoadInst * Dominator = BaseFirst ? Base : Offset ; LoadInst * Dominated = BaseFirst ? Offset : Base ; if ( RAWDeps . count ( Dominated ) ) { InstSet & WritesBefore = RAWDeps [ Dominated ] ; for ( auto Before : WritesBefore ) { if ( Dominator -> comesBefore ( Before ) ) return false ; } } return true ; } ; for ( auto * Base : Loads ) { for ( auto * Offset : Loads ) { if ( Base == Offset || OffsetLoads . count ( Offset ) ) continue ; if ( isConsecutiveAccess ( Base , Offset , * DL , * SE ) && SafeToPair ( Base , Offset ) ) { LoadPairs [ Base ] = Offset ; OffsetLoads . insert ( Offset ) ; break ; } } } LLVM_DEBUG ( if ( ! LoadPairs . empty ( ) ) { dbgs ( ) << <STR_LIT> ; for ( auto & MapIt : LoadPairs ) { LLVM_DEBUG ( dbgs ( ) << * MapIt . first <<

<STR_LIT> << * MapIt . second << <STR_LIT> ) ;


<s> for ( auto Read : Loads ) { MemoryLocation ReadLoc = MemoryLocation ( Read -> getPointerOperand ( ) , Size ) ; if ( ! isModOrRefSet ( intersectModRef ( AA -> getModRefInfo ( Write , ReadLoc ) , ModRefInfo :: ModRef ) ) ) continue ; if ( Write -> comesBefore ( Read ) ) RAWDeps [ Read ] . insert ( Write ) ; } } auto SafeToPair = [ & ] ( LoadInst * Base , LoadInst * Offset ) { bool BaseFirst = Base -> comesBefore ( Offset ) ; LoadInst * Dominator = BaseFirst ? Base : Offset ; LoadInst * Dominated = BaseFirst ? Offset : Base ; if ( RAWDeps . count ( Dominated ) ) { InstSet & WritesBefore = RAWDeps [ Dominated ] ; for ( auto Before : WritesBefore ) { if ( Dominator -> comesBefore (


<s> if ( ! ST -> hasDSP ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } if ( ! ST -> isLittle ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return false ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << F . getName ( ) << <STR_LIT> ) ; bool Changes = MatchSMLAD ( F ) ; return Changes


<s> return R . InsertAcc ( V ) ; case Instruction :: Add : { R . InsertAdd ( I ) ; Value * LHS = I -> getOperand ( <NUM_LIT> ) ; Value * RHS = I -> getOperand ( <NUM_LIT> ) ; bool ValidLHS = Search ( LHS , BB , R ) ; bool ValidRHS = Search ( RHS , BB , R ) ; if ( ValidLHS && ValidRHS ) return true ; return R . InsertAcc ( I ) ; } case Instruction :: Mul : { Value * MulOp0 = I -> getOperand ( <NUM_LIT> ) ; Value * MulOp1 = I -> getOperand ( <NUM_LIT> ) ; return IsNarrowSequence < <NUM_LIT> > ( MulOp0 ) && IsNarrowSequence


<s> WidenedLoad ( SmallVectorImpl < LoadInst * > & Lds , LoadInst * Wide ) : NewLd ( Wide ) { append_range ( Loads ,


<s> } if ( ! Acc ) { Acc = Mul ; continue ; } Acc = Builder . CreateAdd ( Mul , Acc ) ; InsertAfter = cast < Instruction > ( Acc ) ; } if ( ! Acc ) Acc = R . is64Bit ( ) ? ConstantInt :: get ( IntegerType :: get ( M -> getContext ( ) , <NUM_LIT> ) , <NUM_LIT> ) : ConstantInt :: get ( IntegerType :: get ( M -> getContext ( ) , <NUM_LIT> ) , <NUM_LIT> ) ; IntegerType * Ty = IntegerType :: get ( M -> getContext ( ) , <NUM_LIT> ) ; for ( auto & Pair : R . getMulPairs ( ) ) { MulCandidate * LHSMul = Pair . first ; MulCandidate * RHSMul = Pair . second ; LoadInst * BaseLHS = LHSMul -> getBaseLoad ( ) ; LoadInst * BaseRHS = RHSMul -> getBaseLoad ( ) ; LoadInst * WideLHS = WideLoads . count ( BaseLHS ) ? WideLoads [ BaseLHS ] -> getLoad ( ) : CreateWideLoad ( LHSMul -> VecLd , Ty ) ; LoadInst * WideRHS = WideLoads . count ( BaseRHS ) ? WideLoads [ BaseRHS ] -> getLoad (

) : CreateWideLoad ( RHSMul -> VecLd , Ty ) ;


<s> if ( Loads . empty ( ) || Loads . size ( ) > NumLoadLimit ) return false ; using InstSet = std :: set < Instruction * > ; using DepMap = std :: map < Instruction * , InstSet > ; DepMap RAWDeps ; const auto Size = LocationSize :: beforeOrAfterPointer ( ) ; for ( auto Write : Writes ) { for ( auto Read : Loads ) { MemoryLocation ReadLoc = MemoryLocation ( Read -> getPointerOperand ( ) , Size ) ; if ( ! isModOrRefSet ( intersectModRef ( AA -> getModRefInfo ( Write , ReadLoc ) , ModRefInfo :: ModRef ) ) ) continue ; if ( Write -> comesBefore ( Read ) ) RAWDeps [ Read ] . insert ( Write ) ; } } auto SafeToPair = [ & ] ( LoadInst * Base , LoadInst * Offset ) { bool BaseFirst = Base -> comesBefore ( Offset ) ; LoadInst * Dominator = BaseFirst ? Base : Offset ; LoadInst * Dominated = BaseFirst ? Offset : Base ; if ( RAWDeps . count ( Dominated ) ) { InstSet & WritesBefore = RAWDeps [ Dominated ] ; for ( auto Before : WritesBefore ) { if ( Dominator -> comesBefore ( Before ) ) return false ; } } return true ; } ; for ( auto *


<s> for ( auto * I : Writes ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; I -> dump ( ) ) ; assert ( MAC . MemLocs . size ( ) >= <NUM_LIT> && <STR_LIT> ) ; for ( auto & MemLoc : MAC . MemLocs ) { if ( isModOrRefSet ( intersectModRef ( AA -> getModRefInfo ( I , MemLoc ) , ModRefInfo :: ModRef


<s> if ( ! MemOp0 -> isSimple ( ) || ! MemOp1 -> isSimple ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( isConsecutiveAccess ( MemOp0 , MemOp1 ,


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; Ld0 -> dump ( ) ; dbgs ( ) << <STR_LIT> ; Ld1 -> dump ( ) ; ) ; if ( ! Ld0 -> hasOneUse ( ) ||


<s> static void CreateLoadIns ( IRBuilder < NoFolder > & IRB , Instruction * Acc , LoadInst * * VecLd ) { const Type * AccTy = Acc ->


<s> Instruction * <ISA_LIT>ParallelDSP :: CreateSMLADCall ( LoadInst * VecLd0 , LoadInst * VecLd1 , Instruction * Acc , Instruction * InsertAfter ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; VecLd0 -> dump ( ) ; dbgs ( ) << <STR_LIT> ; VecLd1 -> dump ( ) ; dbgs ( ) << <STR_LIT> ; Acc -> dump ( ) ) ; IRBuilder < NoFolder > Builder ( InsertAfter -> getParent ( ) , ++ BasicBlock :: iterator ( InsertAfter ) ) ; CreateLoadIns ( Builder , Acc , & VecLd0 ) ; CreateLoadIns ( Builder , Acc , & VecLd1 ) ; Value * Args [ ] = { VecLd0 , VecLd1 , Acc } ; Function * SMLAD = <STR_LIT> ( M


<s> dbgs ( ) << <STR_LIT> ; VecLd1 -> dump ( ) ; dbgs ( ) << <STR_LIT> ; Acc -> dump ( ) ) ; IRBuilder < NoFolder > Builder ( InsertAfter -> getParent ( ) , ++ BasicBlock :: iterator ( InsertAfter ) ) ; CreateLoadIns ( Builder , Acc , &


<s> Instruction * InsertAfter = Reduction . AccIntAdd ; for ( auto & Pair : PMACPairs ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; Pair . first -> Mul -> dump ( ) ; dbgs ( ) << <STR_LIT> ; Pair . second -> Mul -> dump ( ) ) ; auto * VecLd0 = cast < LoadInst > ( Pair . first -> VecLd [ <NUM_LIT> ] ) ; auto * VecLd1 = cast < LoadInst > ( Pair . second -> VecLd [ <NUM_LIT> ] ) ; Acc = CreateSMLADCall ( VecLd0 , VecLd1 , Acc , InsertAfter ) ; InsertAfter = Acc ; } if ( Acc != Reduction . Phi ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; Acc -> dump ( ) ) ; Reduction . AccIntAdd -> replaceAllUsesWith (


<s> return Reductions ; } for ( PHINode & Phi : Header -> phis ( ) ) { const auto * Ty = Phi . getType ( ) ; if ( ! Ty -> isIntegerTy ( <NUM_LIT> ) ) continue ; const bool IsReduction = RecurrenceDescriptor :: AddReductionVar ( & Phi , RecurrenceDescriptor :: RK_IntegerAdd , TheLoop , HasFnNoNaNAttr , RecDesc ) ; if ( ! IsReduction ) continue ; Instruction * Acc = dyn_cast < Instruction > ( Phi . getIncomingValueForBlock ( Latch ) ) ; if ( ! Acc ) continue ; Reductions . push_back ( Reduction ( & Phi , Acc ) ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; for ( auto R : Reductions ) { dbgs ( ) << <STR_LIT> ; R . Phi


<s> ReductionList Reductions = MatchReductions ( F , L , Header ) ; for ( auto & R : Reductions ) { ParallelMACList MACCandidates = MatchParallelMACs ( R ) ; if ( ! SetMemoryLocations ( MACCandidates ) ) continue ; R . MACCandidates = MACCandidates ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; for ( auto & M : R . MACCandidates


<s> ParallelMAC ( Instruction * I , ValueList & V , bool RdOnly ) : Mul ( I )


<s> ParallelMAC ( Instruction * I , ValueList & V , bool RdOnly ) : Mul ( I ) , VL ( V ) ,


<s> LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; auto & TPC = getAnalysis < TargetPassConfig > ( ) ; BasicBlock * Header = TheLoop -> getHeader ( ) ; if ( ! Header ) return false ; if ( Header != TheLoop -> getLoopLatch ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } Function & F = * Header -> getParent ( ) ; M = F . getParent


<s> } for ( unsigned i = <NUM_LIT> ; i < C . VL . size ( ) ; i += <NUM_LIT> ) { auto * LdOp0 = dyn_cast < LoadInst > ( C . VL [ i ] ) ; auto * LdOp1 = dyn_cast < LoadInst > ( C . VL [ i + <NUM_LIT> ] ) ; if ( ! LdOp0 || ! LdOp1 ) return false ; C . MemLocs . push_back ( MemoryLocation ( LdOp0 -> getPointerOperand ( ) , Size ) ) ; C . MemLocs . push_back ( MemoryLocation ( LdOp1 -> getPointerOperand ( ) , Size ) ) ; } } return


<s> ValueList VL ; if ( IsNarrowSequence < <NUM_LIT> > ( MulOp0 , VL ) && IsNarrowSequence < <NUM_LIT> > ( MulOp1 , VL ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; Mul -> dump ( ) ) ; Candidates . push_back ( ParallelMAC (


<s> auto IsMACCandidate = [ ] ( Instruction * I , ParallelMACList & MACCandidates ) { for ( auto & MAC : MACCandidates ) for ( auto * Val : MAC . VL ) if

( I == MAC . Mul || Val == I ) return true ;


<s> static bool AreAliased ( AliasAnalysis * AA , Instructions AliasCandidates , ParallelMACList & MACCandidates ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; for ( auto * I : AliasCandidates ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; I -> dump ( ) ) ; for ( auto & MAC : MACCandidates ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MAC . Mul -> dump ( ) ) ; assert ( MAC . MemLocs . size ( ) >= <NUM_LIT> && <STR_LIT> ) ; for ( auto & MemLoc : MAC . MemLocs ) { if ( isModOrRefSet ( intersectModRef ( AA -> getModRefInfo ( I , MemLoc ) , ModRefInfo :: ModRef ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; VecLd0 -> dump ( ) ; dbgs ( ) << <STR_LIT> ; VecLd1 -> dump ( ) ; dbgs ( ) << <STR_LIT> ; Acc -> dump ( ) ) ; IRBuilder < NoFolder > Builder ( InsertAfter -> getParent ( ) , ++ BasicBlock :: iterator ( InsertAfter ) ) ; CreateLoadIns ( Builder , Acc , &


<s> dbgs ( ) << <STR_LIT> ; Acc -> dump ( ) ) ; IRBuilder < NoFolder > Builder ( InsertAfter -> getParent ( ) , ++ BasicBlock :: iterator ( InsertAfter ) ) ; CreateLoadIns ( Builder , Acc , & VecLd0 ) ; CreateLoadIns ( Builder , Acc , & VecLd1 ) ; Value * Args [ ] = { VecLd0 , VecLd1 , Acc } ; Function * SMLAD = <STR_LIT> ( M , <STR_LIT> ) ; CallInst * Call = Builder . CreateCall ( SMLAD , Args ) ; return


<s> auto * I = dyn_cast < Instruction > ( V ) ; if ( ! I ) return false ; Value * Val , * LHS , * RHS ; bool isNarrow = false ; if ( match ( V , m_Trunc ( m_Value ( Val ) ) ) ) { if ( cast < TruncInst > ( I ) -> getDestTy ( ) -> getIntegerBitWidth ( ) == BitWidth ) isNarrow = IsNarrowSequence < BitWidth > ( Val , VL ) ; } else if ( match ( V , m_Add ( m_Value ( LHS ) , m_Value ( RHS ) ) ) ) { isNarrow = false ; } else if ( match ( V , m_ZExtOrSExt ( m_Value ( Val ) ) ) ) { if ( cast < CastInst > ( I ) -> getSrcTy ( ) -> getIntegerBitWidth ( ) == BitWidth ) isNarrow = true ; else LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << cast < CastInst > ( I ) -> getSrcTy ( ) -> getIntegerBitWidth ( ) ) ; if ( match ( Val , m_Load ( m_Value ( Val ) ) ) ) { auto * Ld = dyn_cast < LoadInst > ( I -> getOperand ( <NUM_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; Ld -> dump ( ) ) ; VL . push_back ( Ld ) ; isNarrow = true ; } else if ( ! isa < Instruction > ( I -> getOperand ( <NUM_LIT> ) ) ) VL . push_back ( I -> getOperand ( <NUM_LIT> ) ) ; } if ( isNarrow ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; I -> dump ( ) ) ; VL . push_back ( I ) ; } else LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; I -> dump ( ) ) ; return isNarrow


<s> if ( cast < TruncInst > ( I ) -> getDestTy ( ) -> getIntegerBitWidth ( ) == BitWidth ) isNarrow = IsNarrowSequence < BitWidth > ( Val , VL ) ; } else if ( match ( V , m_Add ( m_Value ( LHS ) , m_Value ( RHS ) ) ) ) { isNarrow = false ; } else if ( match ( V , m_ZExtOrSExt ( m_Value ( Val ) ) ) ) { if ( cast < CastInst > ( I ) -> getSrcTy ( ) -> getIntegerBitWidth ( ) == BitWidth ) isNarrow = true ; else LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << cast < CastInst > ( I ) -> getSrcTy ( ) -> getIntegerBitWidth ( ) ) ; if ( match ( Val , m_Load ( m_Value ( Val ) ) ) ) { auto * Ld = dyn_cast < LoadInst > ( I -> getOperand ( <NUM_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; Ld -> dump ( ) ) ; VL . push_back ( Ld ) ; isNarrow = true ; } else if ( ! isa < Instruction > ( I -> getOperand ( <NUM_LIT> ) ) ) VL . push_back ( I -> getOperand ( <NUM_LIT> ) ) ; } if ( isNarrow ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; I -> dump ( ) ) ; VL . push_back ( I )


<s> } while ( match ( Acc , m_Add ( m_Value ( A ) , m_Mul ( m_Value ( MulOp0 ) , m_Value ( MulOp1 ) ) ) ) ) { AddCandidateMAC ( Candidates , Acc , MulOp0 , MulOp1 , <NUM_LIT> ) ; Acc = dyn_cast < Instruction > ( A ) ; } if ( match ( Acc , m_Add ( m_Mul ( m_Value ( MulOp0 ) , m_Value ( MulOp1 ) ) , m_Value ( A ) ) ) ) AddCandidateMAC ( Candidates , Acc , MulOp0 , MulOp1 , <NUM_LIT> ) ; std :: reverse ( Candidates . begin ( ) , Candidates . end ( ) ) ; return Candidates


<s> const bool IsReduction = RecurrenceDescriptor :: AddReductionVar ( & Phi , RecurrenceDescriptor :: RK_IntegerAdd , TheLoop , HasFnNoNaNAttr , RecDesc ) ; if ( ! IsReduction ) continue ; Instruction * Acc = dyn_cast < Instruction > ( Phi . getIncomingValueForBlock ( Latch ) ) ; if ( ! Acc ) continue ; Reductions . push_back ( Reduction ( & Phi , Acc ) ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; for ( auto R : Reductions ) { dbgs ( ) << <STR_LIT> ; R . Phi -> dump ( ) ; dbgs ( ) << <STR_LIT> ; R . AccIntAdd -> dump ( ) ; } ) ; return Reductions


<s> dbgs ( ) << <STR_LIT> ; Header -> dump ( ) ; dbgs ( ) << <STR_LIT> ; L -> dump ( ) ) ; bool Changed = false ; ReductionList Reductions = MatchReductions ( F , L , Header ) ; for ( auto & R : Reductions ) { ParallelMACList MACCandidates = MatchParallelMACs ( R ) ; if ( ! SetMemoryLocations ( MACCandidates ) ) continue ; Instructions Aliases = AliasCandidates ( Header , MACCandidates ) ; if ( AreAliased ( AA , Aliases ,


<s> if ( AreAliased ( AA , Aliases , MACCandidates ) ) continue ; PMACPairList PMACPairs = CreateParallelMACPairs ( MACCandidates ) ; Changed = InsertParallelMACs ( R , PMACPairs ) || Changed ; } LLVM_DEBUG ( if ( Changed ) dbgs ( ) << <STR_LIT> ; Header -> dump ( ) ; ) ; return Changed


<s> ParallelMAC ( Instruction * I , ValueList & V ) : Mul ( I ) , VL (


<s> ParallelMAC ( Instruction * I , ValueList & V ) : Mul ( I )


<s> ValueList RHS ; if ( IsNarrowSequence < <NUM_LIT> > ( MulOp0 , LHS ) && IsNarrowSequence


<s> if ( AreSequentialLoads ( Ld0 , Ld1 , PMul0 -> VecLd ) ) { if ( AreSequentialLoads ( Ld2 , Ld3 , PMul1 -> VecLd ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; PMACPairs . push_back ( std :: make_pair ( PMul0 , PMul1 ) ) ; return true ; } else if ( AreSequentialLoads ( Ld3 , Ld2 , PMul1 -> VecLd ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; PMul1 -> Exchange = true ; PMACPairs . push_back ( std :: make_pair ( PMul0 , PMul1 ) ) ; return true ; } } else if ( AreSequentialLoads ( Ld1 , Ld0 , PMul0 -> VecLd ) && AreSequentialLoads ( Ld2 , Ld3 , PMul1 -> VecLd ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; PMul0 -> Exchange = true ; PMACPairs . push_back ( std :: make_pair ( PMul1 , PMul0 ) ) ; return true ; } } return false ; } ; SmallPtrSet < const Instruction * , <NUM_LIT> > Paired ; for ( unsigned i = <NUM_LIT> ; i < Elems ; ++ i ) { BinOpChain * PMul0 = static_cast < BinOpChain * > ( Candidates [ i ] . get ( ) ) ; if ( Paired . count ( PMul0 -> Root ) ) continue ; for ( unsigned j = <NUM_LIT> ; j < Elems ; ++ j ) { if ( i == j ) continue ; BinOpChain * PMul1 = static_cast < BinOpChain * > ( Candidates [ j ] . get ( ) ) ; if ( Paired . count ( PMul1 -> Root ) ) continue ; const Instruction * Mul0 = PMul0 -> Root ; const Instruction * Mul1 = PMul1 -> Root ; if ( Mul0 == Mul1 ) continue ; assert ( PMul0 != PMul1 &&


<s> bool <ISA_LIT>ParallelDSP :: InsertParallelMACs ( Reduction & Reduction ) { Instruction * Acc = Reduction . Phi ; Instruction * InsertAfter = Reduction . AccIntAdd ; for ( auto & Pair : Reduction . PMACPairs ) { BinOpChain * PMul0 = Pair . first ; BinOpChain * PMul1 = Pair . second ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; PMul0 -> Root -> dump ( ) ; dbgs ( ) << <STR_LIT> ; PMul1 -> Root -> dump ( ) ) ; Acc = CreateSMLADCall ( PMul0 -> VecLd , PMul1 -> VecLd , Acc , PMul1 -> Exchange , InsertAfter ) ; InsertAfter = Acc ; } if ( Acc != Reduction . Phi ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; Acc -> dump ( ) ) ; Reduction . AccIntAdd -> replaceAllUsesWith (


<s> ConstantInt * CInt ; if ( match ( V , m_ConstantInt ( CInt ) ) ) { return false ; } auto * I = dyn_cast < Instruction > ( V ) ; if ( ! I ) return false ; Value * Val , * LHS , * RHS ; if ( match ( V , m_Trunc ( m_Value ( Val ) ) ) ) { if ( cast < TruncInst > ( I ) -> getDestTy ( ) -> getIntegerBitWidth ( ) == MaxBitWidth ) return IsNarrowSequence < MaxBitWidth > ( Val , VL ) ; } else if ( match ( V , m_Add ( m_Value ( LHS ) , m_Value ( RHS ) ) ) ) { return false ; } else if ( match ( V , m_ZExtOrSExt ( m_Value ( Val ) ) ) ) { if ( cast < CastInst > ( I ) -> getSrcTy ( ) -> getIntegerBitWidth ( ) != MaxBitWidth ) return false ; if ( match ( Val , m_Load ( m_Value ( ) ) ) ) { VL . push_back ( Val ) ; VL . push_back ( I ) ; return true


<s> } auto * I = dyn_cast < Instruction > ( V ) ; if ( ! I ) return false ; Value * Val , * LHS , * RHS ; if ( match ( V , m_Trunc ( m_Value ( Val ) ) ) ) { if ( cast < TruncInst > ( I ) -> getDestTy ( ) -> getIntegerBitWidth ( ) == MaxBitWidth ) return IsNarrowSequence < MaxBitWidth > ( Val , VL ) ; } else if ( match ( V , m_Add ( m_Value ( LHS ) , m_Value ( RHS ) ) ) ) { return false ; } else if ( match ( V , m_ZExtOrSExt ( m_Value ( Val ) ) ) ) { if ( cast < CastInst > ( I ) -> getSrcTy ( ) -> getIntegerBitWidth ( ) != MaxBitWidth ) return false ; if ( match ( Val , m_Load ( m_Value ( )


<s> auto * I = dyn_cast < Instruction > ( V ) ; if ( ! I ) return false ; switch ( I -> getOpcode ( ) ) { case Instruction :: Add : if ( Match ( I -> getOperand ( <NUM_LIT> ) ) || ( Match ( I -> getOperand ( <NUM_LIT> ) ) ) ) return true ; break ; case Instruction :: Mul : { Value * MulOp0 = I -> getOperand ( <NUM_LIT> ) ; Value * MulOp1 = I -> getOperand ( <NUM_LIT> ) ; if ( isa < SExtInst > ( MulOp0 ) && isa < SExtInst > ( MulOp1 ) ) AddMACCandidate (


<s> if ( ! IsReduction ) continue ; Instruction * Acc = dyn_cast < Instruction > ( Phi . getIncomingValueForBlock ( Latch ) ) ; if ( ! Acc ) continue ; Reductions . push_back ( Reduction ( & Phi , Acc ) ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; for ( auto & R : Reductions ) { dbgs ( ) << <STR_LIT> ; R . Phi -> dump ( ) ; dbgs ( ) << <STR_LIT> ; R . AccIntAdd ->


<s> if ( ! ST -> allowsUnalignedMem ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } if ( ! ST -> hasDSP ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } if ( ! ST -> isLittle ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } LoopAccessInfo LAI ( L , SE , TLI , AA , DT , LI ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << F . getName ( ) << <STR_LIT> ) ; if ( ! RecordSequentialLoads ( Header ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false


<s> static bool AreSequentialAccesses ( MemInst * MemOp0 , MemInst * MemOp1 , const DataLayout & DL , ScalarEvolution & SE ) { if ( isConsecutiveAccess ( MemOp0 , MemOp1 , DL , SE ) ) return true ; return


<s> if ( isConsecutiveAccess ( MemOp0 , MemOp1 , DL ,


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; Ld0 -> dump ( ) ; dbgs ( ) << <STR_LIT> ; Ld1 -> dump ( ) ; ) ; VecMem . clear ( ) ; VecMem . push_back ( Ld0 ) ; VecMem . push_back ( Ld1 ) ; return


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; R . AddMulPair ( PMul0 , PMul1 ) ; return true ; } else if ( AreSequentialLoads ( Ld3 , Ld2 , PMul1 -> VecLd ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; PMul1 -> Exchange = true ; R . AddMulPair ( PMul0 , PMul1 ) ; return true ; } } else if ( AreSequentialLoads ( Ld1 , Ld0 , PMul0 -> VecLd ) && AreSequentialLoads ( Ld2 , Ld3 , PMul1 -> VecLd ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; PMul0 -> Exchange = true ; R . AddMulPair ( PMul1 , PMul0 ) ; return true ; } return false ; } ; MulCandList & Muls = R . getMuls ( ) ; const unsigned Elems = Muls . size ( ) ; SmallPtrSet < const Instruction * , <NUM_LIT> > Paired ; for ( unsigned i = <NUM_LIT> ; i < Elems ; ++ i ) { MulCandidate * PMul0 = static_cast < MulCandidate * > ( Muls [ i ] . get ( ) ) ; if ( Paired . count ( PMul0 -> Root ) ) continue ; for ( unsigned j = <NUM_LIT> ; j < Elems ; ++ j ) { if ( i == j ) continue ; MulCandidate * PMul1 = static_cast < MulCandidate * > ( Muls [ j ] . get ( ) ) ; if ( Paired . count ( PMul1 -> Root ) ) continue ; const Instruction * Mul0 = PMul0 -> Root ; const Instruction * Mul1 = PMul1 -> Root ; if ( Mul0 == Mul1 ) continue ; assert ( PMul0 != PMul1 && <STR_LIT> ) ; if ( CanPair ( R , PMul0 , PMul1 ) ) { Paired . insert ( Mul0 )


<s> } ; LoadInst * DomLoad = DT -> dominates ( Base , Offset ) ? Base : Offset ; IRBuilder < NoFolder > IRB ( DomLoad -> getParent ( ) , ++ BasicBlock :: iterator ( DomLoad ) ) ; const unsigned AddrSpace = DomLoad -> getPointerAddressSpace ( ) ; Value * VecPtr = IRB . CreateBitCast ( Base -> getPointerOperand ( ) , LoadTy -> getPointerTo ( AddrSpace ) ) ; LoadInst * WideLoad = IRB . CreateAlignedLoad ( LoadTy , VecPtr , Base -> getAlignment ( ) ) ; MoveBefore ( Base -> getPointerOperand ( ) , VecPtr ) ; MoveBefore ( VecPtr , WideLoad ) ; Value * Bottom = IRB . CreateTrunc ( WideLoad , Base -> getType ( ) ) ; BaseSExt -> setOperand ( <NUM_LIT> , Bottom ) ; IntegerType * OffsetTy = cast < IntegerType > ( Offset -> getType ( ) ) ; Value * ShiftVal = ConstantInt :: get ( LoadTy , OffsetTy -> getBitWidth ( ) ) ; Value * Top = IRB . CreateLShr ( WideLoad , ShiftVal ) ; Value * Trunc = IRB . CreateTrunc ( Top , OffsetTy


<s> Value * getAccumulator ( ) { return Acc


<s> SmallPtrSetImpl < Instruction * > & getAdds ( ) { return


<s> FunctionPass :: getAnalysisUsage ( AU ) ; AU . addRequired < AssumptionCacheTracker > ( ) ; AU . addRequired < ScalarEvolutionWrapperPass > ( ) ; AU . addRequired < AAResultsWrapperPass > ( ) ; AU . addRequired < TargetLibraryInfoWrapperPass > (


<s> AU . addRequired < AssumptionCacheTracker > ( ) ; AU . addRequired < ScalarEvolutionWrapperPass > ( ) ; AU . addRequired < AAResultsWrapperPass > ( ) ; AU . addRequired < TargetLibraryInfoWrapperPass > ( ) ; AU . addRequired < DominatorTreeWrapperPass > ( ) ; AU . addRequired < TargetPassConfig > ( ) ; AU . addPreserved < ScalarEvolutionWrapperPass > (


<s> PMACPairList & getMulPairs ( ) { return MulPairs


<s> PMACPairList & getMulPairs ( ) { return


<s> bool InsertAcc ( Value * V ) { if ( Acc ) return false ; Acc = V ; return


<s> void InsertMul ( Instruction * I , ValueList & LHS ,


<s> void InsertMul ( Instruction * I , ValueList & LHS , ValueList & RHS ) { Muls . push_back ( make_unique < MulCandidate > ( I ,


<s> } } case Instruction :: Mul : { Value * MulOp0 = I -> getOperand ( <NUM_LIT> ) ; Value * MulOp1 = I -> getOperand ( <NUM_LIT> ) ; if ( isa < SExtInst > ( MulOp0 ) && isa < SExtInst > ( MulOp1 ) ) { ValueList LHS ; ValueList RHS ; if ( IsNarrowSequence < <NUM_LIT> > ( MulOp0 , LHS ) && IsNarrowSequence < <NUM_LIT> > ( MulOp1 , RHS ) ) { R . InsertMul ( I , LHS , RHS ) ; return true ; } } return false ; } case Instruction :: SExt : return Search ( I -> getOperand ( <NUM_LIT> ) , BB , R ) ; } return false ; } ; bool Changed = false ; for ( auto & BB : F ) { SmallPtrSet < Instruction * , <NUM_LIT> > AllAdds ; if ( ! RecordMemoryOps ( & BB ) ) continue ; for ( Instruction & I : reverse ( BB ) ) { if ( I . getOpcode ( ) != Instruction :: Add ) continue ; if ( AllAdds . count ( & I ) ) continue ; const auto * Ty = I . getType ( ) ; if ( ! Ty -> isIntegerTy ( <NUM_LIT> ) && ! Ty -> isIntegerTy ( <NUM_LIT> ) ) continue ; Reduction R ( & I ) ; if ( ! Search ( & I , & BB , R ) ) continue ; if ( ! CreateParallelPairs ( R ) ) continue ; InsertParallelMACs ( R ) ; Changed = true ; AllAdds . insert ( R . getAdds ( ) . begin ( ) , R . getAdds ( ) . end ( ) ) ; } } return


<s> MulCandidate ( Instruction * I , ValueList & lhs , ValueList &


<s> for ( auto & I : * BB ) { if ( I . mayWriteToMemory ( ) ) Writes . push_back ( & I ) ; auto * Ld = dyn_cast < LoadInst > ( & I ) ; if ( ! Ld || ! Ld -> isSimple ( ) || ! Ld -> hasOneUse ( ) || ! isa < SExtInst > ( Ld -> user_back ( ) ) ) continue ; Loads . push_back ( Ld ) ; } using InstSet = std :: set < Instruction * > ; using DepMap = std :: map < Instruction * , InstSet > ; DepMap RAWDeps ; const auto Size = LocationSize :: unknown ( ) ; for ( auto Read : Loads ) { for ( auto Write : Writes ) { MemoryLocation ReadLoc = MemoryLocation ( Read -> getPointerOperand ( ) , Size ) ; if ( ! isModOrRefSet ( intersectModRef ( AA -> getModRefInfo ( Write , ReadLoc ) , ModRefInfo :: ModRef ) ) ) continue ; if ( DT -> dominates ( Write , Read ) ) RAWDeps [ Read ] . insert ( Write ) ; } } auto SafeToPair = [ & ] ( LoadInst * Base , LoadInst * Offset ) { LoadInst * Dominator = DT -> dominates ( Base , Offset


<s> auto & TPC = getAnalysis < TargetPassConfig > ( ) ; M = F . getParent ( ) ; DL = & M -> getDataLayout ( ) ; auto & TM = TPC . getTM < TargetMachine > ( ) ; auto * ST = & TM . getSubtarget < <ISA_LIT>Subtarget > ( F ) ; if ( ! ST -> allowsUnalignedMem ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } if ( ! ST -> hasDSP ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return


<s> void UpdateRoot ( Instruction * SMLAD ) { Root -> replaceAllUsesWith (


<s> WidenedLoad ( SmallVectorImpl < LoadInst * > & Lds , LoadInst * Wide ) : NewLd ( Wide ) { for ( auto * I : Lds )


<s> if ( Exchange ) SMLAD = Acc -> getType ( ) -> isIntegerTy ( <NUM_LIT> ) ? <STR_LIT> ( M , <STR_LIT> ) : <STR_LIT> ( M , <STR_LIT> ) ; else SMLAD = Acc -> getType ( ) -> isIntegerTy ( <NUM_LIT> ) ? <STR_LIT> ( M , <STR_LIT> ) : <STR_LIT> ( M , <STR_LIT> ) ; IRBuilder < NoFolder > Builder ( InsertAfter -> getParent ( ) , ++ BasicBlock :: iterator ( InsertAfter ) ) ; Instruction * Call = Builder . CreateCall ( SMLAD , Args ) ; NumSMLAD ++ ; return Call ; } ; Instruction * InsertAfter = R . getRoot ( ) ; Value * Acc = R . getAccumulator ( ) ; IRBuilder < NoFolder > Builder ( InsertAfter -> getParent ( ) , ++ BasicBlock :: iterator ( InsertAfter ) ) ; MulCandList & MulCands = R . getMuls ( ) ; for ( auto & MulCand : MulCands ) { if ( MulCand -> Paired ) continue ; Value * Mul = MulCand -> Root ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Mul << <STR_LIT> ) ; if ( R . getType ( ) != Mul ->


<s> void <ISA_LIT>ParallelDSP :: InsertParallelMACs ( Reduction & R ) { auto CreateSMLAD = [ & ] ( LoadInst * WideLd0 , LoadInst * WideLd1 , Value * Acc , bool Exchange , Instruction * InsertAfter ) { Value * Args [ ] = { WideLd0 , WideLd1 , Acc } ; Function * SMLAD = nullptr ; if ( Exchange ) SMLAD = Acc -> getType ( ) -> isIntegerTy ( <NUM_LIT> ) ? <STR_LIT> ( M , <STR_LIT> ) : <STR_LIT> ( M , <STR_LIT> ) ; else SMLAD = Acc -> getType ( ) -> isIntegerTy ( <NUM_LIT> ) ? <STR_LIT> ( M , <STR_LIT> ) : <STR_LIT> ( M , <STR_LIT> ) ; IRBuilder < NoFolder > Builder ( InsertAfter -> getParent ( ) , ++ BasicBlock :: iterator ( InsertAfter ) ) ; Instruction * Call = Builder . CreateCall ( SMLAD , Args ) ; NumSMLAD ++ ; return Call ; } ; Instruction * InsertAfter = R . getRoot ( ) ; Value * Acc = R . getAccumulator ( ) ; IRBuilder < NoFolder > Builder ( InsertAfter -> getParent ( ) , ++ BasicBlock :: iterator ( InsertAfter ) ) ; MulCandList & MulCands = R


<s> MulPairs . push_back ( std :: make_pair


<s> } auto CanPair = [ & ] ( Reduction & R , BinOpChain * PMul0 , BinOpChain * PMul1 ) { if ( ! PMul0 -> AreSymmetrical ( PMul1 ) ) return false ; for ( unsigned x = <NUM_LIT> ; x < PMul0 -> LHS . size ( ) ; x += <NUM_LIT> ) { auto * Ld0 = dyn_cast < LoadInst > ( PMul0 -> LHS [ x ] ) ; auto * Ld1 = dyn_cast < LoadInst > ( PMul1 -> LHS [ x ] ) ; auto * Ld2 = dyn_cast < LoadInst > ( PMul0 -> RHS [ x ] ) ; auto * Ld3 = dyn_cast < LoadInst > ( PMul1 -> RHS [ x ] ) ; if ( ! Ld0 || ! Ld1 || ! Ld2 || ! Ld3 ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << * Ld0 << <STR_LIT> << <STR_LIT> << * Ld1 << <STR_LIT> << <STR_LIT> << * Ld2 << <STR_LIT> << <STR_LIT> << * Ld3 << <STR_LIT> ) ; if ( AreSequentialLoads ( Ld0 , Ld1 , PMul0 -> VecLd ) ) { if ( AreSequentialLoads ( Ld2 , Ld3 , PMul1 -> VecLd ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; R . AddMulPair ( PMul0 , PMul1 ) ; return true ; } else if ( AreSequentialLoads ( Ld3 , Ld2 , PMul1 -> VecLd ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; PMul1 -> Exchange = true ; R . AddMulPair ( PMul0 , PMul1 ) ; return true ; } } else if ( AreSequentialLoads ( Ld1 , Ld0 , PMul0 -> VecLd ) && AreSequentialLoads ( Ld2 , Ld3 , PMul1 -> VecLd ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; PMul0 -> Exchange = true ; R . AddMulPair ( PMul1 , PMul0 ) ; return true ; } } return false ; } ; OpChainList & Muls = R . getMuls ( ) ; const unsigned Elems = Muls . size ( ) ; SmallPtrSet < const Instruction * , <NUM_LIT> > Paired ; for ( unsigned i = <NUM_LIT> ; i < Elems ; ++ i ) { BinOpChain * PMul0 = static_cast < BinOpChain * > ( Muls [ i ] . get ( ) ) ; if ( Paired . count ( PMul0 ->


<s> Muls . push_back ( make_unique < BinOpChain > ( I , LHS ,


<s> auto * I = dyn_cast < Instruction > ( V ) ; if ( ! I ) return false ; Value * Val , * LHS , * RHS ; if ( match ( V , m_Trunc ( m_Value ( Val ) ) ) ) { if ( cast < TruncInst > ( I ) -> getDestTy ( ) -> getIntegerBitWidth ( ) == MaxBitWidth ) return IsNarrowSequence < MaxBitWidth > ( Val , VL ) ; } else if ( match ( V , m_Add ( m_Value ( LHS ) , m_Value ( RHS ) ) ) ) { return false ; } else if ( match ( V , m_ZExtOrSExt ( m_Value ( Val ) ) ) ) { if ( cast < CastInst > ( I ) -> getSrcTy ( ) -> getIntegerBitWidth ( ) != MaxBitWidth ) return false ; if ( match ( Val , m_Load ( m_Value ( ) ) ) ) { auto * Ld = cast < LoadInst > ( Val ) ; if ( ! LoadPairs . count ( Ld ) && ! OffsetLoads . count ( Ld ) ) return false ; VL . push_back (


<s> BasicBlock * Latch = L -> getLoopLatch ( ) ; for ( Instruction & I : reverse ( * Latch ) ) { if ( I . getOpcode ( ) != Instruction :: Add ) continue ; if ( AllAdds . count ( & I ) ) continue ; const auto * Ty = I . getType ( ) ; if ( ! Ty -> isIntegerTy ( <NUM_LIT> ) && ! Ty -> isIntegerTy ( <NUM_LIT> ) ) continue ; Reduction R ( & I ) ; if ( ! Search ( & I , R ) ) continue ; if ( ! CreateParallelPairs ( R ) ) continue ; InsertParallelMACs ( R ) ; Changed = true ; AllAdds . insert ( R . getAdds ( ) . begin ( ) , R . getAdds ( ) . end ( ) ) ; } return Changed


<s> bool runOnLoop ( Loop * TheLoop , LPPassManager & ) override { if ( DisableParallelDSP ) return false ; if ( skipLoop ( TheLoop ) ) return false ; L = TheLoop ; SE = & getAnalysis < ScalarEvolutionWrapperPass > ( ) . getSE ( ) ; AA = & getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; TLI = & getAnalysis < TargetLibraryInfoWrapperPass > ( ) . getTLI ( ) ; DT = & getAnalysis < DominatorTreeWrapperPass > ( ) . getDomTree ( ) ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; auto & TPC = getAnalysis < TargetPassConfig > ( ) ; BasicBlock * Header = TheLoop -> getHeader ( ) ; if ( ! Header ) return false ; if ( Header != TheLoop -> getLoopLatch ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } if ( ! TheLoop -> getLoopPreheader ( ) ) InsertPreheaderForLoop ( L , DT , LI , nullptr , true ) ; Function & F = * Header -> getParent ( ) ; M = F . getParent ( ) ; DL = & M -> getDataLayout ( ) ; auto & TM = TPC . getTM < TargetMachine > ( ) ; auto * ST = & TM . getSubtarget < <ISA_LIT>Subtarget > ( F ) ; if ( ! ST -> allowsUnalignedMem ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } if ( ! ST -> hasDSP ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } if ( ! ST -> isLittle ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return false ; } LoopAccessInfo LAI ( L , SE , TLI , AA , DT , LI ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << F .


<s> bool <ISA_LIT>ParallelDSP :: InsertParallelMACs ( Reduction & Reduction , PMACPairList & PMACPairs ) { Instruction * Acc = Reduction . Phi ; Instruction * InsertAfter = Reduction . AccIntAdd ; for ( auto & Pair : PMACPairs ) { BinOpChain * PMul0 = Pair . first ; BinOpChain * PMul1 = Pair . second ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; PMul0 -> Root -> dump ( ) ; dbgs ( ) << <STR_LIT> ; PMul1 -> Root -> dump ( ) ) ; auto * VecLd0 = cast < LoadInst > ( PMul0 -> VecLd [ <NUM_LIT> ] ) ; auto * VecLd1 = cast < LoadInst > ( PMul1 -> VecLd [ <NUM_LIT> ] ) ; Acc = CreateSMLADCall ( VecLd0 , VecLd1 , Acc , PMul1 -> Exchange , InsertAfter ) ; InsertAfter =


<s> dbgs ( ) << <STR_LIT> ; PMul1 -> Root -> dump ( ) ) ; auto * VecLd0 = cast < LoadInst > ( PMul0 -> VecLd [ <NUM_LIT> ] ) ; auto * VecLd1 = cast < LoadInst > ( PMul1 -> VecLd [ <NUM_LIT> ] ) ; Acc = CreateSMLADCall ( VecLd0 , VecLd1 , Acc , PMul1 -> Exchange , InsertAfter ) ; InsertAfter = Acc ; } if ( Acc != Reduction .


<s> M = F . getParent ( ) ; DL = & M -> getDataLayout ( ) ; auto & TM = TPC . getTM < TargetMachine > ( ) ; auto * ST = & TM . getSubtarget < <ISA_LIT>Subtarget > ( F ) ; if ( ! ST -> allowsUnalignedMem ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } if ( ! ST -> hasDSP ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } LoopAccessInfo LAI ( L , SE , TLI , AA , DT , LI ) ; bool Changes = false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << F . getName ( ) << <STR_LIT> ) ; Changes = MatchSMLAD ( F ) ; return Changes


<s> } } auto SafeToPair = [ & ] ( LoadInst * Base , LoadInst * Offset ) { LoadInst * Dominator = OrderedBB . dominates ( Base , Offset ) ? Base : Offset ; LoadInst * Dominated = OrderedBB . dominates ( Base , Offset ) ? Offset : Base ; if ( RAWDeps . count ( Dominated ) ) { InstSet & WritesBefore = RAWDeps [ Dominated ] ; for ( auto Before : WritesBefore ) { if ( OrderedBB . dominates ( Dominator , Before ) ) return false ; } } return true ; } ; for ( auto * Base : Loads ) { for ( auto * Offset : Loads ) { if ( Base == Offset || OffsetLoads . count ( Offset ) ) continue ; if ( AreSequentialAccesses < LoadInst > ( Base , Offset , * DL , * SE ) && SafeToPair ( Base , Offset ) ) { LoadPairs [ Base ] = Offset ; OffsetLoads . insert ( Offset ) ; break ; } } } LLVM_DEBUG ( if ( ! LoadPairs . empty ( ) ) { dbgs ( ) << <STR_LIT> ; for ( auto & MapIt :


<s> SmallVector < LoadInst * , <NUM_LIT> > Loads ; SmallVector < Instruction * , <NUM_LIT> > Writes ; LoadPairs . clear ( ) ; WideLoads . clear ( ) ; OrderedBasicBlock OrderedBB ( BB ) ; for ( auto & I : * BB ) { if ( I . mayWriteToMemory ( ) ) Writes . push_back ( & I ) ; auto * Ld = dyn_cast < LoadInst > ( & I ) ; if ( ! Ld || ! Ld -> isSimple ( ) || ! Ld -> hasOneUse ( ) || ! isa < SExtInst > ( Ld -> user_back ( ) ) ) continue ; Loads . push_back ( Ld ) ; } if ( Loads . empty ( ) || Loads . size ( ) > NumLoadLimit ) return false ; using InstSet = std :: set < Instruction * > ; using DepMap = std :: map < Instruction * , InstSet > ; DepMap RAWDeps ; const auto Size = LocationSize :: unknown ( ) ; for ( auto Write : Writes ) { for ( auto Read : Loads ) { MemoryLocation ReadLoc = MemoryLocation ( Read -> getPointerOperand ( ) , Size ) ; if ( ! isModOrRefSet ( intersectModRef ( AA -> getModRefInfo ( Write , ReadLoc ) , ModRefInfo :: ModRef ) ) ) continue ; if ( OrderedBB . dominates ( Write , Read ) ) RAWDeps [ Read ] . insert ( Write ) ; } } auto SafeToPair = [ & ] ( LoadInst * Base , LoadInst * Offset ) { LoadInst * Dominator = OrderedBB . dominates ( Base , Offset ) ? Base : Offset ; LoadInst * Dominated = OrderedBB . dominates ( Base , Offset ) ? Offset : Base ; if ( RAWDeps . count ( Dominated ) ) { InstSet & WritesBefore = RAWDeps [ Dominated ] ; for ( auto Before


<s> Acc = CreateSMLADCall ( VecLd0 , VecLd1 , Acc , InsertAfter ) ; InsertAfter = Acc ; } if ( Acc != Reduction . Phi ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; Acc -> dump ( ) ) ; Reduction . AccIntAdd -> replaceAllUsesWith ( Acc ) ; return true


<s> AddMACCandidate ( Candidates , Acc , MulOp0 , MulOp1 , <NUM_LIT> ) ; Acc = dyn_cast < Instruction > ( A ) ; } while ( match ( Acc , m_Add ( m_Value ( A ) , m_Mul ( m_Value ( MulOp0 ) , m_Value ( MulOp1 ) ) ) ) ) { AddMACCandidate ( Candidates , Acc , MulOp0 , MulOp1 , <NUM_LIT> ) ; Acc = dyn_cast < Instruction


<s> for ( auto & M : R . MACCandidates ) M -> Root -> dump ( ) ; dbgs ( ) << <STR_LIT> ; ) ; } Instructions Reads , Writes ; AliasCandidates ( Header , Reads , Writes ) ; for ( auto & R : Reductions ) { if ( AreAliased ( AA , Reads , Writes , R . MACCandidates ) ) return false ; PMACPairList PMACPairs = CreateParallelMACPairs ( R . MACCandidates


<s> if ( Header != TheLoop -> getLoopLatch ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } Function & F = * Header -> getParent ( ) ; M = F . getParent ( ) ; DL = & M -> getDataLayout ( ) ; auto & TM = TPC . getTM < TargetMachine > ( ) ; auto * ST = & TM . getSubtarget < <ISA_LIT>Subtarget > ( F ) ; if ( ! ST -> allowsUnalignedMem ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } if ( ! ST -> hasDSP ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } LoopAccessInfo LAI ( L , SE , TLI , AA , DT , LI ) ; bool Changes = false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Changes = MatchSMLAD (


<s> void SetMemoryLocations ( ) { const auto Size = MemoryLocation :: UnknownSize ; for ( auto * V : AllValues ) { if ( auto * I = dyn_cast < Instruction > ( V ) ) { if ( I -> mayWriteToMemory ( ) ) ReadOnly = false ; if ( auto * Ld = dyn_cast < LoadInst > ( V ) ) MemLocs . push_back (

MemoryLocation ( Ld -> getPointerOperand ( ) , Size ) ) ;


<s> static LoadInst * CreateLoadIns ( IRBuilder < NoFolder > & IRB , LoadInst & BaseLoad , Type * LoadTy ) { const unsigned AddrSpace = BaseLoad . getPointerAddressSpace (


<s> Value * Args [ ] = { NewLd0 , NewLd1 , Acc } ; Function * SMLAD = nullptr ; if ( Exchange ) SMLAD = Acc -> getType ( ) -> isIntegerTy ( <NUM_LIT> ) ? <STR_LIT> ( M , <STR_LIT> ) : <STR_LIT> ( M , <STR_LIT> ) ; else SMLAD = Acc -> getType ( ) -> isIntegerTy ( <NUM_LIT> ) ? <STR_LIT> ( M , <STR_LIT> ) : <STR_LIT> ( M , <STR_LIT> ) ; CallInst * Call = Builder .


<s> for ( auto & C : Candidates ) { if ( C -> size ( ) < <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } C -> PopulateLoads ( ) ; ValueList & LHS = static_cast < BinOpChain * > ( C . get ( ) ) -> LHS ; ValueList & RHS = static_cast < BinOpChain * > (


<s> for ( unsigned i = <NUM_LIT> , e = LHS . size ( ) ; i < e ; i += <NUM_LIT> ) { if ( ! isa < LoadInst > ( LHS [ i ] ) || ! isa < LoadInst > ( RHS [ i ] ) ) return false ; } } return


<s> LoadInst * WideLd1 = WideLoads . count ( VecLd1 [ <NUM_LIT> ] ) ? WideLoads [ VecLd1 [ <NUM_LIT> ] ] -> getLoad ( ) : CreateWideLoad ( VecLd1 , Ty ) ; Value * Args [ ] = { WideLd0 , WideLd1 , Acc } ; Function * SMLAD = nullptr ; if ( Exchange ) SMLAD = Acc -> getType ( ) -> isIntegerTy ( <NUM_LIT> ) ? <STR_LIT> ( M , <STR_LIT> ) : <STR_LIT> ( M , <STR_LIT> ) ; else SMLAD = Acc -> getType ( ) -> isIntegerTy ( <NUM_LIT> ) ? <STR_LIT> ( M , <STR_LIT> ) : <STR_LIT> ( M , <STR_LIT> ) ; IRBuilder < NoFolder > Builder ( InsertAfter -> getParent ( ) , ++ BasicBlock :: iterator ( InsertAfter ) ) ; CallInst * Call = Builder . CreateCall ( SMLAD , Args ) ; NumSMLAD


<s> Instruction * OffsetSExt = dyn_cast < SExtInst > ( Offset -> user_back ( ) ) ; assert ( ( BaseSExt && OffsetSExt ) && <STR_LIT> ) ; std :: function < void ( Value * , Value * ) > MoveBefore = [ & ] ( Value * A , Value * B ) -> void { if ( ! isa < Instruction > ( A ) || ! isa < Instruction > ( B ) ) return ; auto * Source = cast < Instruction > ( A ) ; auto * Sink = cast < Instruction > ( B ) ; if ( DT -> dominates ( Source , Sink ) || Source -> getParent ( ) != Sink -> getParent ( ) || isa < PHINode > ( Source ) || isa < PHINode > ( Sink ) ) return ; Source -> moveBefore ( Sink ) ; for ( auto & U : Source -> uses ( ) ) MoveBefore ( Source , U . getUser ( ) ) ; } ; LoadInst * DomLoad = DT -> dominates ( Base ,


<s> Instruction * InsertAfter = Reduction . AccIntAdd ; for ( auto & Pair : Reduction . PMACPairs ) { BinOpChain * PMul0 = Pair . first ; BinOpChain * PMul1 = Pair . second ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << * PMul0 -> Root << <STR_LIT> << <STR_LIT> << * PMul1 -> Root << <STR_LIT> ) ; Acc = CreateSMLADCall ( PMul0 -> VecLd , PMul1 -> VecLd , Acc , PMul1 -> Exchange , InsertAfter ) ; InsertAfter = Acc ; } if ( Acc != Reduction . Phi ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; Acc -> dump ( ) ) ; Reduction . AccIntAdd -> replaceAllUsesWith ( Acc ) ; return true ; } return


<s> if ( ! I ) return false ; if ( I -> getSrcTy ( ) -> getIntegerBitWidth ( ) != BitWidth


<s> if ( ! I ) return false ; if ( I -> getSrcTy ( ) -> getIntegerBitWidth ( ) != BitWidth ) return false ; return isa < LoadInst > ( I -> getOperand


<s> auto * I = dyn_cast < Instruction > ( V ) ; if ( ! I ) return false ; switch ( I -> getOpcode ( ) ) { case Instruction :: Add : if ( Match ( I -> getOperand ( <NUM_LIT> ) ) || ( Match ( I -> getOperand ( <NUM_LIT> ) ) ) ) return true ; break ; case Instruction :: Mul : { Value * Op0 = I -> getOperand ( <NUM_LIT> ) ; Value * Op1 = I -> getOperand ( <NUM_LIT> ) ; if ( IsExtendingLoad < SExtInst , <NUM_LIT> > ( Op0 ) && IsExtendingLoad < SExtInst , <NUM_LIT> > ( Op1 ) ) { ValueList LHS = { cast < SExtInst > ( Op0 ) -> getOperand ( <NUM_LIT> ) , Op0 } ; ValueList RHS = { cast < SExtInst > ( Op1 ) -> getOperand ( <NUM_LIT> ) , Op1 } ; Candidates . push_back ( make_unique < BinOpChain > ( I , LHS , RHS ) ) ; } return false ; } case Instruction :: SExt : return Match ( I -> getOperand ( <NUM_LIT> ) ) ; } return false ; } ; while ( Match ( Acc


<s> RecurrenceDescriptor RecDesc ; const bool HasFnNoNaNAttr = F . getFnAttribute ( <STR_LIT> ) . getValueAsString ( ) == <STR_LIT> ; BasicBlock * Latch = L -> getLoopLatch ( ) ; for ( PHINode & Phi : Latch -> phis ( ) ) { const auto * Ty = Phi . getType ( ) ; if ( ! Ty -> isIntegerTy ( <NUM_LIT> ) && ! Ty -> isIntegerTy ( <NUM_LIT> ) ) continue ; const bool IsReduction = RecurrenceDescriptor :: AddReductionVar ( & Phi , RecurrenceDescriptor :: RK_IntegerAdd , L , HasFnNoNaNAttr , RecDesc ) ; if ( ! IsReduction ) continue ; Instruction * Acc = dyn_cast < Instruction > ( Phi . getIncomingValueForBlock ( Latch ) ) ; if ( ! Acc ) continue ; Reductions . push_back ( Reduction ( & Phi , Acc ) ) ; } return ! Reductions . empty ( ) ; } ; ReductionList


<s> BasicBlock * Latch = L -> getLoopLatch ( ) ; for ( PHINode & Phi : Latch -> phis ( ) ) { const auto * Ty = Phi . getType ( ) ; if ( ! Ty -> isIntegerTy ( <NUM_LIT> ) && ! Ty -> isIntegerTy ( <NUM_LIT> ) ) continue ; const bool IsReduction = RecurrenceDescriptor :: AddReductionVar ( & Phi , RecurrenceDescriptor :: RK_IntegerAdd , L , HasFnNoNaNAttr , RecDesc ) ; if ( ! IsReduction ) continue ; Instruction * Acc = dyn_cast < Instruction > ( Phi . getIncomingValueForBlock ( Latch ) ) ; if ( ! Acc ) continue ; Reductions . push_back ( Reduction ( & Phi , Acc ) ) ; } return ! Reductions . empty ( ) ; } ; ReductionList Reductions ; if ( ! FindReductions ( Reductions )


<s> SmallVector < LoadInst * , <NUM_LIT> > Loads ; SmallVector < Instruction * , <NUM_LIT> > Writes ; for ( auto & I : * BB ) { if ( I . mayWriteToMemory ( ) ) Writes . push_back ( & I ) ; auto * Ld = dyn_cast < LoadInst > ( & I ) ; if ( ! Ld || ! Ld -> isSimple ( ) || ! Ld -> hasOneUse ( ) || ! isa < SExtInst > ( Ld -> user_back ( ) ) ) continue ; Loads . push_back ( Ld ) ; } using InstSet = std :: set < Instruction * > ; using DepMap = std :: map < Instruction * , InstSet > ; DepMap RAWDeps ; const auto Size = LocationSize :: unknown ( ) ; for ( auto Read : Loads ) { for ( auto Write : Writes ) { MemoryLocation ReadLoc = MemoryLocation ( Read -> getPointerOperand ( ) , Size ) ; if ( ! isModOrRefSet ( intersectModRef ( AA -> getModRefInfo ( Write , ReadLoc ) , ModRefInfo :: ModRef ) ) ) continue ; if ( DT -> dominates ( Write , Read ) ) RAWDeps [ Read ] . insert ( Write ) ; } } auto SafeToPair = [ & ] ( LoadInst * Base , LoadInst * Offset ) { LoadInst * Dominator = DT -> dominates ( Base , Offset ) ? Base : Offset ; LoadInst * Dominated = DT -> dominates ( Base , Offset ) ? Offset : Base ; if ( RAWDeps . count ( Dominated ) ) { InstSet & WritesBefore = RAWDeps [ Dominated ] ; for ( auto Before : WritesBefore ) { if ( DT -> dominates ( Dominator , Before ) ) return false ; } } return true ; } ; for ( auto * Base : Loads ) { for ( auto * Offset : Loads ) { if ( Base == Offset


<s> LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; auto & TPC = getAnalysis < TargetPassConfig > ( ) ; BasicBlock * Header = TheLoop -> getHeader ( ) ; if ( ! Header ) return false ; if ( Header != TheLoop -> getLoopLatch ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } if ( ! TheLoop -> getLoopPreheader ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } Function & F = * Header -> getParent ( ) ; M = F . getParent ( ) ; DL = & M -> getDataLayout ( ) ; auto & TM = TPC . getTM < TargetMachine > ( ) ; auto * ST = & TM . getSubtarget < <ISA_LIT>Subtarget > ( F ) ; if ( ! ST -> allowsUnalignedMem ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } if ( ! ST -> hasDSP ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } if ( ! ST -> isLittle ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ) ; return false ; } LoopAccessInfo LAI ( L , SE , TLI , AA , DT , LI ) ; LLVM_DEBUG ( dbgs ( ) <<


<s> static void AliasCandidates ( BasicBlock * Header , Instructions & Reads , Instructions & Writes ) { for ( auto & I


<s> if ( ! MemOp0 -> isSimple ( ) || ! MemOp1 -> isSimple ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( isConsecutiveAccess ( MemOp0 , MemOp1 , DL , SE ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return true


<s> if ( ! Ld0 || ! Ld1 ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; Ld0 -> dump ( ) ; dbgs ( ) << <STR_LIT> ; Ld1 -> dump ( ) ; ) ; if ( ! Ld0 -> hasOneUse ( ) || ! Ld1 -> hasOneUse ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false


<s> if ( ! Ld0 || ! Ld1 ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; Ld0 -> dump ( ) ; dbgs ( ) << <STR_LIT> ; Ld1 -> dump ( ) ; ) ; if ( ! Ld0 -> hasOneUse ( ) || ! Ld1 -> hasOneUse ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( ! LoadPairs . count ( Ld0 ) || LoadPairs [ Ld0 ] != Ld1 ) return false ; VecMem . clear (


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Pairs << <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> ; i < Pairs ; ++ i ) { const Value * V0 = VL0 [ i ] ; const Value * V1 = VL1 [ i ] ; const auto * Inst0 = dyn_cast < Instruction > ( V0 ) ; const auto * Inst1 = dyn_cast < Instruction > ( V1 ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << i << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; V0 -> dump ( ) ; dbgs ( ) << <STR_LIT> ; V1 -> dump ( ) ) ; if ( ! Inst0 || ! Inst1 ) return false ; if ( Inst0 -> isSameOperationAs ( Inst1 ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ; } const APInt * C0 , * C1 ; if ( ! ( match ( V0 , m_APInt ( C0 ) ) && match ( V1 , m_APInt ( C1 ) ) && C0 == C1 ) ) return false ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return true


<s> BinOpChain ( Instruction * I , ValueList & lhs , ValueList & rhs ) : OpChain ( I , lhs ) , LHS ( lhs ) , RHS


<s> BinOpChain ( Instruction * I , ValueList & lhs , ValueList &


<s> const Type * Ty = IntegerType :: get ( M -> getContext ( ) , <NUM_LIT> ) ; LoadInst * NewLd0 = CreateLoadIns ( Builder , VecLd0 [ <NUM_LIT> ] , Ty ) ; LoadInst * NewLd1 = CreateLoadIns ( Builder , VecLd1 [ <NUM_LIT> ] , Ty ) ; Value * Args [ ] = { NewLd0 , NewLd1 , Acc } ; Function * SMLAD = nullptr ; if ( Exchange ) SMLAD = Acc -> getType ( ) -> isIntegerTy ( <NUM_LIT> ) ? <STR_LIT> ( M , <STR_LIT> ) : <STR_LIT> ( M , <STR_LIT> ) ; else SMLAD = Acc -> getType ( ) -> isIntegerTy ( <NUM_LIT> ) ? <STR_LIT> ( M , <STR_LIT> ) : <STR_LIT> ( M , <STR_LIT> ) ; CallInst * Call = Builder . CreateCall ( SMLAD , Args ) ; NumSMLAD


<s> AU . addRequired < TargetLibraryInfoWrapperPass > ( ) ; AU . addRequired < LoopInfoWrapperPass > ( ) ; AU . addRequired < DominatorTreeWrapperPass > ( ) ; AU . addRequired < TargetPassConfig > ( ) ; AU . addPreserved < LoopInfoWrapperPass > ( ) ; AU . setPreservesCFG (


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { LoopPass :: getAnalysisUsage ( AU ) ; AU . addRequired < AssumptionCacheTracker > ( ) ; AU . addRequired < ScalarEvolutionWrapperPass > ( ) ; AU . addRequired < AAResultsWrapperPass > ( ) ; AU . addRequired < TargetLibraryInfoWrapperPass > ( ) ; AU . addRequired < LoopInfoWrapperPass > ( ) ; AU . addRequired < DominatorTreeWrapperPass > ( ) ; AU . addRequired < TargetPassConfig > ( ) ; AU . addPreserved < LoopInfoWrapperPass > ( ) ; AU . setPreservesCFG (


<s> } else if ( match ( V , m_Add ( m_Value ( LHS ) , m_Value ( RHS ) ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; I -> dump ( ) ) ; return false ; } else if ( match ( V , m_ZExtOrSExt ( m_Value ( Val ) ) ) ) { if ( cast < CastInst > ( I ) -> getSrcTy ( ) -> getIntegerBitWidth ( ) != MaxBitWidth ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << cast < CastInst > ( I ) -> getSrcTy ( ) -> getIntegerBitWidth ( ) << <STR_LIT> ) ; return false ; } if ( match ( Val , m_Load ( m_Value


<s> static void MatchParallelMACSequences ( Reduction & R , OpChainList & Candidates ) { Instruction * Acc = R . AccIntAdd ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Acc ) ; std :: function < bool ( Value * ) > Match = [ & Candidates , & Match ] ( Value * V ) -> bool { auto * I = dyn_cast < Instruction > ( V ) ; if ( ! I ) return false ; switch ( I -> getOpcode ( ) ) { case Instruction :: Add : if ( Match ( I -> getOperand ( <NUM_LIT> ) ) || ( Match ( I -> getOperand ( <NUM_LIT> ) ) ) ) return true ; break ; case Instruction :: Mul : { Value * MulOp0 = I -> getOperand ( <NUM_LIT> ) ; Value * MulOp1 = I -> getOperand ( <NUM_LIT> ) ; if ( isa < SExtInst > ( MulOp0 ) && isa < SExtInst > ( MulOp1 ) ) AddMACCandidate ( Candidates , I , MulOp0 , MulOp1 ) ; return


<s> case Instruction :: Add : if ( Match ( I -> getOperand ( <NUM_LIT> ) ) || ( Match ( I -> getOperand ( <NUM_LIT> ) ) ) ) return true ; break ; case Instruction :: Mul : { Value * MulOp0 = I -> getOperand ( <NUM_LIT> ) ; Value * MulOp1 = I -> getOperand ( <NUM_LIT> ) ; if ( isa < SExtInst > ( MulOp0 ) && isa < SExtInst > ( MulOp1 ) ) AddMACCandidate ( Candidates , I , MulOp0 , MulOp1 ) ; return false


<s> if ( ! TheLoop -> getLoopPreheader ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return ; } for ( PHINode & Phi : Header -> phis ( ) ) { const auto * Ty = Phi . getType ( ) ; if ( ! Ty -> isIntegerTy ( <NUM_LIT> ) && ! Ty -> isIntegerTy ( <NUM_LIT> ) ) continue ; const bool IsReduction = RecurrenceDescriptor :: AddReductionVar ( & Phi , RecurrenceDescriptor :: RK_IntegerAdd , TheLoop , HasFnNoNaNAttr , RecDesc ) ; if ( ! IsReduction ) continue ; Instruction * Acc = dyn_cast < Instruction > (

Phi . getIncomingValueForBlock ( Latch ) ) ;


<s> dbgs ( ) << <STR_LIT> ; Header -> dump ( ) ; dbgs ( ) << <STR_LIT> ; L -> dump ( ) ) ; bool Changed = false ; ReductionList Reductions ; MatchReductions ( F , L , Header , Reductions ) ; for ( auto & R : Reductions ) { OpChainList MACCandidates ; MatchParallelMACSequences ( R , MACCandidates ) ; if ( ! CheckMACMemory ( MACCandidates ) ) continue ; R . MACCandidates = std :: move ( MACCandidates ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; for ( auto & M : R . MACCandidates ) M -> Root -> dump ( ) ; dbgs ( ) << <STR_LIT> ; ) ; } Instructions Reads , Writes ; AliasCandidates ( Header , Reads , Writes ) ; for ( auto & R :


<s> for ( auto * Ld1 : Loads ) { if ( Ld0 == Ld1 ) continue ; if ( AreSequentialAccesses < LoadInst > ( Ld0 , Ld1 , * DL , * SE ) ) { LoadPairs [ Ld0 ] = Ld1 ; if ( BaseLoads . count ( Ld0 ) ) { LoadInst * Base = BaseLoads [ Ld0 ] ; BaseLoads [ Ld1 ] = Base ; SequentialLoads [ Base ] . push_back ( Ld1 ) ; } else


<s> for ( auto * Ld0 : Loads ) { for ( auto * Ld1 : Loads ) { if ( Ld0 == Ld1 ) continue ; if ( AreSequentialAccesses < LoadInst > ( Ld0 , Ld1 , * DL , * SE ) ) { LoadPairs [ Ld0 ] = Ld1 ; if ( BaseLoads . count ( Ld0 ) ) { LoadInst * Base = BaseLoads [ Ld0 ] ; BaseLoads [ Ld1 ] = Base ; SequentialLoads [ Base ] . push_back ( Ld1 ) ; } else { BaseLoads [ Ld1 ] = Ld0 ; SequentialLoads [ Ld0 ] . push_back ( Ld1 ) ; } } } } return LoadPairs . size ( ) >


<s> Reduction ( PHINode * P , Instruction * Acc ) : Phi


<s> Reduction ( PHINode * P , Instruction * Acc ) :


<s> for ( auto * V : AllValues ) { if ( auto * I = dyn_cast < Instruction > ( V ) ) { if ( I -> mayWriteToMemory (


<s> <ISA_LIT>PLT :: <ISA_LIT>PLT ( LDSection & pSection , <ISA_LIT>GOT & pGOTPLT ) : PLT ( pSection ) , m_GOT (


<s> <ISA_LIT>PLT1 * plt1_entry = new ( std :: nothrow


<s> iterator it = begin ( ) ; unsigned char * buffer = pRegion . begin ( ) ; memcpy ( buffer , llvm :: cast < <ISA_LIT>PLT0 > ( ( * it ) ) . getValue ( ) , <ISA_LIT>PLT0 :: EntrySize ) ; result += <ISA_LIT>PLT0 :: EntrySize ; ++ it ; <ISA_LIT>PLT1 * plt1 = <NUM_LIT> ; <ISA_LIT>PLT :: iterator ie = end (


<s> case G_MUL : case G_SDIV : case G_UDIV : case G_SEXT : case G_ZEXT : case G_TRUNC : case G_GEP : OperandsMapping = & <STR_LIT> [ <STR_LIT> ] ; break ; case G_LOAD : case G_STORE : OperandsMapping = Ty . getSizeInBits ( ) == <NUM_LIT> ? getOperandsMapping ( { & <STR_LIT> [ <STR_LIT> ] , & <STR_LIT> [ <STR_LIT> ] } ) : & <STR_LIT> [ <STR_LIT> ] ; break ; case G_FADD : assert ( ( Ty . getSizeInBits ( ) == <NUM_LIT> || Ty . getSizeInBits ( ) == <NUM_LIT> ) && <STR_LIT> ) ; OperandsMapping = Ty . getSizeInBits ( ) == <NUM_LIT> ? & <STR_LIT> [ <STR_LIT> ] : & <STR_LIT> [ <STR_LIT> ] ; break ; case G_CONSTANT : case G_FRAME_INDEX : OperandsMapping = getOperandsMapping ( { & <STR_LIT> [ <STR_LIT> ] , nullptr } ) ; break ; case G_SEQUENCE : { LLT Ty1 = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; LLT Ty2 = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( Ty . getSizeInBits ( ) != <NUM_LIT> || Ty1 . getSizeInBits ( ) != <NUM_LIT> || Ty2 . getSizeInBits ( ) != <NUM_LIT> ) return InstructionMapping { } ; OperandsMapping = getOperandsMapping ( { & <STR_LIT> [ <STR_LIT> ] , & <STR_LIT> [ <STR_LIT> ] , nullptr , & <STR_LIT> [ <STR_LIT> ] , nullptr } ) ; break ; } case G_EXTRACT : { LLT Ty1 = MRI . getType ( MI . getOperand (


<s> assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . getSize (


<s> AlreadyInit = true ; const RegisterBank & RBGPR = getRegBank ( <STR_LIT> ) ; ( void ) RBGPR ; assert ( & <STR_LIT> == & RBGPR && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass


<s> using namespace TargetOpcode ; unsigned NumOperands = MI . getNumOperands ( ) ; const ValueMapping * OperandsMapping = & <STR_LIT> [ <NUM_LIT> ] ; switch ( Opc ) { case G_ADD : case G_LOAD : case G_SEXT : case G_ZEXT : OperandsMapping = & <STR_LIT> [ <NUM_LIT> ] ; break ; case G_FRAME_INDEX : OperandsMapping = getOperandsMapping ( { & <STR_LIT> [ <NUM_LIT> ]


<s> static llvm :: once_flag InitializeRegisterBankFlag ; static auto InitializeRegisterBankOnce = [ & ] ( ) { const RegisterBank & RBGPR = getRegBank ( <STR_LIT> ) ; ( void ) RBGPR ; assert ( & <STR_LIT> == & RBGPR && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT>


<s> ( void ) RBGPR ; assert ( & <STR_LIT> == & RBGPR && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . getSize ( ) == <NUM_LIT> && <STR_LIT> ) ; <STR_LIT> (


<s> const RegisterBank & <ISA_LIT>RegisterBankInfo :: getRegBankFromRegClass ( const TargetRegisterClass & RC , LLT ) const { using namespace <ISA_LIT> ; switch ( RC . getID ( ) ) { case GPRRegClassID : case GPRwithAPSRRegClassID : case GPRnoipRegClassID : case GPRnopcRegClassID : case GPRnoip_and_GPRnopcRegClassID : case rGPRRegClassID : case GPRspRegClassID : case GPRnoip_and_tcGPRRegClassID : case tcGPRRegClassID : case


<s> assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . getSize ( ) == <NUM_LIT> && <STR_LIT> ) ; <STR_LIT> (


<s> assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . getSize ( ) == <NUM_LIT>


<s> case tcGPRRegClassID : case tGPRRegClassID : case tGPREvenRegClassID : case tGPROddRegClassID : case tGPR_and_tGPREvenRegClassID : case tGPR_and_tGPROddRegClassID : case tGPREven_and_tcGPRRegClassID : case tGPREven_and_tGPR_and_tcGPRRegClassID : case tGPROdd_and_tcGPRRegClassID : return getRegBank ( <STR_LIT> ) ; case HPRRegClassID : case


<s> case GPRnopcRegClassID : case tGPR_and_tcGPRRegClassID : case tGPRRegClassID : return getRegBank ( <STR_LIT> ) ; case SPR_8RegClassID : case SPRRegClassID : case DPR_8RegClassID : case DPRRegClassID : return getRegBank ( <STR_LIT> ) ; default : llvm_unreachable ( <STR_LIT> ) ; } llvm_unreachable ( <STR_LIT> )


<s> static llvm :: once_flag InitializeRegisterBankFlag ; static auto InitializeRegisterBankOnce = [ & ] ( ) { const RegisterBank & RBGPR = getRegBank ( <STR_LIT> ) ; ( void ) RBGPR ; assert ( & <STR_LIT> == & RBGPR && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass


<s> case tGPR_and_tcGPRRegClassID : case tGPRRegClassID : return getRegBank ( <STR_LIT> ) ; case SPR_8RegClassID : case SPRRegClassID : case DPR_8RegClassID : case DPRRegClassID : return getRegBank ( <STR_LIT>


<s> RegisterBankInfo :: InstructionMapping <ISA_LIT>RegisterBankInfo :: getInstrMapping ( const MachineInstr & MI ) const { auto Opc = MI . getOpcode ( ) ; if ( ! isPreISelGenericOpcode ( Opc ) ) { InstructionMapping Mapping = getInstrMappingImpl ( MI ) ; if ( Mapping . isValid ( ) ) return Mapping ; } using namespace TargetOpcode ; unsigned NumOperands = MI . getNumOperands ( ) ; const ValueMapping * OperandsMapping = & <STR_LIT> [


<s> case GPRRegClassID : case tGPR_and_tcGPRRegClassID : return getRegBank ( <STR_LIT> ) ; default : llvm_unreachable ( <STR_LIT> ) ; } llvm_unreachable ( <STR_LIT>


<s> const MachineFunction & MF = * MI . getParent ( ) -> getParent ( ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; LLT Ty = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; unsigned NumOperands = MI . getNumOperands ( ) ; const ValueMapping * OperandsMapping = & <STR_LIT> [ <STR_LIT> ] ; switch ( Opc ) { case G_ADD : case G_SEXT : case G_ZEXT : case G_GEP : OperandsMapping = & <STR_LIT> [ <STR_LIT> ] ; break ; case G_LOAD : case G_STORE : OperandsMapping = Ty . getSizeInBits ( ) == <NUM_LIT> ? getOperandsMapping ( { & <STR_LIT> [ <STR_LIT> ] , & <STR_LIT> [ <STR_LIT> ] } ) : & <STR_LIT> [ <STR_LIT> ] ; break ; case G_FADD : assert ( ( Ty . getSizeInBits ( ) == <NUM_LIT> || Ty . getSizeInBits ( ) == <NUM_LIT> ) && <STR_LIT> ) ; OperandsMapping = Ty . getSizeInBits ( ) == <NUM_LIT> ? & <STR_LIT> [ <STR_LIT> ] : & <STR_LIT> [ <STR_LIT> ] ; break ; case G_CONSTANT : case G_FRAME_INDEX : OperandsMapping = getOperandsMapping ( { & <STR_LIT> [ <STR_LIT> ] , nullptr } ) ; break ; case G_SEQUENCE : { LLT Ty1 = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; LLT Ty2 = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( Ty . getSizeInBits ( ) != <NUM_LIT> || Ty1 . getSizeInBits ( ) != <NUM_LIT> || Ty2 . getSizeInBits ( ) != <NUM_LIT> ) return InstructionMapping { } ; OperandsMapping = getOperandsMapping ( { & <STR_LIT> [ <STR_LIT> ] , & <STR_LIT> [ <STR_LIT> ] , nullptr , & <STR_LIT> [ <STR_LIT> ] , nullptr } ) ; break ; } case G_EXTRACT : { LLT Ty1 = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( Ty . getSizeInBits ( ) != <NUM_LIT> || Ty1 . getSizeInBits ( ) != <NUM_LIT> || MI . getOperand ( <NUM_LIT> ) . getImm ( ) % <NUM_LIT> != <NUM_LIT> ) return InstructionMapping { } ; OperandsMapping = getOperandsMapping ( { & <STR_LIT> [ <STR_LIT> ] , & <STR_LIT> [ <STR_LIT> ] , nullptr , nullptr } ) ; break ; } default : return InstructionMapping { } ; } for ( unsigned i = <NUM_LIT> ; i < NumOperands ; i ++ ) { for ( const auto & Mapping : OperandsMapping [ i ]


<s> switch ( Opc ) { case G_ADD : case G_SEXT : case G_ZEXT : OperandsMapping = & <STR_LIT> [ <STR_LIT> ] ; break ; case G_LOAD : OperandsMapping = Ty . getSizeInBits ( ) == <NUM_LIT> ? getOperandsMapping ( { & <STR_LIT> [ <STR_LIT> ] , & <STR_LIT> [ <STR_LIT> ] } ) : & <STR_LIT> [ <STR_LIT> ] ; break ; case G_FADD : assert ( ( Ty . getSizeInBits ( ) == <NUM_LIT> || Ty . getSizeInBits ( ) == <NUM_LIT> ) && <STR_LIT> ) ; OperandsMapping = Ty . getSizeInBits ( ) == <NUM_LIT> ? & <STR_LIT> [ <STR_LIT> ] : & <STR_LIT> [ <STR_LIT> ] ; break ; case G_FRAME_INDEX : OperandsMapping = getOperandsMapping ( { & <STR_LIT> [ <STR_LIT> ] , nullptr } ) ; break ; case G_SEQUENCE : { LLT Ty1 = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; LLT Ty2 = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( Ty . getSizeInBits ( ) != <NUM_LIT> || Ty1 . getSizeInBits ( ) != <NUM_LIT> || Ty2 . getSizeInBits ( ) != <NUM_LIT> ) return InstructionMapping { } ; OperandsMapping = getOperandsMapping ( { & <STR_LIT> [ <STR_LIT> ] , & <STR_LIT> [ <STR_LIT> ] ,

nullptr , & <STR_LIT> [ <STR_LIT> ] , nullptr } ) ;


<s> case GPRnopcRegClassID : case tGPR_and_tcGPRRegClassID : return getRegBank ( <STR_LIT> ) ; case SPR_8RegClassID : case SPRRegClassID : case DPR_8RegClassID : case DPRRegClassID : return getRegBank ( <STR_LIT> ) ; default : llvm_unreachable ( <STR_LIT> ) ; } llvm_unreachable (


<s> OperandsMapping = getOperandsMapping ( { & <STR_LIT> [ <STR_LIT> ] , nullptr , & <STR_LIT> [ <STR_LIT> ] , & <STR_LIT> [ <STR_LIT> ] } ) ; break ; } case G_MERGE_VALUES : { LLT Ty1 = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; LLT Ty2 = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( Ty . getSizeInBits ( ) != <NUM_LIT> || Ty1 . getSizeInBits ( ) != <NUM_LIT> || Ty2 . getSizeInBits ( ) != <NUM_LIT> ) return getInvalidInstructionMapping ( ) ; OperandsMapping = getOperandsMapping ( { & <STR_LIT> [ <STR_LIT> ] , & <STR_LIT> [ <STR_LIT> ] , & <STR_LIT> [ <STR_LIT> ] } ) ; break ; } case G_UNMERGE_VALUES : { LLT Ty1 = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; LLT Ty2 = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( Ty . getSizeInBits ( ) != <NUM_LIT> || Ty1 . getSizeInBits ( ) != <NUM_LIT> || Ty2 . getSizeInBits ( ) != <NUM_LIT> ) return getInvalidInstructionMapping ( ) ; OperandsMapping = getOperandsMapping ( { & <STR_LIT> [ <STR_LIT> ] , & <STR_LIT> [ <STR_LIT> ] , & <STR_LIT> [ <STR_LIT> ] } ) ; break ; } default : return getInvalidInstructionMapping (


<s> break ; case G_SELECT : { LLT Ty2 = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; ( void ) Ty2 ; assert ( Ty . getSizeInBits ( ) == <NUM_LIT> && <STR_LIT> ) ; assert ( Ty2 . getSizeInBits ( ) == <NUM_LIT> && <STR_LIT> ) ; OperandsMapping = getOperandsMapping ( { & <STR_LIT> [ <STR_LIT> ] , & <STR_LIT> [ <STR_LIT> ] , & <STR_LIT> [ <STR_LIT> ] , & <STR_LIT> [ <STR_LIT> ] } ) ; break ; } case G_ICMP : { LLT Ty2 = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; ( void ) Ty2 ; assert ( Ty2 . getSizeInBits ( ) == <NUM_LIT> && <STR_LIT> ) ; OperandsMapping = getOperandsMapping ( { & <STR_LIT> [ <STR_LIT> ] , nullptr , & <STR_LIT> [ <STR_LIT> ] , & <STR_LIT> [ <STR_LIT> ] } ) ; break ; } case G_FCMP : { LLT Ty1 = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; ( void ) Ty1 ; LLT Ty2 = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; ( void ) Ty2 ; assert ( Ty . getSizeInBits ( ) == <NUM_LIT> && <STR_LIT> ) ; assert ( Ty1 . getSizeInBits ( ) == <NUM_LIT> && <STR_LIT> ) ; assert ( Ty2 . getSizeInBits ( ) == <NUM_LIT> && <STR_LIT> ) ; OperandsMapping = getOperandsMapping ( { & <STR_LIT> [ <STR_LIT> ] , nullptr , & <STR_LIT> [ <STR_LIT> ] , & <STR_LIT> [ <STR_LIT> ] } ) ; break ; } case G_MERGE_VALUES : { LLT Ty1 = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; LLT Ty2 = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( Ty . getSizeInBits ( ) != <NUM_LIT> || Ty1 . getSizeInBits ( ) != <NUM_LIT> || Ty2 . getSizeInBits ( ) != <NUM_LIT> ) return getInvalidInstructionMapping ( ) ; OperandsMapping = getOperandsMapping ( { & <STR_LIT> [ <STR_LIT> ] , & <STR_LIT> [ <STR_LIT> ] , & <STR_LIT> [ <STR_LIT> ] } ) ; break ; } case G_UNMERGE_VALUES : { LLT Ty1 = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; LLT Ty2 = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( Ty . getSizeInBits ( ) != <NUM_LIT> || Ty1 . getSizeInBits ( ) != <NUM_LIT> || Ty2 . getSizeInBits ( ) !=


<s> const RegisterBank & RBGPR = getRegBank ( <STR_LIT> ) ; ( void ) RBGPR ; assert ( & <STR_LIT> == & RBGPR && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT> ) ) && <STR_LIT> ) ; assert ( RBGPR . covers ( * TRI . getRegClass ( <STR_LIT>