<s> getTTypeGlobalReference ( const GlobalValue * GV , Mangler * Mang , MachineModuleInfo * MMI , unsigned Encoding , MCStreamer & Streamer ) const {


<s> if ( T . isOSNaCl ( ) ) return TargetLoweringObjectFileELF :: getTTypeGlobalReference ( GV , Mang , MMI , Encoding , Streamer ) ; return MCSymbolRefExpr :: Create ( getSymbol ( * Mang , GV ) , MCSymbolRefExpr :: VK_<ISA_LIT>_TARGET2 , getContext ( ) ) ;


<s> if ( TM . getMCAsmInfo ( ) -> getExceptionHandlingType ( ) != ExceptionHandling :: <ISA_LIT> ) return TargetLoweringObjectFileELF :: getTTypeGlobalReference ( GV , Encoding , Mang , TM , MMI , Streamer ) ; assert ( Encoding == DW_EH_PE_absptr && <STR_LIT> ) ; Triple T ( MMI -> getModule ( ) -> getTargetTriple ( ) ) ; if ( T . isOSNaCl ( ) ) return TargetLoweringObjectFileELF :: getTTypeGlobalReference ( GV , Encoding , Mang , TM , MMI , Streamer ) ; return MCSymbolRefExpr :: Create ( TM . getSymbol ( GV , Mang ) , MCSymbolRefExpr :: VK_<ISA_LIT>_TARGET2 , getContext ( ) ) ;


<s> const MCExpr * <ISA_LIT>ElfTargetObjectFile :: getTTypeGlobalReference ( const GlobalValue * GV , unsigned Encoding , Mangler & Mang , const TargetMachine & TM , MachineModuleInfo * MMI , MCStreamer & Streamer ) const { if ( TM . getMCAsmInfo ( ) -> getExceptionHandlingType ( ) != ExceptionHandling :: <ISA_LIT> ) return TargetLoweringObjectFileELF :: getTTypeGlobalReference ( GV , Encoding , Mang , TM , MMI , Streamer ) ;

assert ( Encoding == DW_EH_PE_absptr && <STR_LIT> ) ;


<s> void <ISA_LIT>ElfTargetObjectFile :: Initialize ( MCContext & Ctx , const TargetMachine & TM ) { bool isAAPCS_ABI = static_cast < const <ISA_LIT>TargetMachine & > ( TM ) . TargetABI == <ISA_LIT>TargetMachine :: <STR_LIT> ; TargetLoweringObjectFileELF :: Initialize ( Ctx , TM ) ;


<s> if ( isAAPCS_ABI && ! Triple ( TM . getTargetTriple ( ) ) . isOSNaCl ( ) ) { LSDASection = nullptr ; } AttributesSection = getContext ( ) . getELFSection ( <STR_LIT> , ELF :: SHT_<ISA_LIT>_ATTRIBUTES , <NUM_LIT> ) ;


<s> unsigned <ISA_LIT>MachOTargetObjectFile :: getTTypeEncoding ( ) const {


<s> void <ISA_LIT>MachOTargetObjectFile :: Initialize ( MCContext & Ctx , const TargetMachine & TM ) { TargetLoweringObjectFileMachO :: Initialize ( Ctx , TM ) ;

LSDASection = getMachOSection ( <STR_LIT> , <STR_LIT> , <NUM_LIT> , SectionKind :: getReadOnlyWithRel ( ) ) ;


<s> TargetLoweringObjectFileELF :: Initialize ( Ctx , TM ) ; InitializeELF ( isAAPCS_ABI ) ; if ( isAAPCS_ABI && ! TM . getSubtarget < <ISA_LIT>Subtarget > ( ) . isTargetNaCl ( ) ) { LSDASection = nullptr ;


<s> const MCRegister <ISA_LIT>ElfTargetObjectFile :: getStaticBase ( ) const { return <STR_LIT> ;


<s> virtual const MCSection * getAttributesSection ( ) const { return AttributesSection ;


<s> virtual const MCSection * getAttributesSection ( ) const {


<s> const <ISA_LIT>BaseTargetMachine & <ISA_LIT>_TM = static_cast < const <ISA_LIT>BaseTargetMachine & > ( TM ) ; bool isAAPCS_ABI = <ISA_LIT>_TM . TargetABI == <ISA_LIT>BaseTargetMachine :: <STR_LIT> ;

TargetLoweringObjectFileELF :: Initialize ( Ctx , TM ) ;


<s> const <ISA_LIT>BaseTargetMachine & <ISA_LIT>_TM = static_cast < const <ISA_LIT>BaseTargetMachine & > ( TM ) ; bool isAAPCS_ABI = <ISA_LIT>_TM . TargetABI == <ISA_LIT>BaseTargetMachine :: <STR_LIT> ; TargetLoweringObjectFileELF :: Initialize ( Ctx , TM ) ; InitializeELF ( isAAPCS_ABI ) ; if ( isAAPCS_ABI ) { LSDASection = nullptr ; }


<s> <ISA_LIT>ElfTargetObjectFile ( ) { PLTRelativeVariantKind = MCSymbolRefExpr :: VK_<ISA_LIT>_PREL31 ;


<s> <ISA_LIT>ElfTargetObjectFile ( ) : TargetLoweringObjectFileELF ( ) {

PLTRelativeVariantKind = MCSymbolRefExpr :: VK_<ISA_LIT>_PREL31 ;


<s> const MCExpr * <ISA_LIT>ElfTargetObjectFile :: getDebugThreadLocalSymbol ( const MCSymbol * Sym ) const {

return MCSymbolRefExpr :: Create ( Sym , MCSymbolRefExpr :: VK_<ISA_LIT>_TLSLDO , getContext ( ) ) ;


<s> const MCExpr * <ISA_LIT>ElfTargetObjectFile :: getDebugThreadLocalSymbol ( const MCSymbol * Sym ) const { return MCSymbolRefExpr :: Create ( Sym , MCSymbolRefExpr :: VK_<ISA_LIT>_TLSLDO , getContext ( ) ) ;


<s> if ( TM . getMCAsmInfo ( ) -> getExceptionHandlingType ( ) != ExceptionHandling :: <ISA_LIT> ) return TargetLoweringObjectFileELF :: getTTypeGlobalReference ( GV , Encoding , Mang , TM , MMI , Streamer ) ; assert ( Encoding == DW_EH_PE_absptr && <STR_LIT> ) ; return MCSymbolRefExpr :: Create ( TM . getSymbol ( GV , Mang ) , MCSymbolRefExpr :: VK_<ISA_LIT>_TARGET2 , getContext ( ) ) ;


<s> if ( TM . getMCAsmInfo ( ) -> getExceptionHandlingType ( ) != ExceptionHandling :: <ISA_LIT> ) return TargetLoweringObjectFileELF :: getTTypeGlobalReference ( GV , Encoding , Mang , TM , MMI , Streamer ) ; assert ( Encoding == DW_EH_PE_absptr && <STR_LIT> ) ;

return MCSymbolRefExpr :: Create ( TM . getSymbol ( GV , Mang ) , MCSymbolRefExpr :: VK_<ISA_LIT>_TARGET2 , getContext ( ) ) ;


<s> void <ISA_LIT>ElfTargetObjectFile :: Initialize ( MCContext & Ctx , const TargetMachine & TM ) { bool isAAPCS_ABI = TM . getSubtarget < <ISA_LIT>Subtarget > ( ) . isAAPCS_ABI ( ) ;


<s> if ( TT . isOSBinFormatELF ( ) ) return create<ISA_LIT>ObjectTargetELFStreamer ( S ) ;


<s> void <ISA_LIT>TargetStreamer :: emitArch ( <STR_LIT> Arch ) {


<s> void <ISA_LIT>TargetStreamer :: emitArchExtension ( uint64_t ArchExt ) {


<s> void <ISA_LIT>TargetStreamer :: emit<ISA_LIT>WinCFIAllocStack ( unsigned Size , bool Wide ) {


<s> void <ISA_LIT>TargetStreamer :: emit<ISA_LIT>WinCFICustom ( unsigned Opcode ) {


<s> void <ISA_LIT>TargetStreamer :: emit<ISA_LIT>WinCFIEpilogEnd ( ) {


<s> void <ISA_LIT>TargetStreamer :: emit<ISA_LIT>WinCFIEpilogStart ( unsigned Condition ) {


<s> void <ISA_LIT>TargetStreamer :: emit<ISA_LIT>WinCFINop ( bool Wide ) {


<s> void <ISA_LIT>TargetStreamer :: emit<ISA_LIT>WinCFIPrologEnd ( bool Fragment ) {


<s> void <ISA_LIT>TargetStreamer :: emit<ISA_LIT>WinCFISaveFRegs ( unsigned First , unsigned Last ) {


<s> void <ISA_LIT>TargetStreamer :: emit<ISA_LIT>WinCFISaveLR ( unsigned Offset ) {


<s> void <ISA_LIT>TargetStreamer :: emit<ISA_LIT>WinCFISaveRegMask ( unsigned Mask , bool Wide ) {


<s> void <ISA_LIT>TargetStreamer :: emit<ISA_LIT>WinCFISaveSP ( unsigned Reg ) {


<s> ConstantPools -> emitAll ( Streamer ) ;


<s> void <ISA_LIT>TargetStreamer :: emitConstantPools ( ) {


<s> ConstantPools -> emitForCurrentSection ( Streamer ) ;

ConstantPools -> clearCacheForCurrentSection ( Streamer ) ;


<s> Size = ( Suffix == 'n' ? <NUM_LIT> : <NUM_LIT> ) ; for ( unsigned II = <NUM_LIT> , IE = Size ; II != IE ; II = II + <NUM_LIT> ) { const unsigned I0 = LittleEndian ? II + <NUM_LIT> : II + <NUM_LIT> ; const unsigned I1 = LittleEndian ? II + <NUM_LIT> : II + <NUM_LIT> ; Buffer [ Size - II - <NUM_LIT> ] = uint8_t ( Inst >> I0 * CHAR_BIT ) ; Buffer [ Size - II - <NUM_LIT> ] = uint8_t ( Inst >> I1 * CHAR_BIT ) ; }


<s> void <ISA_LIT>TargetStreamer :: emitObjectArch ( <STR_LIT> Arch ) {


<s> else if ( STI . hasFeature ( <STR_LIT> ) ) { if ( STI . hasFeature ( <STR_LIT> ) ) return <STR_LIT> ; return <STR_LIT> ; } else if ( STI . hasFeature ( <STR_LIT> ) ) return <STR_LIT> ; else if ( STI . hasFeature ( <STR_LIT> ) ) return <STR_LIT> ; else if ( STI . hasFeature ( <STR_LIT> ) ) { if ( STI . hasFeature ( <STR_LIT> ) && STI . hasFeature ( <STR_LIT> ) ) return <STR_LIT> ; return <STR_LIT> ; } else if ( STI . hasFeature ( <STR_LIT> ) ) return <STR_LIT> ;


<s> return ( STI . hasFeature ( <STR_LIT> ) && ! STI . hasFeature ( <STR_LIT> ) ) || STI . hasFeature ( <STR_LIT> ) ;


<s> void <ISA_LIT>TargetStreamer :: reset ( ) {


<s> void <ISA_LIT>TargetStreamer :: AnnotateTLSDescriptorSequence ( const MCSymbolRefExpr * SRE ) {


<s> <ISA_LIT>TargetStreamer :: <ISA_LIT>TargetStreamer ( MCStreamer & S ) : MCTargetStreamer ( S ) , ConstantPools ( new AssemblerConstantPools ( ) ) {


<s> void <ISA_LIT>TargetStreamer :: emitArch ( unsigned Arch ) {


<s> void <ISA_LIT>TargetStreamer :: emitArchExtension ( unsigned ArchExt ) {


<s> void <ISA_LIT>TargetStreamer :: emitAttribute ( unsigned Attribute , unsigned Value ) {


<s> void <ISA_LIT>TargetStreamer :: emitCantUnwind ( ) {


<s> void <ISA_LIT>TargetStreamer :: emitCurrentConstantPool ( ) { ConstantPools -> emitForCurrentSection ( Streamer ) ;


<s> void <ISA_LIT>TargetStreamer :: emitFnEnd ( ) {


<s> void <ISA_LIT>TargetStreamer :: emitFnStart ( ) {


<s> void <ISA_LIT>TargetStreamer :: emitFPU ( unsigned FPU ) {


<s> void <ISA_LIT>TargetStreamer :: emitHandlerData ( ) {


<s> void <ISA_LIT>TargetStreamer :: emitInst ( uint32_t Inst , char Suffix ) {


<s> void <ISA_LIT>TargetStreamer :: emitIntTextAttribute ( unsigned Attribute , unsigned IntValue , StringRef StringValue ) {


<s> void <ISA_LIT>TargetStreamer :: emitMovSP ( unsigned Reg , int64_t Offset ) {


<s> void <ISA_LIT>TargetStreamer :: emitObjectArch ( unsigned Arch ) {


<s> void <ISA_LIT>TargetStreamer :: emitPad ( int64_t Offset ) {


<s> void <ISA_LIT>TargetStreamer :: emitPersonality ( const MCSymbol * Personality ) {


<s> void <ISA_LIT>TargetStreamer :: emitPersonalityIndex ( unsigned Index ) {


<s> void <ISA_LIT>TargetStreamer :: emitRegSave ( const SmallVectorImpl < unsigned > & RegList , bool isVector ) {


<s> void <ISA_LIT>TargetStreamer :: emitSetFP ( unsigned FpReg , unsigned SpReg , int64_t Offset ) {


<s> void <ISA_LIT>TargetStreamer :: emitTextAttribute ( unsigned Attribute , StringRef String ) {


<s> void <ISA_LIT>TargetStreamer :: emitThumbSet ( MCSymbol * Symbol , const MCExpr * Value ) {


<s> void <ISA_LIT>TargetStreamer :: emitUnwindRaw ( int64_t StackOffset , const SmallVectorImpl < uint8_t > & Opcodes ) {


<s> ConstantPools -> emitAll ( Streamer ) ;


<s> void <ISA_LIT>TargetStreamer :: finishAttributeSection ( ) {


<s> void <ISA_LIT>TargetStreamer :: switchVendor ( StringRef Vendor ) {


<s> <ISA_LIT>TargetStreamer :: ~ <ISA_LIT>TargetStreamer ( ) {


<s> return ConstantPools -> addEntry ( Streamer , Expr ) ;


<s> const MCExpr * AssemblerConstantPools :: addEntry ( MCStreamer & Streamer , const MCExpr * Expr ) { const MCSection * Section = Streamer . getCurrentSection ( ) . first ;

return getOrCreateConstantPool ( Section ) . addEntry ( Expr , Streamer . getContext ( ) ) ;


<s> void <ISA_LIT>TargetStreamer :: AnnotateTLSDescriptorSequence ( const MCSymbolRefExpr * SRE ) {


<s> for ( ConstantPoolMapTy :: iterator CPI = ConstantPools . begin ( ) , CPE = ConstantPools . end ( ) ;


<s> void AssemblerConstantPools :: emitAll ( MCStreamer & Streamer ) { for ( ConstantPoolMapTy :: iterator CPI = ConstantPools . begin ( ) , CPE = ConstantPools . end ( ) ; CPI != CPE ; ++ CPI ) { const MCSection * Section = CPI -> first ; ConstantPool & CP = CPI -> second ; emitConstantPool ( Streamer , Section , CP ) ; }


<s> void <ISA_LIT>TargetStreamer :: emitArch ( unsigned Arch ) { llvm_unreachable ( <STR_LIT> ) ;


<s> static void emitConstantPool ( MCStreamer & Streamer , const MCSection * Section , ConstantPool & CP ) { if ( ! CP . empty ( ) ) { Streamer . SwitchSection ( Section ) ; CP . emitEntries ( Streamer ) ; }


<s> Streamer . EmitCodeAlignment ( <NUM_LIT> ) ; Streamer . EmitDataRegion ( MCDR_DataRegion ) ; for ( EntryVecTy :: const_iterator I = Entries . begin ( ) , E = Entries . end ( ) ; I != E ; ++ I ) { Streamer . EmitLabel ( I -> first ) ;

Streamer . EmitValue ( I -> second , <NUM_LIT> ) ;


<s> void <ISA_LIT>TargetStreamer :: emitFnEnd ( ) { llvm_unreachable ( <STR_LIT> ) ;


<s> void <ISA_LIT>TargetStreamer :: emitFnStart ( ) {


<s> const MCSection * Section = Streamer . getCurrentSection ( ) . first ;

if ( ConstantPool * CP = getConstantPool ( Section ) ) {


<s> void AssemblerConstantPools :: emitForCurrentSection ( MCStreamer & Streamer ) { const MCSection * Section = Streamer . getCurrentSection ( ) . first ; if ( ConstantPool * CP = getConstantPool ( Section ) ) { emitConstantPool ( Streamer , Section , * CP ) ; }


<s> void <ISA_LIT>TargetStreamer :: emitHandlerData ( ) { llvm_unreachable ( <STR_LIT> ) ;


<s> void <ISA_LIT>TargetStreamer :: emitInst ( uint32_t Inst , char Suffix ) {


<s> void <ISA_LIT>TargetStreamer :: emitIntTextAttribute ( unsigned Attribute , unsigned IntValue , StringRef StringValue ) {


<s> void <ISA_LIT>TargetStreamer :: emitIntTextAttribute ( unsigned Attribute , unsigned IntValue , StringRef StringValue ) { llvm_unreachable ( <STR_LIT> ) ;


<s> void <ISA_LIT>TargetStreamer :: emitMovSP ( unsigned Reg , int64_t Offset ) {


<s> void <ISA_LIT>TargetStreamer :: emitObjectArch ( unsigned Arch ) {


<s> void <ISA_LIT>TargetStreamer :: emitPersonality ( const MCSymbol * Personality ) {


<s> void <ISA_LIT>TargetStreamer :: emitPersonalityIndex ( unsigned Index ) {


<s> void <ISA_LIT>TargetStreamer :: emitRegSave ( const SmallVectorImpl < unsigned > & RegList , bool isVector ) { llvm_unreachable ( <STR_LIT> ) ;


<s> void <ISA_LIT>TargetStreamer :: emitSetFP ( unsigned FpReg , unsigned SpReg , int64_t Offset ) { llvm_unreachable ( <STR_LIT> ) ;


<s> void <ISA_LIT>TargetStreamer :: emitUnwindRaw ( int64_t StackOffset , const SmallVectorImpl < uint8_t > & Opcodes ) { llvm_unreachable ( <STR_LIT> ) ;


<s> void <ISA_LIT>TargetStreamer :: emitUnwindRaw ( int64_t StackOffset , const SmallVectorImpl < uint8_t > & Opcodes ) {


<s> bool ConstantPool :: empty ( ) { return Entries . empty ( ) ;


<s> void <ISA_LIT>TargetStreamer :: finishAttributeSection ( ) { llvm_unreachable ( <STR_LIT> ) ;


<s> void <ISA_LIT>TargetStreamer :: finishAttributeSection ( ) {


<s> ConstantPoolMapTy :: iterator CP = ConstantPools . find ( Section ) ; if ( CP == ConstantPools . end ( ) ) return <NUM_LIT> ; return & CP -> second ;


<s> ConstantPool & AssemblerConstantPools :: getOrCreateConstantPool ( const MCSection * Section ) {


<s> } break ; case 'n' : case 'w' : Size = ( Suffix == 'n' ? <NUM_LIT> : <NUM_LIT> ) ; for ( unsigned II = <NUM_LIT> , IE = Size ; II != IE ; II = II + <NUM_LIT> ) { const unsigned I0 = LittleEndian ? II + <NUM_LIT> : II + <NUM_LIT> ; const unsigned I1 = LittleEndian ? II + <NUM_LIT> : II + <NUM_LIT> ; Buffer [ Size - II - <NUM_LIT> ] = uint8_t ( Inst >> I0 * CHAR_BIT ) ; Buffer [ Size - II - <NUM_LIT> ] = uint8_t ( Inst >> I1 * CHAR_BIT ) ; }


<s> Size = ( Suffix == 'n' ? <NUM_LIT> : <NUM_LIT> ) ; for ( unsigned II = <NUM_LIT> , IE = Size ; II != IE ; II = II + <NUM_LIT> ) { const unsigned I0 = LittleEndian ? II + <NUM_LIT> : II + <NUM_LIT> ; const unsigned I1 = LittleEndian ? II + <NUM_LIT> : II + <NUM_LIT> ; Buffer [ Size - II - <NUM_LIT> ] = uint8_t ( Inst >> I0 * CHAR_BIT ) ; Buffer [ Size - II - <NUM_LIT> ] = uint8_t ( Inst >> I1 * CHAR_BIT ) ; } break ; default :


<s> } else if ( STI . hasFeature ( <STR_LIT> ) ) emitFPU ( <STR_LIT> ) ; else emitFPU ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) ; if ( STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) ; } else { if ( STI . hasFeature ( <STR_LIT> ) ) emitFPU ( STI . hasFeature ( <STR_LIT> ) ? ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) : <STR_LIT> ) ; else if ( STI . hasFeature ( <STR_LIT> ) ) emitFPU ( STI . hasFeature ( <STR_LIT> ) ? ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) : <STR_LIT> ) ; else if ( STI . hasFeature ( <STR_LIT> ) ) emitFPU ( STI . hasFeature ( <STR_LIT> ) ? ( STI . hasFeature ( <STR_LIT> ) ? ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) : ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) ) : ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) ) ; else if ( STI . hasFeature ( <STR_LIT> ) ) emitFPU ( <STR_LIT> ) ; } if ( STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ; if ( STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ; if ( STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ; if ( STI . hasFeature ( <STR_LIT> ) && ! STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ; if ( STI . hasFeature ( <STR_LIT> ) && isV8M ( STI ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ; if ( STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ; else emitAttribute ( <STR_LIT> , <STR_LIT> ) ; if ( STI . hasFeature ( <STR_LIT> ) && STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ; else if ( STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ;


<s> if ( STI . getCPU ( ) == <STR_LIT> ) return <STR_LIT> ; if ( STI . hasFeature ( <STR_LIT> ) ) { if ( STI . hasFeature ( <STR_LIT> ) ) return <STR_LIT> ; return <STR_LIT> ; } else if ( STI . hasFeature ( <STR_LIT> ) ) return <STR_LIT> ; else if ( STI . hasFeature ( <STR_LIT> ) ) { if ( STI . hasFeature ( <STR_LIT> ) && STI . hasFeature ( <STR_LIT> ) ) return <STR_LIT> ; return <STR_LIT> ; } else if ( STI . hasFeature ( <STR_LIT> ) ) return <STR_LIT> ;


<s> ConstantPoolMapTy :: iterator CP = ConstantPools . find ( Section ) ; if ( CP == ConstantPools . end ( ) ) return nullptr ;


<s> const MCExpr * <ISA_LIT>TargetStreamer :: addConstantPoolEntry ( const MCExpr * Expr ) { return ConstantPools -> addEntry ( Streamer , Expr , <NUM_LIT> ) ;


<s> emitAttribute ( <STR_LIT> , <STR_LIT> ) ; } else if ( STI . hasFeature ( <STR_LIT> ) ) { emitAttribute ( <STR_LIT> , <STR_LIT> ) ; } else if ( STI . hasFeature ( <STR_LIT> ) ) { emitAttribute ( <STR_LIT> , <STR_LIT> ) ; } if ( STI . hasFeature ( <STR_LIT> ) ) { if ( STI . hasFeature ( <STR_LIT> ) ) { if ( STI . hasFeature ( <STR_LIT> ) ) emitFPU ( <STR_LIT> ) ; else emitFPU ( <STR_LIT> ) ; } else if ( STI . hasFeature ( <STR_LIT> ) ) emitFPU ( <STR_LIT> ) ; else emitFPU ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) ; if ( STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) ; } else { if ( STI . hasFeature ( <STR_LIT> ) ) emitFPU ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) ) ; else if ( STI . hasFeature ( <STR_LIT> ) ) emitFPU ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) ) ; else if ( STI . hasFeature ( <STR_LIT> ) ) emitFPU ( STI . hasFeature ( <STR_LIT> ) ? ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) : ( STI . hasFeature ( <STR_LIT> ) ? ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) : ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) ) ) ; else if ( STI . hasFeature ( <STR_LIT> ) ) emitFPU ( <STR_LIT> ) ; } if ( STI . hasFeature ( <STR_LIT> ) && ! STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ;

if ( STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ;


<s> } else if ( STI . hasFeature ( <STR_LIT> ) ) emitFPU ( <STR_LIT> ) ; else emitFPU ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) ; if ( STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) ; } else { if ( STI . hasFeature ( <STR_LIT> ) ) emitFPU ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) ) ; else if ( STI . hasFeature ( <STR_LIT> ) ) emitFPU ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) ) ; else if ( STI . hasFeature ( <STR_LIT> ) ) emitFPU ( STI . hasFeature ( <STR_LIT> ) ? ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) : ( STI . hasFeature ( <STR_LIT> ) ? ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) : ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) ) ) ; else if ( STI . hasFeature ( <STR_LIT> ) ) emitFPU ( <STR_LIT> ) ; } if ( STI . hasFeature ( <STR_LIT> ) && ! STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ; if ( STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ; if ( STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ; if ( STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ; else if ( STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ; if ( STI . hasFeature ( <STR_LIT> ) && ! STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ;

if ( STI . hasFeature ( <STR_LIT> ) && isV8M ( STI ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ;


<s> } else if ( STI . hasFeature ( <STR_LIT> ) ) emitFPU ( <STR_LIT> ) ; else emitFPU ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) ; if ( STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) ; } else { if ( STI . hasFeature ( <STR_LIT> ) ) emitFPU ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) ) ; else if ( STI . hasFeature ( <STR_LIT> ) ) emitFPU ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) ) ; else if ( STI . hasFeature ( <STR_LIT> ) ) emitFPU ( STI . hasFeature ( <STR_LIT> ) ? ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) : ( STI . hasFeature ( <STR_LIT> ) ? ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) : ( STI . hasFeature ( <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ) ) ) ; else if ( STI . hasFeature ( <STR_LIT> ) ) emitFPU ( <STR_LIT> ) ; } if ( STI . hasFeature ( <STR_LIT> ) && ! STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ; if ( STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ; if ( STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ; if ( STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ; else if ( STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ; if ( STI . hasFeature ( <STR_LIT> ) && ! STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ; if ( STI . hasFeature ( <STR_LIT> ) && isV8M ( STI ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ; if ( STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ; else emitAttribute ( <STR_LIT> , <STR_LIT> ) ; if ( STI . hasFeature ( <STR_LIT> ) && STI . hasFeature ( <STR_LIT> ) ) emitAttribute ( <STR_LIT> , <STR_LIT> ) ;


<s> const FeatureBitset & CallerBits = TM . getSubtargetImpl ( * Caller ) -> getFeatureBits ( ) ; const FeatureBitset & CalleeBits = TM . getSubtargetImpl ( * Callee ) -> getFeatureBits ( ) ;

bool MatchExact = ( CallerBits & ~ InlineFeatureWhitelist ) == ( CalleeBits & ~ InlineFeatureWhitelist ) ;


<s> const FeatureBitset & CallerBits = TM . getSubtargetImpl ( * Caller ) -> getFeatureBits ( ) ;

const FeatureBitset & CalleeBits = TM . getSubtargetImpl ( * Callee ) -> getFeatureBits ( ) ;


<s> explicit <ISA_LIT>TTIImpl ( const <ISA_LIT>BaseTargetMachine * TM , const Function & F ) : BaseT ( TM , F . getParent ( ) -> getDataLayout ( ) ) , ST ( TM -> getSubtargetImpl ( F ) ) , TLI ( ST -> getTargetLowering ( ) ) {


<s> bool enableInterleavedAccessVectorization ( ) { return true ;


<s> bool forceScalarizeMaskedGather ( VectorType * VTy , Align Alignment ) { return true ;


<s> bool forceScalarizeMaskedScatter ( VectorType * VTy , Align Alignment ) {

return forceScalarizeMaskedGather ( VTy , Alignment ) ;


<s> bool forceScalarizeMaskedScatter ( VectorType * VTy , Align Alignment ) { return forceScalarizeMaskedGather ( VTy , Alignment ) ;


<s> unsigned NumVectorInstToHideOverhead = <NUM_LIT> ; int MaxMergeDistance = <NUM_LIT> ; if ( Ty -> isVectorTy ( ) && SE && ! BaseT :: isConstantStridedAccessLessThan ( SE , Ptr , MaxMergeDistance + <NUM_LIT> ) ) return NumVectorInstToHideOverhead ; return <NUM_LIT> ;


<s> int <ISA_LIT>TTIImpl :: getAddressComputationCost ( Type * Ty , ScalarEvolution * SE , const SCEV * Ptr ) { unsigned NumVectorInstToHideOverhead = <NUM_LIT> ; int MaxMergeDistance = <NUM_LIT> ; if ( Ty -> isVectorTy ( ) && SE && ! BaseT :: isConstantStridedAccessLessThan ( SE , Ptr , MaxMergeDistance + <NUM_LIT> ) ) return NumVectorInstToHideOverhead ;


<s> int ISDOpcode = TLI -> InstructionOpcodeToISD ( Opcode ) ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Ty ) ; const unsigned FunctionCallDivCost = <NUM_LIT> ; const unsigned ReciprocalDivCost = <NUM_LIT> ; static const CostTblEntry CostTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , } ; if ( ST -> hasNEON ( ) ) if ( const auto * Entry = CostTableLookup ( CostTbl , ISDOpcode , LT . second ) ) return LT . first * Entry -> Cost ; int Cost = BaseT :: getArithmeticInstrCost ( Opcode , Ty , Op1Info , Op2Info , Opd1PropInfo , Opd2PropInfo ) ; if ( LT . second == <STR_LIT> && Op2Info == TargetTransformInfo :: OK_UniformConstantValue ) Cost += <NUM_LIT> ; return Cost ;


<s> std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Ty ) ; const unsigned FunctionCallDivCost = <NUM_LIT> ; const unsigned ReciprocalDivCost = <NUM_LIT> ; static const CostTblEntry CostTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , } ; if ( ST -> hasNEON ( ) ) if ( const auto * Entry = CostTableLookup ( CostTbl , ISDOpcode , LT . second ) ) return LT . first * Entry -> Cost ; int Cost = BaseT :: getArithmeticInstrCost ( Opcode , Ty , Op1Info , Op2Info , Opd1PropInfo , Opd2PropInfo ) ; if ( LT . second == <STR_LIT> && Op2Info == TargetTransformInfo :: OK_UniformConstantValue ) Cost += <NUM_LIT> ; return Cost ;


<s> EVT SelValTy = TLI -> getValueType ( DL , ValTy ) ; if ( SelCondTy . isSimple ( ) && SelValTy . isSimple ( ) ) { if ( const auto * Entry = ConvertCostTableLookup ( NEONVectorSelectTbl , ISD , SelCondTy . getSimpleVT ( ) , SelValTy . getSimpleVT ( ) ) ) return Entry -> Cost ; } std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ; return LT . first ; }

return BaseT :: getCmpSelInstrCost ( Opcode , ValTy , CondTy , I ) ;


<s> unsigned NumElts = VecTy -> getVectorNumElements ( ) ; auto * SubVecTy = VectorType :: get ( VecTy -> getScalarType ( ) , NumElts / Factor ) ; if ( NumElts % Factor == <NUM_LIT> && TLI -> isLegalInterleavedAccessType ( SubVecTy , DL ) ) return Factor * TLI -> getNumInterleavedAccesses ( SubVecTy , DL ) ; }

return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment , AddressSpace , UseMaskForCond , UseMaskForGaps ) ;


<s> if ( Opcode == Instruction :: And ) return std :: min ( getIntImmCost ( Imm , Ty ) , getIntImmCost ( ~ Imm , Ty ) ) ; if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty ) , getIntImmCost ( - Imm , Ty ) ) ;

if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == <NUM_LIT> ) {


<s> unsigned getMaxInterleaveFactor ( unsigned VF ) {

return ST -> getMaxInterleaveFactor ( ) ;


<s> std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Src ) ; if ( Src -> isVectorTy ( ) && Alignment != <NUM_LIT> && Src -> getVectorElementType ( ) -> isDoubleTy ( ) ) {


<s> int <ISA_LIT>TTIImpl :: getMemoryOpCost ( unsigned Opcode , Type * Src , unsigned Alignment , unsigned AddressSpace , const Instruction * I ) { std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Src ) ; if ( Src -> isVectorTy ( ) && Alignment != <NUM_LIT> && Src -> getVectorElementType ( ) -> isDoubleTy ( ) ) {


<s> if ( ST -> hasMVEIntegerOps ( ) ) return <NUM_LIT> ; return <NUM_LIT> ; } if ( ST -> isThumb1Only ( ) ) return <NUM_LIT> ; return <NUM_LIT> ;


<s> if ( ST -> hasNEON ( ) ) return TypeSize :: getFixed ( <NUM_LIT> ) ; if ( ST -> hasMVEIntegerOps ( ) ) return TypeSize :: getFixed ( <NUM_LIT> ) ;


<s> if ( ST -> hasNEON ( ) ) return TypeSize :: getFixed ( <NUM_LIT> ) ; if ( ST -> hasMVEIntegerOps ( ) ) return TypeSize :: getFixed ( <NUM_LIT> ) ; return TypeSize :: getFixed ( <NUM_LIT> ) ; case TargetTransformInfo :: RGK_ScalableVector :

return TypeSize :: getScalable ( <NUM_LIT> ) ;


<s> if ( const auto * Entry = CostTableLookup ( NEONDupTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost ; return BaseT :: getShuffleCost ( Kind , Tp , Index , SubTp ) ; } if ( Kind == TTI :: SK_Reverse ) { static const CostTblEntry NEONShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( NEONShuffleTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost ; return BaseT :: getShuffleCost ( Kind , Tp , Index , SubTp ) ; } if ( Kind == TTI :: SK_Select ) { static const CostTblEntry NEONSelShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ;


<s> int <ISA_LIT>TTIImpl :: getShuffleCost ( TTI :: ShuffleKind Kind , Type * Tp , int Index , Type * SubTp ) { if ( Kind == TTI :: SK_Broadcast ) { static const CostTblEntry NEONDupTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( NEONDupTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost ; return BaseT :: getShuffleCost ( Kind , Tp , Index , SubTp ) ; } if ( Kind == TTI :: SK_Reverse ) {

static const CostTblEntry NEONShuffleTbl [ ] = {


<s> const <ISA_LIT>Subtarget * getST ( ) const { return ST ;


<s> if ( ExitingBlocks . size ( ) > <NUM_LIT> ) return ; if ( ST -> hasBranchPredictor ( ) && L -> getNumBlocks ( ) > <NUM_LIT> ) return ; unsigned Cost = <NUM_LIT> ; for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { ImmutableCallSite CS ( & I ) ; if ( const Function * F = CS . getCalledFunction ( ) ) { if ( ! isLoweredToCall ( F ) ) continue ; } return ; } SmallVector < const Value * , <NUM_LIT> > Operands ( I . value_op_begin ( ) , I . value_op_end ( ) ) ; Cost += getUserCost ( & I , Operands ) ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ) ; UP . Partial = true ; UP . Runtime = true ; UP . UnrollRemainder = true ; UP . DefaultUnrollRuntimeCount = <NUM_LIT> ;


<s> L -> getExitingBlocks ( ExitingBlocks ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << L -> getNumBlocks ( ) << <STR_LIT> << <STR_LIT> << ExitingBlocks . size ( ) << <STR_LIT> ) ; if ( ExitingBlocks . size ( ) > <NUM_LIT> ) return ; if ( ST -> hasBranchPredictor ( ) && L -> getNumBlocks ( ) > <NUM_LIT> ) return ; unsigned Cost = <NUM_LIT> ; for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { ImmutableCallSite CS ( & I ) ; if ( const Function * F = CS . getCalledFunction ( ) ) { if ( ! isLoweredToCall ( F ) ) continue ; } return ; } SmallVector < const Value * , <NUM_LIT> > Operands ( I . value_op_begin ( ) , I . value_op_end ( ) ) ; Cost += getUserCost ( & I , Operands ) ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ) ; UP . Partial = true ; UP . Runtime = true ; UP . UnrollRemainder = true ; UP . DefaultUnrollRuntimeCount = <NUM_LIT> ; UP . UnrollAndJam = true ; UP . UnrollAndJamInnerLoopThreshold = <NUM_LIT> ; if ( Cost < <NUM_LIT> ) UP . Force = true ;


<s> if ( ST -> hasSlowLoadDSubregister ( ) && Opcode == Instruction :: InsertElement && ValTy -> isVectorTy ( ) && ValTy -> getScalarSizeInBits ( ) <= <NUM_LIT> ) return <NUM_LIT> ; if ( ( Opcode == Instruction :: InsertElement || Opcode == Instruction :: ExtractElement ) ) { if ( ValTy -> getVectorElementType ( ) -> isIntegerTy ( ) ) return <NUM_LIT> ;

if ( ValTy -> isVectorTy ( ) && ValTy -> getScalarSizeInBits ( ) <= <NUM_LIT> ) return std :: max ( BaseT :: getVectorInstrCost ( Opcode , ValTy , Index ) , <NUM_LIT> ) ;


<s> if ( ST -> hasSlowLoadDSubregister ( ) && Opcode == Instruction :: InsertElement && ValTy -> isVectorTy ( ) && ValTy -> getScalarSizeInBits ( ) <= <NUM_LIT> ) return <NUM_LIT> ; if ( ( Opcode == Instruction :: InsertElement || Opcode == Instruction :: ExtractElement ) ) { if ( ValTy -> getVectorElementType ( ) -> isIntegerTy ( ) ) return <NUM_LIT> ; if ( ValTy -> isVectorTy ( ) && ValTy -> getScalarSizeInBits ( ) <= <NUM_LIT> ) return std :: max ( BaseT :: getVectorInstrCost ( Opcode , ValTy , Index ) , <NUM_LIT> ) ; }

return BaseT :: getVectorInstrCost ( Opcode , ValTy , Index ) ;


<s> unsigned EltSz = VT . getScalarSizeInBits ( ) ; if ( EltSz != <NUM_LIT> && EltSz != <NUM_LIT> && EltSz != <NUM_LIT> ) return false ; unsigned BlockElts = M [ <NUM_LIT> ] + <NUM_LIT> ; if ( M [ <NUM_LIT> ] < <NUM_LIT> ) BlockElts = BlockSize / EltSz ;

if ( BlockSize <= EltSz || BlockSize != BlockElts * EltSz ) return false ;


<s> bool shouldBuildLookupTablesForConstant ( Constant * C ) const {

if ( ST -> isROPI ( ) || ST -> isRWPI ( ) ) return ! C -> needsDynamicRelocation ( ) ;


<s> if ( ST -> isROPI ( ) || ST -> isRWPI ( ) ) return ! C -> needsDynamicRelocation ( ) ; return true ;


<s> bool shouldExpandReduction ( const IntrinsicInst * II ) const { return false ;


<s> if ( Ty -> isVectorTy ( ) && IsComplex ) return NumVectorInstToHideOverhead ; return <NUM_LIT> ;


<s> int <ISA_LIT>TTIImpl :: getAddressComputationCost ( Type * Ty , bool IsComplex ) { unsigned NumVectorInstToHideOverhead = <NUM_LIT> ;

if ( Ty -> isVectorTy ( ) && IsComplex ) return NumVectorInstToHideOverhead ;


<s> static const TypeConversionCostTblEntry NEONVectorSelectTbl [ ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; EVT SelCondTy = TLI -> getValueType ( DL , CondTy ) ; EVT SelValTy = TLI -> getValueType ( DL , ValTy ) ; if ( SelCondTy . isSimple ( ) && SelValTy . isSimple ( ) ) {

if ( const auto * Entry = ConvertCostTableLookup ( NEONVectorSelectTbl , ISD , SelCondTy . getSimpleVT ( ) , SelValTy . getSimpleVT ( ) ) ) return Entry -> Cost ;


<s> return ST -> isFPOnlySP ( ) ? TargetTransformInfo :: TCC_Expensive : TargetTransformInfo :: TCC_Basic ; } }

return TargetTransformInfo :: TCC_Expensive ;


<s> unsigned NumElts = VecTy -> getVectorNumElements ( ) ; Type * SubVecTy = VectorType :: get ( VecTy -> getScalarType ( ) , NumElts / Factor ) ; unsigned SubVecSize = DL . getTypeSizeInBits ( SubVecTy ) ; if ( NumElts % Factor == <NUM_LIT> && ( SubVecSize == <NUM_LIT> || SubVecSize == <NUM_LIT> ) ) return Factor ; }

return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment , AddressSpace ) ;


<s> if ( Bits == <NUM_LIT> || Bits > <NUM_LIT> ) return <NUM_LIT> ; int32_t SImmVal = Imm . getSExtValue ( ) ; uint32_t ZImmVal = Imm . getZExtValue ( ) ; if ( ! ST -> isThumb ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ; } if ( ST -> isThumb2 ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ;


<s> std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Src ) ; if ( Src -> isVectorTy ( ) && Alignment != <NUM_LIT> && Src -> getVectorElementType ( ) -> isDoubleTy ( ) ) {


<s> return BaseT :: getShuffleCost ( Kind , Tp , Index , SubTp ) ; } if ( Kind == TTI :: SK_Alternate ) { static const CostTblEntry NEONAltShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( NEONAltShuffleTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost ;


<s> if ( ( Opcode == Instruction :: InsertElement || Opcode == Instruction :: ExtractElement ) ) { if ( ValTy -> getVectorElementType ( ) -> isIntegerTy ( ) ) return <NUM_LIT> ;


<s> if ( ST -> isSwift ( ) && Opcode == Instruction :: InsertElement && ValTy -> isVectorTy ( ) && ValTy -> getScalarSizeInBits ( ) <= <NUM_LIT> ) return <NUM_LIT> ; if ( ( Opcode == Instruction :: InsertElement || Opcode == Instruction :: ExtractElement ) ) {

if ( ValTy -> getVectorElementType ( ) -> isIntegerTy ( ) ) return <NUM_LIT> ;


<s> <ISA_LIT>TTIImpl ( <ISA_LIT>TTIImpl && Arg ) : BaseT ( std :: move ( static_cast < BaseT & > ( Arg ) ) ) , ST ( std :: move ( Arg . ST ) ) , TLI ( std :: move ( Arg . TLI ) ) {


<s> if ( const auto * Entry = ConvertCostTableLookup ( NEONVectorSelectTbl , ISD , SelCondTy . getSimpleVT ( ) , SelValTy . getSimpleVT ( ) ) ) return Entry -> Cost ; } std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ; return LT . first ; } int BaseCost = ST -> hasMVEIntegerOps ( ) && ValTy -> isVectorTy ( ) ? ST -> getMVEVectorCostFactor ( ) : <NUM_LIT> ;

return BaseCost * BaseT :: getCmpSelInstrCost ( Opcode , ValTy , CondTy , I ) ;


<s> } } ; EVT SelCondTy = TLI -> getValueType ( DL , CondTy ) ; EVT SelValTy = TLI -> getValueType ( DL , ValTy ) ; if ( SelCondTy . isSimple ( ) && SelValTy . isSimple ( ) ) { if ( const auto * Entry = ConvertCostTableLookup ( NEONVectorSelectTbl , ISD , SelCondTy . getSimpleVT ( ) , SelValTy . getSimpleVT ( ) ) ) return Entry -> Cost ; } std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ;


<s> if ( EltSize * NumElems != <NUM_LIT> || NumElems < <NUM_LIT> ) return ScalarCost ; if ( Alignment < EltSize / <NUM_LIT> ) return ScalarCost ; if ( EltSize == <NUM_LIT> ) return VectorCost ; if ( EltSize != <NUM_LIT> && EltSize != <NUM_LIT> ) return ScalarCost ; if ( auto BC = dyn_cast < BitCastInst > ( Ptr ) ) Ptr = BC -> getOperand ( <NUM_LIT> ) ; if ( auto * GEP = dyn_cast < GetElementPtrInst > ( Ptr ) ) { if ( GEP -> getNumOperands ( ) != <NUM_LIT> ) return ScalarCost ; unsigned Scale = DL . getTypeAllocSize ( GEP -> getResultElementType ( ) ) ;

if ( Scale != <NUM_LIT> && Scale * <NUM_LIT> != EltSize ) return ScalarCost ;


<s> int BaseCost = ST -> hasMVEIntegerOps ( ) ? ST -> getMVEVectorCostFactor ( ) : <NUM_LIT> ; if ( NumElts % Factor == <NUM_LIT> && TLI -> isLegalInterleavedAccessType ( Factor , SubVecTy , DL ) ) return Factor * BaseCost * TLI -> getNumInterleavedAccesses ( SubVecTy , DL ) ; if ( ST -> hasMVEIntegerOps ( ) && Factor == <NUM_LIT> && NumElts / Factor > <NUM_LIT> && VecTy -> isIntOrIntVectorTy ( ) && DL . getTypeSizeInBits ( SubVecTy ) <= <NUM_LIT> ) return <NUM_LIT> * BaseCost ;


<s> int <ISA_LIT>TTIImpl :: getIntImmCostInst ( unsigned Opcode , unsigned Idx , const APInt & Imm , Type * Ty ) { if ( ( Opcode == Instruction :: SDiv || Opcode == Instruction :: UDiv || Opcode == Instruction :: SRem || Opcode == Instruction :: URem ) && Idx == <NUM_LIT> ) return <NUM_LIT> ; if ( Opcode == Instruction :: And ) {

if ( Imm == <NUM_LIT> || Imm == <NUM_LIT> ) return <NUM_LIT> ;


<s> return std :: min ( getIntImmCost ( Imm , Ty ) , getIntImmCost ( ~ Imm , Ty ) ) ; } if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty ) , getIntImmCost ( - Imm , Ty ) ) ;

if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == <NUM_LIT> ) {


<s> if ( ST -> isCortexA15 ( ) || ST -> isSwift ( ) ) return <NUM_LIT> ;


<s> int <ISA_LIT>TTIImpl :: getMemoryOpCost ( unsigned Opcode , Type * Src , MaybeAlign Alignment , unsigned AddressSpace , const Instruction * I ) {

std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Src ) ;


<s> if ( ST -> hasNEON ( ) && Src -> isVectorTy ( ) && ( Alignment && * Alignment != Align ( <NUM_LIT> ) ) && Src -> getVectorElementType ( ) -> isDoubleTy ( ) ) { return LT . first * <NUM_LIT> ; } int BaseCost = ST -> hasMVEIntegerOps ( ) && Src -> isVectorTy ( ) ? ST -> getMVEVectorCostFactor ( ) : <NUM_LIT> ; return BaseCost * LT . first ;


<s> if ( Vector ) { if ( ST -> hasNEON ( ) ) return <NUM_LIT> ; return <NUM_LIT> ;


<s> { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( NEONSelShuffleTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost ; } } if ( ST -> hasMVEIntegerOps ( ) ) { if ( Kind == TTI :: SK_Broadcast ) { static const CostTblEntry MVEDupTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( MVEDupTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost * ST -> getMVEVectorCostFactor ( ) ; } } int BaseCost = ST -> hasMVEIntegerOps ( ) && Tp -> isVectorTy ( ) ? ST -> getMVEVectorCostFactor ( ) : <NUM_LIT> ; return BaseCost * BaseT :: getShuffleCost ( Kind , Tp , Index , SubTp ) ;


<s> UP . OptSizeThreshold = <NUM_LIT> ; UP . PartialOptSizeThreshold = <NUM_LIT> ; if ( L -> getHeader ( ) -> getParent ( ) -> hasOptSize ( ) ) return ; if ( ! ST -> isThumb2 ( ) ) return ; SmallVector < BasicBlock * , <NUM_LIT> > ExitingBlocks ; L -> getExitingBlocks ( ExitingBlocks ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << L -> getNumBlocks ( ) << <STR_LIT> << <STR_LIT> << ExitingBlocks . size ( ) << <STR_LIT> ) ; if ( ExitingBlocks . size ( ) > <NUM_LIT> ) return ; if ( ST -> hasBranchPredictor ( ) && L -> getNumBlocks ( ) > <NUM_LIT> ) return ; unsigned Cost = <NUM_LIT> ; for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( I . getType ( ) -> isVectorTy ( ) ) return ; if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { ImmutableCallSite CS ( & I ) ; if ( const Function * F = CS . getCalledFunction ( ) ) { if ( ! isLoweredToCall ( F ) ) continue ; } return ; }

SmallVector < const Value * , <NUM_LIT> > Operands ( I . value_op_begin ( ) , I . value_op_end ( ) ) ;


<s> if ( ST -> hasBranchPredictor ( ) && L -> getNumBlocks ( ) > <NUM_LIT> ) return ; unsigned Cost = <NUM_LIT> ; for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( I . getType ( ) -> isVectorTy ( ) ) return ; if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { ImmutableCallSite CS ( & I ) ; if ( const Function * F = CS . getCalledFunction ( ) ) { if ( ! isLoweredToCall ( F ) ) continue ; } return ; } SmallVector < const Value * , <NUM_LIT> > Operands ( I . value_op_begin ( ) , I . value_op_end ( ) ) ; Cost += getUserCost ( & I , Operands ) ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ) ; UP . Partial = true ; UP . Runtime = true ; UP . UpperBound = true ; UP . UnrollRemainder = true ; UP . DefaultUnrollRuntimeCount = <NUM_LIT> ; UP . UnrollAndJam = true ; UP . UnrollAndJamInnerLoopThreshold = <NUM_LIT> ;

if ( Cost < <NUM_LIT> ) UP . Force = true ;


<s> if ( ValTy -> isVectorTy ( ) && ValTy -> getScalarSizeInBits ( ) <= <NUM_LIT> ) return std :: max ( BaseT :: getVectorInstrCost ( Opcode , ValTy , Index ) , <NUM_LIT> ) ; } if ( ST -> hasMVEIntegerOps ( ) && ( Opcode == Instruction :: InsertElement || Opcode == Instruction :: ExtractElement ) ) {

return std :: max ( BaseT :: getVectorInstrCost ( Opcode , ValTy , Index ) , ST -> getMVEVectorCostFactor ( ) ) * ValTy -> getVectorNumElements ( ) / <NUM_LIT> ;


<s> bool <ISA_LIT>TTIImpl :: isLegalMaskedGather ( Type * Ty , MaybeAlign Alignment ) { if ( ! EnableMaskedGatherScatters || ! ST -> hasMVEIntegerOps ( ) ) return false ; if ( isa < VectorType > ( Ty ) ) return false ; unsigned EltWidth = Ty -> getScalarSizeInBits ( ) ; return ( ( EltWidth == <NUM_LIT> && ( ! Alignment || Alignment >= <NUM_LIT> ) ) || ( EltWidth == <NUM_LIT> && ( ! Alignment || Alignment >= <NUM_LIT> ) ) || EltWidth == <NUM_LIT> ) ;


<s> unsigned EltWidth = DataTy -> getScalarSizeInBits ( ) ;

return ( EltWidth == <NUM_LIT> && ( ! Alignment || Alignment >= <NUM_LIT> ) ) || ( EltWidth == <NUM_LIT> && ( ! Alignment || Alignment >= <NUM_LIT> ) ) || ( EltWidth == <NUM_LIT> ) ;


<s> } unsigned EltWidth = DataTy -> getScalarSizeInBits ( ) ;


<s> int <ISA_LIT>TTIImpl :: getIntImmCost ( unsigned Opcode , unsigned Idx , const APInt & Imm , Type * Ty ) { if ( ( Opcode == Instruction :: SDiv || Opcode == Instruction :: UDiv || Opcode == Instruction :: SRem || Opcode == Instruction :: URem ) && Idx == <NUM_LIT> ) return <NUM_LIT> ; if ( Opcode == Instruction :: And ) return std :: min ( getIntImmCost ( Imm , Ty ) , getIntImmCost ( ~ Imm , Ty ) ) ; if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty ) , getIntImmCost ( - Imm , Ty ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == <NUM_LIT> ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; if ( ST -> isThumb ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; } return getIntImmCost ( Imm , Ty ) ;


<s> int <ISA_LIT>TTIImpl :: getArithmeticInstrCost ( unsigned Opcode , Type * Ty , TTI :: OperandValueKind Op1Info , TTI :: OperandValueKind Op2Info , TTI :: OperandValueProperties Opd1PropInfo , TTI :: OperandValueProperties Opd2PropInfo ) { int ISDOpcode = TLI -> InstructionOpcodeToISD ( Opcode ) ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Ty ) ; const unsigned FunctionCallDivCost = <NUM_LIT> ; const unsigned ReciprocalDivCost = <NUM_LIT> ; static const CostTblEntry < <STR_LIT> > CostTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , } ; if ( ST -> hasNEON ( ) ) if ( const auto * Entry = CostTableLookup ( CostTbl , ISDOpcode , LT . second ) ) return LT . first * Entry -> Cost ; int Cost = BaseT :: getArithmeticInstrCost ( Opcode , Ty , Op1Info , Op2Info , Opd1PropInfo , Opd2PropInfo ) ; if ( LT . second == <STR_LIT> && Op2Info == TargetTransformInfo :: OK_UniformConstantValue ) Cost += <NUM_LIT> ; return Cost ;


<s> int <ISA_LIT>TTIImpl :: getArithmeticInstrCost ( unsigned Opcode , Type * Ty , TTI :: OperandValueKind Op1Info , TTI :: OperandValueKind Op2Info , TTI :: OperandValueProperties Opd1PropInfo , TTI :: OperandValueProperties Opd2PropInfo ) { int ISDOpcode = TLI -> InstructionOpcodeToISD ( Opcode ) ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Ty ) ; const unsigned FunctionCallDivCost = <NUM_LIT> ; const unsigned ReciprocalDivCost = <NUM_LIT> ; static const CostTblEntry < <STR_LIT> > CostTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , ReciprocalDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> * FunctionCallDivCost } , } ; if ( ST -> hasNEON ( ) ) if ( const auto * Entry = CostTableLookup ( CostTbl , ISDOpcode , LT . second ) ) return LT . first * Entry -> Cost ; int Cost = BaseT :: getArithmeticInstrCost ( Opcode , Ty , Op1Info , Op2Info , Opd1PropInfo , Opd2PropInfo ) ;

if ( LT . second == <STR_LIT> && Op2Info == TargetTransformInfo :: OK_UniformConstantValue ) Cost += <NUM_LIT> ;


<s> int ISD = TLI -> InstructionOpcodeToISD ( Opcode ) ; if ( ST -> hasNEON ( ) && ValTy -> isVectorTy ( ) && ISD == <STR_LIT> ) { static const TypeConversionCostTblEntry < <STR_LIT> > NEONVectorSelectTbl [ ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> * <NUM_LIT> + <NUM_LIT> * <NUM_LIT> + <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; EVT SelCondTy = TLI -> getValueType ( DL , CondTy ) ; EVT SelValTy = TLI -> getValueType ( DL , ValTy ) ; if ( SelCondTy . isSimple ( ) && SelValTy . isSimple ( ) ) { if ( const auto * Entry = ConvertCostTableLookup ( NEONVectorSelectTbl , ISD , SelCondTy . getSimpleVT ( ) , SelValTy . getSimpleVT ( ) ) ) return Entry -> Cost ; } std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ; return LT . first ; } return BaseT :: getCmpSelInstrCost ( Opcode , ValTy , CondTy ) ;


<s> unsigned SubVecSize = DL . getTypeAllocSizeInBits ( SubVecTy ) ; if ( NumElts % Factor == <NUM_LIT> && ( SubVecSize == <NUM_LIT> || SubVecSize == <NUM_LIT> ) ) return Factor ; }


<s> return BaseT :: getShuffleCost ( Kind , Tp , Index , SubTp ) ; } if ( Kind == TTI :: SK_Alternate ) { static const CostTblEntry < <STR_LIT> > NEONAltShuffleTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < int , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ;

if ( const auto * Entry = CostTableLookup ( NEONAltShuffleTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost ;


<s> } ; if ( SrcTy . isFloatingPoint ( ) && ST -> hasNEON ( ) ) { int Idx = ConvertCostTableLookup < MVT > ( NEONFloatConversionTbl , array_lengthof ( NEONFloatConversionTbl ) , ISD , DstTy . getSimpleVT ( ) , SrcTy . getSimpleVT ( ) ) ; if ( Idx != - <NUM_LIT> ) return NEONFloatConversionTbl [ Idx ] . Cost ; } static const TypeConversionCostTblEntry < MVT > NEONIntegerConversionTbl [ ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; if ( SrcTy . isInteger ( ) && ST -> hasNEON ( ) ) { int Idx = ConvertCostTableLookup < MVT > ( NEONIntegerConversionTbl , array_lengthof ( NEONIntegerConversionTbl ) , ISD , DstTy . getSimpleVT ( ) , SrcTy . getSimpleVT ( ) ) ; if ( Idx != - <NUM_LIT> ) return NEONIntegerConversionTbl [ Idx ] . Cost ; } static const TypeConversionCostTblEntry < MVT > <ISA_LIT>IntegerConversionTbl [ ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; if ( SrcTy . isInteger ( ) ) { int Idx = ConvertCostTableLookup < MVT > ( <ISA_LIT>IntegerConversionTbl , array_lengthof ( <ISA_LIT>IntegerConversionTbl ) , ISD , DstTy . getSimpleVT ( ) , SrcTy . getSimpleVT ( ) ) ; if ( Idx != - <NUM_LIT> ) return <ISA_LIT>IntegerConversionTbl [ Idx ] . Cost ; } return TargetTransformInfo :: getCastInstrCost ( Opcode , Dst , Src ) ;


<s> unsigned <STR_LIT> ( unsigned Opcode , Type * ValTy , Type * CondTy ) const {

int ISD = TLI -> InstructionOpcodeToISD ( Opcode ) ;


<s> unsigned <STR_LIT> ( unsigned Opcode , Type * ValTy , Type * CondTy ) const { int ISD = TLI -> InstructionOpcodeToISD ( Opcode ) ;

if ( ST -> hasNEON ( ) && ValTy -> isVectorTy ( ) && ISD == <STR_LIT> ) {


<s> bool isLegalMaskedScatter ( Type * Ty , MaybeAlign Alignment ) {


<s> bool isLegalMaskedScatter ( Type * Ty , MaybeAlign Alignment ) { return false ;


<s> if ( L -> getHeader ( ) -> getParent ( ) -> hasOptSize ( ) ) return false ; return ST -> isMClass ( ) && ST -> isThumb2 ( ) && L -> getNumBlocks ( ) == <NUM_LIT> ;


<s> if ( isa < FPExtInst > ( & I ) || isa < FPTruncInst > ( & I ) ) return false ; if ( isa < SExtInst > ( & I ) || isa < ZExtInst > ( & I ) ) if ( ! I . getOperand ( <NUM_LIT> ) -> hasOneUse ( ) || ! isa < LoadInst > ( I . getOperand ( <NUM_LIT> ) ) ) return false ; if ( isa < TruncInst > ( & I ) ) if ( ! I . hasOneUse ( ) || ! isa < StoreInst > ( * I . user_begin ( ) ) ) return false ;


<s> if ( isa < ICmpInst > ( & I ) && ++ ICmpCount > <NUM_LIT> ) return false ; if ( auto * II = dyn_cast < IntrinsicInst > ( & I ) ) if ( ( II -> getIntrinsicID ( ) == <STR_LIT> || II -> getIntrinsicID ( ) == <STR_LIT> || II -> getIntrinsicID ( ) == <STR_LIT> || II -> getIntrinsicID ( ) == <STR_LIT> ) && ++ ICmpCount > <NUM_LIT> ) return false ;

if ( isa < FCmpInst > ( & I ) ) return false ;


<s> if ( isa < PHINode > ( & I ) ) continue ; if ( ! canTailPredicateInstruction ( I , ICmpCount ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; I . dump ( ) ) ; return false ; } Type * T = I . getType ( ) ; if ( T -> isPointerTy ( ) ) T = T -> getPointerElementType ( ) ; if ( T -> getScalarSizeInBits ( ) > <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; T -> dump ( ) ) ; return false ; } if ( isa < StoreInst > ( I ) || isa < LoadInst > ( I ) ) { Value * Ptr = getLoadStorePointerOperand ( & I ) ; Type * AccessTy = getLoadStoreType ( & I ) ; int64_t NextStride = getPtrStride ( PSE , AccessTy , Ptr , L ) ; if ( NextStride == <NUM_LIT> ) { continue ; } else if ( NextStride == - <NUM_LIT> || ( NextStride == <NUM_LIT> && MVEMaxSupportedInterleaveFactor >= <NUM_LIT> ) || ( NextStride == <NUM_LIT> && MVEMaxSupportedInterleaveFactor >= <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } else if ( EnableMaskedGatherScatters ) {

const SCEV * PtrScev = PSE . getSE ( ) -> getSCEV ( Ptr ) ;


<s> bool <ISA_LIT>TTIImpl :: emitGetActiveLaneMask ( ) const { if ( ! ST -> hasMVEIntegerOps ( ) || ! EnableTailPredication ) return false ; return true ;


<s> unsigned NumVectorInstToHideOverhead = <NUM_LIT> ; int MaxMergeDistance = <NUM_LIT> ; if ( ST -> hasNEON ( ) ) { if ( Ty -> isVectorTy ( ) && SE && ! BaseT :: isConstantStridedAccessLessThan ( SE , Ptr , MaxMergeDistance + <NUM_LIT> ) ) return NumVectorInstToHideOverhead ;


<s> static const CostTblEntry CostTblAdd [ ] { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , } ; if ( const auto * Entry = CostTableLookup ( CostTblAdd , ISD , LT . second ) ) return Entry -> Cost * ST -> getMVEVectorCostFactor ( CostKind ) * LT . first ; return BaseT :: getArithmeticReductionCost ( Opcode , ValTy , FMF , CostKind ) ;


<s> std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ; static const CostTblEntry CostTblAdd [ ] { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , } ;

if ( const auto * Entry = CostTableLookup ( CostTblAdd , ISD , LT . second ) ) return Entry -> Cost * ST -> getMVEVectorCostFactor ( CostKind ) * LT . first ;


<s> InstructionCost <ISA_LIT>TTIImpl :: getCFInstrCost ( unsigned Opcode , TTI :: TargetCostKind CostKind , const Instruction * I ) { if ( CostKind == TTI :: TCK_RecipThroughput && ( ST -> hasNEON ( ) || ST -> hasMVEIntegerOps ( ) ) ) { return <NUM_LIT> ;


<s> if ( CostKind == TTI :: TCK_RecipThroughput && ( ST -> hasNEON ( ) || ST -> hasMVEIntegerOps ( ) ) ) { return <NUM_LIT> ; }

return BaseT :: getCFInstrCost ( Opcode , CostKind , I ) ;


<s> if ( ST -> hasMVEIntegerOps ( ) && ValTy -> isVectorTy ( ) && ( Opcode == Instruction :: ICmp || Opcode == Instruction :: FCmp ) && cast < FixedVectorType > ( ValTy ) -> getNumElements ( ) > <NUM_LIT> ) { FixedVectorType * VecValTy = cast < FixedVectorType > ( ValTy ) ; FixedVectorType * VecCondTy = dyn_cast_or_null < FixedVectorType > ( CondTy ) ; if ( ! VecCondTy ) VecCondTy = cast < FixedVectorType > ( CmpInst :: makeCmpResultType ( VecValTy ) ) ; if ( Opcode == Instruction :: FCmp && ! ST -> hasMVEFloatOps ( ) ) { return BaseT :: getScalarizationOverhead ( VecValTy , false , true ) + BaseT :: getScalarizationOverhead ( VecCondTy , true , false ) + VecValTy -> getNumElements ( ) * getCmpSelInstrCost ( Opcode , ValTy -> getScalarType ( ) , VecCondTy -> getScalarType ( ) , VecPred , CostKind , I ) ; } std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , ValTy ) ; int BaseCost = ST -> getMVEVectorCostFactor ( CostKind ) ; if ( LT . second . getVectorNumElements ( ) > <NUM_LIT> ) { if ( LT . first > <NUM_LIT> ) return LT . first * BaseCost + BaseT :: getScalarizationOverhead ( VecCondTy , true , false ) ; return BaseCost ; } } int BaseCost = <NUM_LIT> ;

if ( ST -> hasMVEIntegerOps ( ) && ValTy -> isVectorTy ( ) ) BaseCost = ST -> getMVEVectorCostFactor ( CostKind ) ;


<s> InstructionCost <ISA_LIT>TTIImpl :: getExtendedAddReductionCost ( bool IsMLA , bool IsUnsigned , Type * ResTy , VectorType * ValTy , TTI :: TargetCostKind CostKind ) { EVT ValVT = TLI -> getValueType ( DL , ValTy ) ; EVT ResVT = TLI -> getValueType ( DL , ResTy ) ;

if ( ST -> hasMVEIntegerOps ( ) && ValVT . isSimple ( ) && ResVT . isSimple ( ) ) {


<s> if ( ValVT . getSizeInBits ( ) <= <NUM_LIT> && ( ( LT . second == <STR_LIT> && RevVTSize <= <NUM_LIT> ) || ( LT . second == <STR_LIT> && RevVTSize <= ( IsMLA ? <NUM_LIT> : <NUM_LIT> ) ) || ( LT . second == <STR_LIT> && RevVTSize <= <NUM_LIT> ) ) ) return ST -> getMVEVectorCostFactor ( CostKind ) * LT . first ;


<s> unsigned NumElems = VTy -> getNumElements ( ) ; unsigned EltSize = VTy -> getScalarSizeInBits ( ) ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , DataTy ) ; InstructionCost VectorCost = NumElems * LT . first * ST -> getMVEVectorCostFactor ( CostKind ) ; InstructionCost ScalarCost = NumElems * LT . first + BaseT :: getScalarizationOverhead ( VTy , true , false ) + BaseT :: getScalarizationOverhead ( VTy , false , true ) ; if ( EltSize < <NUM_LIT> || Alignment < EltSize / <NUM_LIT> ) return ScalarCost ; unsigned ExtSize = EltSize ; if ( I != nullptr ) { if ( ( I -> getOpcode ( ) == Instruction :: Load || match ( I , m_Intrinsic < <STR_LIT> > ( ) ) ) && I -> hasOneUse ( ) ) { const User * Us = * I -> users ( ) . begin ( ) ; if ( isa < ZExtInst > ( Us ) || isa < SExtInst > ( Us ) ) { unsigned TypeSize = cast < Instruction > ( Us ) -> getType ( ) -> getScalarSizeInBits ( ) ; if ( ( ( TypeSize == <NUM_LIT> && ( EltSize == <NUM_LIT> || EltSize == <NUM_LIT> ) ) || ( TypeSize == <NUM_LIT> && EltSize == <NUM_LIT> ) ) && TypeSize * NumElems == <NUM_LIT> ) { ExtSize = TypeSize ; } } } TruncInst * T ; if ( ( I -> getOpcode ( ) == Instruction :: Store || match ( I , m_Intrinsic < <STR_LIT> > ( ) ) ) && ( T = dyn_cast < TruncInst > ( I -> getOperand ( <NUM_LIT> ) ) ) ) { unsigned TypeSize = T -> getOperand ( <NUM_LIT> ) -> getType ( ) -> getScalarSizeInBits ( ) ; if ( ( ( EltSize == <NUM_LIT> && TypeSize == <NUM_LIT> ) || ( EltSize == <NUM_LIT> && ( TypeSize == <NUM_LIT> || TypeSize == <NUM_LIT> ) ) ) && TypeSize * NumElems == <NUM_LIT> ) ExtSize = TypeSize ; } } if ( ExtSize * NumElems != <NUM_LIT> || NumElems < <NUM_LIT> ) return ScalarCost ;

if ( ExtSize == <NUM_LIT> ) return VectorCost ;


<s> if ( NumElts % Factor == <NUM_LIT> && TLI -> isLegalInterleavedAccessType ( Factor , SubVecTy , Alignment , DL ) ) return Factor * BaseCost * TLI -> getNumInterleavedAccesses ( SubVecTy , DL ) ; if ( ST -> hasMVEIntegerOps ( ) && Factor == <NUM_LIT> && NumElts / Factor > <NUM_LIT> && VecTy -> isIntOrIntVectorTy ( ) && DL . getTypeSizeInBits ( SubVecTy ) . getFixedSize ( ) <= <NUM_LIT> ) return <NUM_LIT> * BaseCost ; }

return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment , AddressSpace , CostKind , UseMaskForCond , UseMaskForGaps ) ;


<s> bool EltIs64Bits = DL . getTypeSizeInBits ( VecTy -> getScalarType ( ) ) == <NUM_LIT> ; if ( Factor <= TLI -> getMaxSupportedInterleaveFactor ( ) && ! EltIs64Bits && ! UseMaskForCond && ! UseMaskForGaps ) { unsigned NumElts = cast < FixedVectorType > ( VecTy ) -> getNumElements ( ) ; auto * SubVecTy = FixedVectorType :: get ( VecTy -> getScalarType ( ) , NumElts / Factor ) ; int BaseCost = ST -> hasMVEIntegerOps ( ) ? ST -> getMVEVectorCostFactor ( CostKind ) : <NUM_LIT> ; if ( NumElts % Factor == <NUM_LIT> && TLI -> isLegalInterleavedAccessType ( Factor , SubVecTy , Alignment , DL ) ) return Factor * BaseCost * TLI -> getNumInterleavedAccesses ( SubVecTy , DL ) ; if ( ST -> hasMVEIntegerOps ( ) && Factor == <NUM_LIT> && NumElts / Factor > <NUM_LIT> && VecTy -> isIntOrIntVectorTy ( ) && DL . getTypeSizeInBits ( SubVecTy ) . getFixedSize ( ) <= <NUM_LIT> ) return <NUM_LIT> * BaseCost ; } return BaseT :: getInterleavedMemoryOpCost ( Opcode , VecTy , Factor , Indices , Alignment , AddressSpace , CostKind , UseMaskForCond , UseMaskForGaps ) ;


<s> InstructionCost <ISA_LIT>TTIImpl :: getIntImmCodeSizeCost ( unsigned Opcode , unsigned Idx , const APInt & Imm , Type * Ty ) { if ( Imm . isNonNegative ( ) && Imm . getLimitedValue ( ) < <NUM_LIT> ) return <NUM_LIT> ; return <NUM_LIT> ;


<s> unsigned Bits = Ty -> getPrimitiveSizeInBits ( ) ; if ( Bits == <NUM_LIT> || Imm . getActiveBits ( ) >= <NUM_LIT> ) return <NUM_LIT> ; int64_t SImmVal = Imm . getSExtValue ( ) ; uint64_t ZImmVal = Imm . getZExtValue ( ) ; if ( ! ST -> isThumb ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ; } if ( ST -> isThumb2 ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ; } if ( Bits == <NUM_LIT> || ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) ) return <NUM_LIT> ; if ( ( ~ SImmVal < <NUM_LIT> ) || <STR_LIT> ( ZImmVal ) ) return <NUM_LIT> ;


<s> int64_t SImmVal = Imm . getSExtValue ( ) ; uint64_t ZImmVal = Imm . getZExtValue ( ) ; if ( ! ST -> isThumb ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ; return ST -> hasV6T2Ops ( ) ? <NUM_LIT> : <NUM_LIT> ; } if ( ST -> isThumb2 ( ) ) { if ( ( SImmVal >= <NUM_LIT> && SImmVal < <NUM_LIT> ) || ( <STR_LIT> ( ZImmVal ) != - <NUM_LIT> ) || ( <STR_LIT> ( ~ ZImmVal ) != - <NUM_LIT> ) ) return <NUM_LIT> ;


<s> if ( Imm == <NUM_LIT> || Imm == <NUM_LIT> ) return <NUM_LIT> ; return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( ~ Imm , Ty , CostKind ) ) ; } if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( - Imm , Ty , CostKind ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == <NUM_LIT> ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; if ( ST -> isThumb ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; } if ( Opcode == Instruction :: Xor && Imm . isAllOnes ( ) ) return <NUM_LIT> ; if ( Inst && ( ( ST -> hasV6Ops ( ) && ! ST -> isThumb ( ) ) || ST -> isThumb2 ( ) ) && Ty -> getIntegerBitWidth ( ) <= <NUM_LIT> ) {

if ( isSSATMinMaxPattern ( Inst , Imm ) || ( isa < ICmpInst > ( Inst ) && Inst -> hasOneUse ( ) && isSSATMinMaxPattern ( cast < Instruction > ( * Inst -> user_begin ( ) ) , Imm ) ) ) return <NUM_LIT> ;


<s> if ( Opcode == Instruction :: GetElementPtr && Idx != <NUM_LIT> ) return <NUM_LIT> ; if ( Opcode == Instruction :: And ) { if ( Imm == <NUM_LIT> || Imm == <NUM_LIT> ) return <NUM_LIT> ; return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( ~ Imm , Ty , CostKind ) ) ; } if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( - Imm , Ty , CostKind ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == <NUM_LIT> ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ;

if ( ST -> isThumb ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ;


<s> case <STR_LIT> : case <STR_LIT> : { if ( ! ST -> hasMVEIntegerOps ( ) ) break ; Type * VT = ICA . getReturnType ( ) ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , VT ) ; if ( LT . second == <STR_LIT> || LT . second == <STR_LIT> || LT . second == <STR_LIT> ) { unsigned Instrs = LT . second . getScalarSizeInBits ( ) == VT -> getScalarSizeInBits ( ) ? <NUM_LIT> : <NUM_LIT> ; return LT . first * ST -> getMVEVectorCostFactor ( CostKind ) * Instrs ; } break ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( ! ST -> hasMVEIntegerOps ( ) ) break ; Type * VT = ICA . getReturnType ( ) ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , VT ) ; if ( LT . second == <STR_LIT> || LT . second == <STR_LIT> || LT . second == <STR_LIT> ) return LT . first * ST -> getMVEVectorCostFactor ( CostKind ) ; break ; } case <STR_LIT> : case <STR_LIT> : { if ( ! ST -> hasMVEFloatOps ( ) ) break ; Type * VT = ICA . getReturnType ( ) ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , VT ) ; if ( LT . second == <STR_LIT> || LT . second == <STR_LIT> ) return LT . first * ST -> getMVEVectorCostFactor ( CostKind ) ; break ;


<s> if ( ST -> hasMVEIntegerOps ( ) ) { if ( Opcode == Instruction :: Load && isLegalMaskedLoad ( Src , Alignment ) ) return ST -> getMVEVectorCostFactor ( CostKind ) ;

if ( Opcode == Instruction :: Store && isLegalMaskedStore ( Src , Alignment ) ) return ST -> getMVEVectorCostFactor ( CostKind ) ;


<s> int NumOps = getNumMemOps ( cast < IntrinsicInst > ( I ) ) ; if ( NumOps == - <NUM_LIT> ) return <NUM_LIT> ; return NumOps ;


<s> if ( ST -> hasNEON ( ) && Src -> isVectorTy ( ) && ( Alignment && * Alignment != Align ( <NUM_LIT> ) ) && cast < VectorType > ( Src ) -> getElementType ( ) -> isDoubleTy ( ) ) { std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Src ) ; return LT . first * <NUM_LIT> ; } if ( ST -> hasMVEFloatOps ( ) && isa < FixedVectorType > ( Src ) && I && ( ( Opcode == Instruction :: Load && I -> hasOneUse ( ) && isa < FPExtInst > ( * I -> user_begin ( ) ) ) || ( Opcode == Instruction :: Store && isa < FPTruncInst > ( I -> getOperand ( <NUM_LIT> ) ) ) ) ) { FixedVectorType * SrcVTy = cast < FixedVectorType > ( Src ) ; Type * DstTy = Opcode == Instruction :: Load ? ( * I -> user_begin ( ) ) -> getType ( ) : cast < Instruction > ( I -> getOperand ( <NUM_LIT> ) ) -> getOperand ( <NUM_LIT> ) -> getType ( ) ; if ( SrcVTy -> getNumElements ( ) == <NUM_LIT> && SrcVTy -> getScalarType ( ) -> isHalfTy ( ) && DstTy -> getScalarType ( ) -> isFloatTy ( ) ) return ST -> getMVEVectorCostFactor ( CostKind ) ;


<s> InstructionCost <ISA_LIT>TTIImpl :: getMemoryOpCost ( unsigned Opcode , Type * Src , MaybeAlign Alignment , unsigned AddressSpace , TTI :: TargetCostKind CostKind , const Instruction * I ) { if ( CostKind != TTI :: TCK_RecipThroughput ) return <NUM_LIT> ; if ( TLI -> getValueType ( DL , Src , true ) == <STR_LIT> ) return BaseT :: getMemoryOpCost ( Opcode , Src , Alignment , AddressSpace , CostKind ) ; if ( ST -> hasNEON ( ) && Src -> isVectorTy ( ) && ( Alignment && * Alignment != Align ( <NUM_LIT> ) ) && cast < VectorType > ( Src ) -> getElementType ( ) -> isDoubleTy ( ) ) { std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Src ) ; return LT . first * <NUM_LIT> ; }

if ( ST -> hasMVEFloatOps ( ) && isa < FixedVectorType > ( Src ) && I && ( ( Opcode == Instruction :: Load && I -> hasOneUse ( ) && isa < FPExtInst > ( * I -> user_begin ( ) ) ) || ( Opcode == Instruction :: Store && isa < FPTruncInst > ( I -> getOperand ( <NUM_LIT> ) ) ) ) ) {


<s> } else if ( const auto * MS = dyn_cast < MemSetInst > ( I ) ) { ConstantInt * C = dyn_cast < ConstantInt > ( MS -> getLength ( ) ) ; if ( ! C ) return - <NUM_LIT> ; const unsigned Size = C -> getValue ( ) . getZExtValue ( ) ; const Align DstAlign = * MS -> getDestAlign ( ) ; MOp = MemOp :: Set ( Size , false , DstAlign , false , false ) ; DstAddrSpace = MS -> getDestAddressSpace ( ) ; } else llvm_unreachable ( <STR_LIT> ) ; unsigned Limit , Factor = <NUM_LIT> ; switch ( I -> getIntrinsicID ( ) ) { case <STR_LIT> : Limit = TLI -> getMaxStoresPerMemcpy ( F -> hasMinSize ( ) ) ; break ; case <STR_LIT> : Limit = TLI -> getMaxStoresPerMemmove ( F -> hasMinSize ( ) ) ; break ; case <STR_LIT> :

Limit = TLI -> getMaxStoresPerMemset ( F -> hasMinSize ( ) ) ;


<s> void <ISA_LIT>TTIImpl :: getPeelingPreferences ( Loop * L , ScalarEvolution & SE , TTI :: PeelingPreferences & PP ) { BaseT :: getPeelingPreferences ( L , SE , PP ) ;


<s> if ( ST -> isMClass ( ) && ST -> isThumb2 ( ) && L -> getNumBlocks ( ) == <NUM_LIT> ) return TTI :: AMK_PreIndexed ;


<s> if ( const auto * Entry = CostTableLookup ( NEONSelShuffleTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost ; } } if ( ST -> hasMVEIntegerOps ( ) ) { if ( Kind == TTI :: SK_Broadcast ) { static const CostTblEntry MVEDupTbl [ ] = { { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> } } ; std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( const auto * Entry = CostTableLookup ( MVEDupTbl , <STR_LIT> , LT . second ) ) return LT . first * Entry -> Cost * ST -> getMVEVectorCostFactor ( TTI :: TCK_RecipThroughput ) ; } if ( ! Mask . empty ( ) ) { std :: pair < InstructionCost , MVT > LT = TLI -> getTypeLegalizationCost ( DL , Tp ) ; if ( Mask . size ( ) <= LT . second . getVectorNumElements ( ) && ( isVREVMask ( Mask , LT . second , <NUM_LIT> ) || isVREVMask ( Mask , LT . second , <NUM_LIT> ) || isVREVMask ( Mask , LT . second , <NUM_LIT> ) ) ) return ST -> getMVEVectorCostFactor ( TTI :: TCK_RecipThroughput ) * LT . first ; } } int BaseCost = ST -> hasMVEIntegerOps ( ) && Tp -> isVectorTy ( ) ? ST -> getMVEVectorCostFactor ( TTI :: TCK_RecipThroughput ) : <NUM_LIT> ;

return BaseCost * BaseT :: getShuffleCost ( Kind , Tp , Mask , Index , SubTp ) ;


<s> if ( L -> getHeader ( ) -> getParent ( ) -> hasOptSize ( ) ) return ; SmallVector < BasicBlock * , <NUM_LIT> > ExitingBlocks ; L -> getExitingBlocks ( ExitingBlocks ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << L -> getNumBlocks ( ) << <STR_LIT> << <STR_LIT> << ExitingBlocks . size ( ) << <STR_LIT> ) ; if ( ExitingBlocks . size ( ) > <NUM_LIT> ) return ; if ( ST -> hasBranchPredictor ( ) && L -> getNumBlocks ( ) > <NUM_LIT> ) return ; if ( getBooleanLoopAttribute ( L , <STR_LIT> ) ) return ; InstructionCost Cost = <NUM_LIT> ; for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( I . getType ( ) -> isVectorTy ( ) ) return ; if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { if ( const Function * F = cast < CallBase > ( I ) . getCalledFunction ( ) ) { if ( ! isLoweredToCall ( F ) ) continue ; }


<s> if ( ST -> hasSlowLoadDSubregister ( ) && Opcode == Instruction :: InsertElement && ValTy -> isVectorTy ( ) && ValTy -> getScalarSizeInBits ( ) <= <NUM_LIT> ) return <NUM_LIT> ; if ( ST -> hasNEON ( ) && ( Opcode == Instruction :: InsertElement || Opcode == Instruction :: ExtractElement ) ) { if ( cast < VectorType > ( ValTy ) -> getElementType ( ) -> isIntegerTy ( ) ) return <NUM_LIT> ; if ( ValTy -> isVectorTy ( ) && ValTy -> getScalarSizeInBits ( ) <= <NUM_LIT> ) return std :: max < InstructionCost > ( BaseT :: getVectorInstrCost ( Opcode , ValTy , Index ) , <NUM_LIT> ) ;


<s> if ( cast < VectorType > ( ValTy ) -> getElementType ( ) -> isIntegerTy ( ) ) return <NUM_LIT> ; if ( ValTy -> isVectorTy ( ) && ValTy -> getScalarSizeInBits ( ) <= <NUM_LIT> ) return std :: max < InstructionCost > ( BaseT :: getVectorInstrCost ( Opcode , ValTy , Index ) , <NUM_LIT> ) ; } if ( ST -> hasMVEIntegerOps ( ) && ( Opcode == Instruction :: InsertElement || Opcode == Instruction :: ExtractElement ) ) { std :: pair < InstructionCost , MVT > LT = getTLI ( ) -> getTypeLegalizationCost ( DL , ValTy -> getScalarType ( ) ) ; return LT . first * ( ValTy -> getScalarType ( ) -> isIntegerTy ( ) ? <NUM_LIT> : <NUM_LIT> ) ;


<s> Value * Arg = II . getArgOperand ( <NUM_LIT> ) ; Value * ArgArg ; if ( match ( Arg , PatternMatch :: m_Intrinsic < <STR_LIT> > ( PatternMatch :: m_Value ( ArgArg ) ) ) ) { return IC . replaceInstUsesWith ( II , ArgArg ) ; } if ( ! II . getMetadata ( LLVMContext :: MD_range ) ) { Type * IntTy32 = Type :: getInt32Ty ( II . getContext ( ) ) ; Metadata * M [ ] = { ConstantAsMetadata :: get ( ConstantInt :: get ( IntTy32 , <NUM_LIT> ) ) , ConstantAsMetadata :: get ( ConstantInt :: get ( IntTy32 , <NUM_LIT> ) ) } ; II . setMetadata ( LLVMContext :: MD_range , MDNode :: get ( II . getContext ( ) , M ) ) ; return & II ; } break ; } case <STR_LIT> : case <STR_LIT> : { unsigned CarryOp = ( II . getIntrinsicID ( ) == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; assert ( II . getArgOperand ( CarryOp ) -> getType ( ) -> getScalarSizeInBits ( ) == <NUM_LIT> && <STR_LIT> ) ; KnownBits CarryKnown ( <NUM_LIT> ) ; if ( IC . SimplifyDemandedBits ( & II , CarryOp , APInt :: getOneBitSet ( <NUM_LIT> , <NUM_LIT> ) , CarryKnown ) ) { return & II ; } break ; } case <STR_LIT> : { Instruction * I = cast < Instruction > ( & II ) ; if ( I -> hasOneUse ( ) ) { auto * User = cast < Instruction > ( * I -> user_begin ( ) ) ; Value * OpZ ; if ( match ( User , m_c_Add ( m_Specific ( I ) , m_Value ( OpZ ) ) ) && match ( I -> getOperand ( <NUM_LIT> ) , m_Zero ( ) ) ) { Value * OpX = I -> getOperand ( <NUM_LIT> ) ; Value * OpY = I -> getOperand ( <NUM_LIT> ) ; Type * OpTy = OpX -> getType ( ) ; IC . Builder . SetInsertPoint ( User ) ; Value * V = IC . Builder . CreateIntrinsic ( <STR_LIT> , { OpTy } , { I -> getOperand ( <NUM_LIT> ) , I -> getOperand ( <NUM_LIT> ) , I -> getOperand ( <NUM_LIT> ) , OpZ , OpX , OpY } ) ;

IC . replaceInstUsesWith ( * User , V ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { Align MemAlign = getKnownAlignment ( II . getArgOperand ( <NUM_LIT> ) , IC . getDataLayout ( ) , & II , & IC . getAssumptionCache ( ) , & IC . getDominatorTree ( ) ) ; unsigned AlignArg = II . arg_size ( ) - <NUM_LIT> ; Value * AlignArgOp = II . getArgOperand ( AlignArg ) ; MaybeAlign Align = cast < ConstantInt > ( AlignArgOp ) -> getMaybeAlignValue ( ) ; if ( Align && * Align < MemAlign ) { return IC . replaceOperand ( II , AlignArg , ConstantInt :: get ( Type :: getInt32Ty ( II . getContext ( ) ) , MemAlign . value ( ) , false ) ) ; } break ; } case <STR_LIT> : { Value * Arg = II . getArgOperand ( <NUM_LIT> ) ; Value * ArgArg ; if ( match ( Arg , PatternMatch :: m_Intrinsic < <STR_LIT> > ( PatternMatch :: m_Value ( ArgArg ) ) ) && II . getType ( ) == ArgArg -> getType ( ) ) { return IC . replaceInstUsesWith ( II , ArgArg ) ; } Constant * XorMask ; if ( match ( Arg , m_Xor ( PatternMatch :: m_Intrinsic < <STR_LIT> > ( PatternMatch :: m_Value ( ArgArg ) ) , PatternMatch :: m_Constant ( XorMask ) ) ) && II . getType ( ) == ArgArg -> getType ( ) ) { if ( auto * CI = dyn_cast < ConstantInt > ( XorMask ) ) { if ( CI -> getValue ( ) . trunc ( <NUM_LIT> ) . isAllOnes ( ) ) { auto TrueVector = IC . Builder . CreateVectorSplat ( cast < FixedVectorType > ( II . getType ( ) ) -> getNumElements ( ) , IC . Builder . getTrue ( ) ) ; return BinaryOperator :: Create ( Instruction :: Xor , ArgArg , TrueVector ) ; } } } KnownBits ScalarKnown ( <NUM_LIT> ) ; if ( IC . SimplifyDemandedBits ( & II , <NUM_LIT> , APInt :: getLowBitsSet ( <NUM_LIT> , <NUM_LIT> ) , ScalarKnown , <NUM_LIT> ) ) { return & II ; } break ; } case <STR_LIT> : { Value * Arg = II . getArgOperand ( <NUM_LIT> ) ; Value * ArgArg ; if ( match ( Arg , PatternMatch :: m_Intrinsic < <STR_LIT> > ( PatternMatch :: m_Value ( ArgArg ) ) ) ) { return IC . replaceInstUsesWith ( II , ArgArg ) ; } if ( ! II . getMetadata ( LLVMContext :: MD_range ) ) { Type * IntTy32 = Type :: getInt32Ty ( II . getContext ( ) ) ;


<s> const SCEV * TripCountSCEV = SE . getAddExpr ( BackedgeTakenCount , SE . getOne ( BackedgeTakenCount -> getType ( ) ) ) ; if ( SE . getUnsignedRangeMax ( TripCountSCEV ) . getBitWidth ( ) > <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto IsHardwareLoopIntrinsic = [ ] ( Instruction & I ) { if ( auto * Call = dyn_cast < IntrinsicInst > ( & I ) ) { switch ( Call -> getIntrinsicID ( ) ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; } } return false ; } ; bool IsTailPredLoop = false ; auto ScanLoop = [ & ] ( Loop * L ) { for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( maybeLoweredToCall ( I ) || IsHardwareLoopIntrinsic ( I ) || isa < InlineAsm > ( I ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << I << <STR_LIT> ) ; return false ; } if ( auto * II = dyn_cast < IntrinsicInst > ( & I ) ) IsTailPredLoop |= II -> getIntrinsicID ( ) == <STR_LIT> || II -> getIntrinsicID ( ) == <STR_LIT> || II -> getIntrinsicID ( ) == <STR_LIT> || II -> getIntrinsicID ( ) == <STR_LIT> || II -> getIntrinsicID ( ) == <STR_LIT> ; } } return true ; } ; for ( auto Inner : * L ) if ( ! ScanLoop ( Inner ) ) return false ; if ( ! ScanLoop ( L ) ) return false ; LLVMContext & C = L -> getHeader ( ) -> getContext ( ) ; HWLoopInfo . CounterInReg = true ; HWLoopInfo . IsNestingLegal = false ; HWLoopInfo . PerformEntryTest = AllowWLSLoops && ! IsTailPredLoop ; HWLoopInfo . CountType = Type :: getInt32Ty ( C ) ; HWLoopInfo . LoopDecrement = ConstantInt :: get ( HWLoopInfo . CountType , <NUM_LIT> ) ; return true ;


<s> if ( ! EnableMaskedGatherScatters || ! ST -> hasMVEIntegerOps ( ) ) return false ; if ( isa < VectorType > ( Ty ) ) return false ; unsigned EltWidth = Ty -> getScalarSizeInBits ( ) ; return ( ( EltWidth == <NUM_LIT> && Alignment >= <NUM_LIT> ) || ( EltWidth == <NUM_LIT> && Alignment >= <NUM_LIT> ) || EltWidth == <NUM_LIT> ) ;


<s> if ( ! EnableMaskedGatherScatters || ! ST -> hasMVEIntegerOps ( ) ) return false ; if ( isa < VectorType > ( Ty ) ) return false ; unsigned EltWidth = Ty -> getScalarSizeInBits ( ) ;

return ( ( EltWidth == <NUM_LIT> && Alignment >= <NUM_LIT> ) || ( EltWidth == <NUM_LIT> && Alignment >= <NUM_LIT> ) || EltWidth == <NUM_LIT> ) ;


<s> if ( VecWidth != <NUM_LIT> && VecTy -> getElementType ( ) -> isFloatingPointTy ( ) ) return false ; } unsigned EltWidth = DataTy -> getScalarSizeInBits ( ) ;

return ( EltWidth == <NUM_LIT> && Alignment >= <NUM_LIT> ) || ( EltWidth == <NUM_LIT> && Alignment >= <NUM_LIT> ) || ( EltWidth == <NUM_LIT> ) ;


<s> auto isSSatMin = [ & ] ( Value * MinInst ) { if ( isa < SelectInst > ( MinInst ) ) { Value * MinLHS , * MinRHS ; ConstantInt * MinC ; SelectPatternFlavor MinSPF = matchSelectPattern ( MinInst , MinLHS , MinRHS ) . Flavor ; if ( MinSPF == SPF_SMIN && PatternMatch :: match ( MinRHS , PatternMatch :: m_ConstantInt ( MinC ) ) && MinC -> getValue ( ) == ( ( - Imm ) - <NUM_LIT> ) ) return true ; } return false ; } ; if ( isSSatMin ( Inst -> getOperand ( <NUM_LIT> ) ) || ( Inst -> hasNUses ( <NUM_LIT> ) && ( isSSatMin ( * Inst -> user_begin ( ) ) || isSSatMin ( * ( ++ Inst -> user_begin ( ) ) ) ) ) ) return true ;


<s> } if ( VT . isInteger ( ) && VT . getSizeInBits ( ) >= <NUM_LIT> ) { switch ( ISD ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; } } if ( ! VT . isFloatingPoint ( ) ) return false ; if ( TLI -> useSoftFloat ( ) ) { switch ( I . getOpcode ( ) ) { default : return true ; case Instruction :: Alloca : case Instruction :: Load : case Instruction :: Store : case Instruction :: Select : case Instruction :: PHI : return false ; } }

if ( I . getType ( ) -> isDoubleTy ( ) && ! ST -> hasFP64 ( ) ) return true ;


<s> bool <ISA_LIT>TTIImpl :: preferInLoopReduction ( unsigned Opcode , Type * Ty , TTI :: ReductionFlags Flags ) const { if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; unsigned ScalarBits = Ty -> getScalarSizeInBits ( ) ; switch ( Opcode ) { case Instruction :: Add : return ScalarBits <= <NUM_LIT> ; default : return false ;


<s> bool <ISA_LIT>TTIImpl :: preferPredicatedReductionSelect ( unsigned Opcode , Type * Ty , TTI :: ReductionFlags Flags ) const { if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; return true ;


<s> if ( ! ST -> hasMVEIntegerOps ( ) ) return false ;


<s> if ( ! ST -> hasMVEIntegerOps ( ) ) return false ; if ( L -> getNumBlocks ( ) > <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } assert ( L -> isInnermost ( ) && <STR_LIT> ) ; HardwareLoopInfo HWLoopInfo ( L ) ; if ( ! HWLoopInfo . canAnalyze ( * LI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } if ( ! isHardwareLoopProfitable ( L , SE , AC , TLI , HWLoopInfo ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ;


<s> UndefElts &= APInt :: getSplat ( NumElts , ! IsTop ? APInt :: getLowBitsSet ( <NUM_LIT> , <NUM_LIT> ) : APInt :: getHighBitsSet ( <NUM_LIT> , <NUM_LIT> ) ) ; return None ; } ; switch ( II . getIntrinsicID ( ) ) { default : break ; case <STR_LIT> : SimplifyNarrowInstrTopBottom ( <NUM_LIT> ) ; break ; case <STR_LIT> : SimplifyNarrowInstrTopBottom ( <NUM_LIT> ) ; break ; case <STR_LIT> :


<s> auto * BCastInst = Builder . CreateBitCast ( II . getArgOperand ( <NUM_LIT> ) , PointerType :: get ( II . getType ( ) , <NUM_LIT> ) ) ; return Builder . CreateAlignedLoad ( II . getType ( ) , BCastInst , Align ( Alignment ) ) ;


<s> unsigned Alignment = IntrAlign -> getLimitedValue ( ) < MemAlign ? MemAlign : IntrAlign -> getLimitedValue ( ) ; if ( ! isPowerOf2_32 ( Alignment ) ) return nullptr ; auto * BCastInst = Builder . CreateBitCast ( II . getArgOperand ( <NUM_LIT> ) , PointerType :: get ( II . getType ( ) , <NUM_LIT> ) ) ;

return Builder . CreateAlignedLoad ( II . getType ( ) , BCastInst , Align ( Alignment ) ) ;


<s> unsigned Cost = <NUM_LIT> ; for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { ImmutableCallSite CS ( & I ) ; if ( const Function * F = CS . getCalledFunction ( ) ) { if ( ! isLoweredToCall ( F ) ) continue ; } return ; } SmallVector < const Value * , <NUM_LIT> > Operands ( I . value_op_begin ( ) , I . value_op_end ( ) ) ; Cost += getUserCost ( & I , Operands ) ; }


<s> if ( ST -> hasBranchPredictor ( ) && L -> getNumBlocks ( ) > <NUM_LIT> ) return ; unsigned Cost = <NUM_LIT> ; for ( auto * BB : L -> getBlocks ( ) ) { for ( auto & I : * BB ) { if ( isa < CallInst > ( I ) || isa < InvokeInst > ( I ) ) { ImmutableCallSite CS ( & I ) ; if ( const Function * F = CS . getCalledFunction ( ) ) { if ( ! isLoweredToCall ( F ) ) continue ; } return ; } SmallVector < const Value * , <NUM_LIT> > Operands ( I . value_op_begin ( ) , I . value_op_end ( ) ) ; Cost += getUserCost ( & I , Operands ) ; } } DEBUG ( dbgs ( ) << <STR_LIT> << Cost << <STR_LIT> ) ; UP . Partial = true ; UP . Runtime = true ; UP . UnrollRemainder = true ; UP . DefaultUnrollRuntimeCount = <NUM_LIT> ; if ( Cost < <NUM_LIT> ) UP . Force = true ;


<s> int <ISA_LIT>TTIImpl :: getIntImmCost ( unsigned Opcode , unsigned Idx , const APInt & Imm , Type * Ty ) { if ( ( Opcode == Instruction :: SDiv || Opcode == Instruction :: UDiv || Opcode == Instruction :: SRem || Opcode == Instruction :: URem ) && Idx == <NUM_LIT> ) return <NUM_LIT> ; return getIntImmCost ( Imm , Ty ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; T -> dump ( ) ) ; return false ; } if ( isa < StoreInst > ( I ) || isa < LoadInst > ( I ) ) { Value * Ptr = getLoadStorePointerOperand ( & I ) ; Type * AccessTy = getLoadStoreType ( & I ) ; int64_t NextStride = getPtrStride ( PSE , AccessTy , Ptr , L ) ; if ( NextStride == <NUM_LIT> ) { continue ; } else if ( NextStride == - <NUM_LIT> || ( NextStride == <NUM_LIT> && MVEMaxSupportedInterleaveFactor >= <NUM_LIT> ) || ( NextStride == <NUM_LIT> && MVEMaxSupportedInterleaveFactor >= <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } else if ( EnableMaskedGatherScatters ) { const SCEV * PtrScev = PSE . getSE ( ) -> getSCEV ( Ptr ) ; if ( auto AR = dyn_cast < SCEVAddRecExpr > ( PtrScev ) ) { const SCEV * Step = AR -> getStepRecurrence ( * PSE . getSE ( ) ) ; if ( PSE . getSE ( ) -> isLoopInvariant ( Step , L ) ) continue ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } } }


<s> } if ( Opcode == Instruction :: Add ) return std :: min ( getIntImmCost ( Imm , Ty , CostKind ) , getIntImmCost ( - Imm , Ty , CostKind ) ) ; if ( Opcode == Instruction :: ICmp && Imm . isNegative ( ) && Ty -> getIntegerBitWidth ( ) == <NUM_LIT> ) { int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; if ( ST -> isThumb ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; } if ( Opcode == Instruction :: Xor && Imm . isAllOnes ( ) ) return <NUM_LIT> ; if ( Inst && ( ( ST -> hasV6Ops ( ) && ! ST -> isThumb ( ) ) || ST -> isThumb2 ( ) ) && Ty -> getIntegerBitWidth ( ) <= <NUM_LIT> ) { if ( isSSATMinMaxPattern ( Inst , Imm ) || ( isa < ICmpInst > ( Inst ) && Inst -> hasOneUse ( ) && isSSATMinMaxPattern ( cast < Instruction > ( * Inst -> user_begin ( ) ) , Imm ) ) ) return <NUM_LIT> ; } if ( Inst && ST -> hasVFP2Base ( ) && isFPSatMinMaxPattern ( Inst , Imm ) ) return <NUM_LIT> ;

if ( Inst && Opcode == Instruction :: ICmp && Idx == <NUM_LIT> && Imm . isAllOnesValue ( ) ) {


<s> int64_t NegImm = - Imm . getSExtValue ( ) ; if ( ST -> isThumb2 ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; if ( ST -> isThumb ( ) && NegImm < <NUM_LIT> << <NUM_LIT> ) return <NUM_LIT> ; } if ( Opcode == Instruction :: Xor && Imm . isAllOnes ( ) ) return <NUM_LIT> ; if ( Inst && ( ( ST -> hasV6Ops ( ) && ! ST -> isThumb ( ) ) || ST -> isThumb2 ( ) ) && Ty -> getIntegerBitWidth ( ) <= <NUM_LIT> ) {


<s> static bool isFPSatMinMaxPattern ( Instruction * Inst , const APInt & Imm ) { if ( Imm . getBitWidth ( ) != <NUM_LIT> || Imm != APInt :: getHighBitsSet ( <NUM_LIT> , <NUM_LIT> ) ) return false ; Value * FP = isSSATMinMaxPattern ( Inst , Imm ) ; if ( ! FP && isa < ICmpInst > ( Inst ) && Inst -> hasOneUse ( ) ) FP = isSSATMinMaxPattern ( cast < Instruction > ( * Inst -> user_begin ( ) ) , Imm ) ; if ( ! FP ) return false ; return isa < FPToSIInst > ( FP ) ;


<s> static bool isFPSatMinMaxPattern ( Instruction * Inst , const APInt & Imm ) {

if ( Imm . getBitWidth ( ) != <NUM_LIT> || Imm != APInt :: getHighBitsSet ( <NUM_LIT> , <NUM_LIT> ) ) return false ;


<s> if ( ! EnableMaskedGatherScatters || ! ST -> hasMVEIntegerOps ( ) ) return false ; unsigned EltWidth = Ty -> getScalarSizeInBits ( ) ;


<s> Value * LHS , * RHS ; ConstantInt * C ; SelectPatternFlavor InstSPF = matchSelectPattern ( Inst , LHS , RHS ) . Flavor ; if ( InstSPF == SPF_SMAX && PatternMatch :: match ( RHS , PatternMatch :: m_ConstantInt ( C ) ) && C -> getValue ( ) == Imm && Imm . isNegative ( ) && Imm . isNegatedPowerOf2 ( ) ) { auto isSSatMin = [ & ] ( Value * MinInst ) { if ( isa < SelectInst > ( MinInst ) ) { Value * MinLHS , * MinRHS ; ConstantInt * MinC ; SelectPatternFlavor MinSPF = matchSelectPattern ( MinInst , MinLHS , MinRHS ) . Flavor ; if ( MinSPF == SPF_SMIN && PatternMatch :: match ( MinRHS , PatternMatch :: m_ConstantInt ( MinC ) ) && MinC -> getValue ( ) == ( ( - Imm ) - <NUM_LIT> ) ) return true ; } return false ; } ; if ( isSSatMin ( Inst -> getOperand ( <NUM_LIT> ) ) ) return cast < Instruction > ( Inst -> getOperand ( <NUM_LIT> ) ) -> getOperand ( <NUM_LIT> ) ; if ( Inst -> hasNUses ( <NUM_LIT> ) && ( isSSatMin ( * Inst -> user_begin ( ) ) || isSSatMin ( * ( ++ Inst -> user_begin ( ) ) ) ) ) return Inst -> getOperand ( <NUM_LIT> ) ;


<s> SelectPatternFlavor MinSPF = matchSelectPattern ( MinInst , MinLHS , MinRHS ) . Flavor ; if ( MinSPF == SPF_SMIN && PatternMatch :: match ( MinRHS , PatternMatch :: m_ConstantInt ( MinC ) ) && MinC -> getValue ( ) == ( ( - Imm ) - <NUM_LIT> ) ) return true ; } return false ; } ; if ( isSSatMin ( Inst -> getOperand ( <NUM_LIT> ) ) ) return cast < Instruction > ( Inst -> getOperand ( <NUM_LIT> ) ) -> getOperand ( <NUM_LIT> ) ; if ( Inst -> hasNUses ( <NUM_LIT> ) && ( isSSatMin ( * Inst -> user_begin ( ) ) || isSSatMin ( * ( ++ Inst -> user_begin ( ) ) ) ) ) return Inst -> getOperand ( <NUM_LIT> ) ;


<s> unsigned <STR_LIT> ( Type * Ty , bool IsComplex ) const {

unsigned NumVectorInstToHideOverhead = <NUM_LIT> ;