<s> static tree cygwin_d_handle_target_object_format ( void ) { const char * objfmt = "coff" ; return build_string_literal ( strlen ( objfmt ) + 1 , objfmt ) ; } </s>

Handle a call to ` __traits ( getTargetInfo , `` objectFormat '' ) ' .


<s> static void cygwin_d_os_builtins ( void ) { d_add_builtin_version ( "Windows" ) ; d_add_builtin_version ( "Cygwin" ) ; d_add_builtin_version ( "Posix" ) ; d_add_builtin_version ( "CRuntime_Newlib" ) ; } </s>

Implement TARGET_D_OS_VERSIONS for Cygwin targets .


<s> static void cygwin_d_register_target_info ( void ) { const struct d_target_info_spec handlers [ ] = { { "objectFormat" , cygwin_d_handle_target_object_format } , { NULL , NULL } , } ; d_add_target_info_handlers ( handlers ) ; } </s>

Implement TARGET_D_REGISTER_OS_TARGET_INFO for Cygwin targets .


<s> void i386_djgpp_asm_named_section ( const char * name , unsigned int flags , tree ) { char flagchars [ 8 ] , * f = flagchars ; if ( flags & SECTION_WRITE ) * f ++ = 'w' ; if ( flags & SECTION_CODE ) * f ++ = 'x' ; if ( strncmp ( name , LTO_SECTION_NAME_PREFIX , strlen ( LTO_SECTION_NAME_PREFIX ) ) == 0 ) * f ++ = '0' ; * f ++ = '\0' ; fprintf ( asm_out_file , "\t.section\t%s,\"%s\"\n" , name , flagchars ) ; } </s>


<s>
static
char
*
describe_cache
(
struct
cache_desc
level1
,
struct
cache_desc
level2
)
{
char
size
[
100
]
,
line
[
100
]
,
size2
[
100
]
;
snprintf
(
size
,
sizeof
(
size
)
,
"--param l1-cache-size=%u "
,
level1
.
sizekb
)
;
snprintf
(
line
,
sizeof
(
line
)
,
"--param l1-cache-line-size=%u "
,
level1
.
line
)
;
snprintf
(
size2
,
sizeof
(
size2
)
,
"--param l2-cache-size=%u "
,
level2
.
sizekb
)
;
return
concat
(
size
,
line
,
size2
,
)
;
}
</s>

Returns parameters that describe L1_ASSOC associative cache of size L1_SIZEKB with lines of size L1_LINE , and L2_SIZEKB .


<s> static const char * detect_caches_amd ( unsigned max_ext_level ) { unsigned eax , ebx , ecx , edx ; struct cache_desc level1 , level2 = { 0 , 0 , 0 } ; if ( max_ext_level < 0x80000005 ) return "" ; __cpuid ( 0x80000005 , eax , ebx , ecx , edx ) ; level1 . sizekb = ( ecx >> 24 ) & 0xff ; level1 . assoc = ( ecx >> 16 ) & 0xff ; level1 . line = ecx & 0xff ; if ( max_ext_level >= 0x80000006 ) detect_l2_cache ( & level2 ) ; return describe_cache ( level1 , level2 ) ; } </s>

Returns the description of caches for an AMD processor .


<s> static void detect_caches_cpuid2 ( bool xeon_mp , struct cache_desc * level1 , struct cache_desc * level2 ) { unsigned regs [ 4 ] ; int nreps , i ; __cpuid ( 2 , regs [ 0 ] , regs [ 1 ] , regs [ 2 ] , regs [ 3 ] ) ; nreps = regs [ 0 ] & 0x0f ; regs [ 0 ] &= ~ 0x0f ; while ( -- nreps >= 0 ) { for ( i = 0 ; i < 4 ; i ++ ) if ( regs [ i ] && ! ( ( regs [ i ] >> 31 ) & 1 ) ) decode_caches_intel ( regs [ i ] , xeon_mp , level1 , level2 ) ; if ( nreps ) __cpuid ( 2 , regs [ 0 ] , regs [ 1 ] , regs [ 2 ] , regs [ 3 ] ) ; } } </s>

Detect cache parameters using CPUID function 2 .


<s> static const char * detect_caches_intel ( bool xeon_mp , unsigned max_level , unsigned max_ext_level , unsigned * l2sizekb ) { struct cache_desc level1 = { 0 , 0 , 0 } , level2 = { 0 , 0 , 0 } , level3 = { 0 , 0 , 0 } ; if ( max_level >= 4 ) detect_caches_cpuid4 ( & level1 , & level2 , & level3 ) ; else if ( max_level >= 2 ) detect_caches_cpuid2 ( xeon_mp , & level1 , & level2 ) ; else return "" ; if ( level1 . sizekb == 0 ) return "" ; if ( level3 . sizekb ) level2 = level3 ; if ( level2 . sizekb == 0 && max_ext_level >= 0x80000006 ) detect_l2_cache ( & level2 ) ; * l2sizekb = level2 . sizekb ; return describe_cache ( level1 , level2 ) ; } </s>

Returns the description of caches for an Intel processor .


<s> static void detect_l2_cache ( struct cache_desc * level2 ) { unsigned eax , ebx , ecx , edx ; unsigned assoc ; __cpuid ( 0x80000006 , eax , ebx , ecx , edx ) ; level2 -> sizekb = ( ecx >> 16 ) & 0xffff ; level2 -> line = ecx & 0xff ; assoc = ( ecx >> 12 ) & 0xf ; if ( assoc == 6 ) assoc = 8 ; else if ( assoc == 8 ) assoc = 16 ; else if ( assoc >= 0xa && assoc <= 0xc ) assoc = 32 + ( assoc - 0xa ) * 16 ; else if ( assoc >= 0xd && assoc <= 0xe ) assoc = 96 + ( assoc - 0xd ) * 32 ; level2 -> assoc = assoc ; } </s>

Detect L2 cache parameters using CPUID extended function 0x80000006 .


<s>
static
void
*
cygwin_gt_pch_get_address
(
size_t
sz
,
int
fd
)
{
void
*
base
;
off_t
p
=
lseek
(
fd
,
0
,
SEEK_CUR
)
;
if
(
p
==
(
off_t
)
1
)
fatal_error
(
"can't get position in PCH file: %m"
)
;
if
(
(
size_t
)
p
<
sz
)
{
if
(
ftruncate
(
fd
,
sz
)
==
1
)
fatal_error
(
"can't extend PCH file: %m"
)
;
}
base
=
mmap
(
,
sz
,
PROT_READ
|
PROT_WRITE
,
MAP_PRIVATE
,
fd
,
0
)
;
if
(
base
==
MAP_FAILED
)
base
=
;
else
munmap
(
base
,
sz
)
;
if
(
lseek
(
fd
,
p
,
SEEK_SET
)
==
(
off_t
)
1
)
fatal_error
(
"can't set position in PCH file: %m"
)
;
return
base
;
}
</s>

Identify an address that 's likely to be free in a subsequent invocation of the compiler . The area should be able to hold SIZE bytes . FD is an open file descriptor if the host would like to probe with mmap .


<s> static size_t cygwin_gt_pch_alloc_granularity ( void ) { return va_granularity ; } </s>

Return the alignment required for allocating virtual memory .


<s>
static
void
*
cygwin_gt_pch_get_address
(
size_t
sz
,
int
fd
)
{
void
*
base
;
off_t
p
=
lseek
(
fd
,
0
,
SEEK_CUR
)
;
if
(
p
==
(
off_t
)
1
)
fatal_error
(
input_location
,
"cannot get position in PCH file: %m"
)
;
if
(
(
size_t
)
p
<
sz
)
{
if
(
ftruncate
(
fd
,
sz
)
==
1
)
fatal_error
(
input_location
,
"cannot extend PCH file: %m"
)
;
}
base
=
mmap
(
,
sz
,
PROT_READ
|
PROT_WRITE
,
MAP_PRIVATE
,
fd
,
0
)
;
if
(
base
==
MAP_FAILED
)
base
=
;
else
munmap
(
base
,
sz
)
;
if
(
lseek
(
fd
,
p
,
SEEK_SET
)
==
(
off_t
)
1
)
fatal_error
(
input_location
,
"cannot set position in PCH file: %m"
)
;
return
base
;
}
</s>


<s> static void * mingw32_gt_pch_get_address ( size_t size , int ) { void * res ; res = VirtualAlloc ( NULL , size , MEM_RESERVE | MEM_TOP_DOWN , PAGE_NOACCESS ) ; if ( ! res ) w32_error ( __FUNCTION__ , __FILE__ , __LINE__ , "VirtualAlloc" ) ; else VirtualFree ( res , 0 , MEM_RELEASE ) ; return res ; } </s>


<s> static int mingw32_gt_pch_use_address ( void * & addr , size_t size , int fd , size_t offset ) { void * mmap_addr ; HANDLE mmap_handle ; char * object_name = NULL ; OSVERSIONINFO version_info ; int r ; version_info . dwOSVersionInfoSize = sizeof ( version_info ) ; if ( size == 0 ) return 0 ; if ( ( offset & ( va_granularity - 1 ) ) != 0 ) return - 1 ; GetVersionEx ( & version_info ) ; char local_object_name [ sizeof ( OBJECT_NAME_FMT ) + sizeof ( DWORD ) * 2 ] ; if ( version_info . dwMajorVersion > 4 ) { snprintf ( local_object_name , sizeof ( local_object_name ) , OBJECT_NAME_FMT "%lx" , GetCurrentProcessId ( ) ) ; object_name = local_object_name ; } mmap_handle = CreateFileMappingA ( ( HANDLE ) _get_osfhandle ( fd ) , NULL , PAGE_WRITECOPY | SEC_COMMIT , 0 , 0 , object_name ) ; if ( mmap_handle == NULL ) { w32_error ( __FUNCTION__ , __FILE__ , __LINE__ , "CreateFileMapping" ) ; return - 1 ; } for ( r = 0 ; r < 5 ; r ++ ) { mmap_addr = MapViewOfFileEx ( mmap_handle , FILE_MAP_COPY , 0 , offset , size , addr ) ; if ( mmap_addr == addr ) break ; if ( r != 4 ) Sleep ( 500 ) ; } if ( mmap_addr != addr ) { w32_error ( __FUNCTION__ , __FILE__ , __LINE__ , "MapViewOfFileEx" ) ; CloseHandle ( mmap_handle ) ; return - 1 ; } return 1 ; } </s>

ADDR is an address returned by gt_pch_get_address . Attempt to allocate SIZE bytes at the same address and load it with the data from FD at OFFSET . Return -1 if we could n't allocate memory at ADDR , return 0 if the memory is allocated but the data not loaded , return 1 if done .


<s> static size_t mingw32_gt_pch_alloc_granularity ( void ) { return va_granularity ; } </s>

Granularity for reserving address space .


<s> static void * mingw32_gt_pch_get_address ( size_t size , int fd ATTRIBUTE_UNUSED ) { void * res ; size = ( size + va_granularity - 1 ) & ~ ( va_granularity - 1 ) ; if ( size > pch_VA_max_size ) return NULL ; res = VirtualAlloc ( NULL , pch_VA_max_size , MEM_RESERVE | MEM_TOP_DOWN , PAGE_NOACCESS ) ; if ( ! res ) w32_error ( __FUNCTION__ , __FILE__ , __LINE__ , "VirtualAlloc" ) ; else VirtualFree ( res , 0 , MEM_RELEASE ) ; return res ; } </s>


<s> static int mingw32_gt_pch_use_address ( void * addr , size_t size , int fd , size_t offset ) { void * mmap_addr ; static HANDLE mmap_handle ; if ( size == 0 ) return 0 ; if ( ( offset & ( va_granularity - 1 ) ) != 0 || size > pch_VA_max_size ) return - 1 ; mmap_handle = CreateFileMapping ( ( HANDLE ) _get_osfhandle ( fd ) , NULL , PAGE_WRITECOPY | SEC_COMMIT , 0 , 0 , NULL ) ; if ( mmap_handle == NULL ) { w32_error ( __FUNCTION__ , __FILE__ , __LINE__ , "CreateFileMapping" ) ; return - 1 ; } mmap_addr = MapViewOfFileEx ( mmap_handle , FILE_MAP_COPY , 0 , offset , size , addr ) ; if ( mmap_addr != addr ) { w32_error ( __FUNCTION__ , __FILE__ , __LINE__ , "MapViewOfFileEx" ) ; CloseHandle ( mmap_handle ) ; return - 1 ; } return 1 ; } </s>


<s> static int mingw32_gt_pch_use_address ( void * addr , size_t size , int fd , size_t offset ) { void * mmap_addr ; HANDLE mmap_handle ; char * object_name = NULL ; OSVERSIONINFO version_info ; int r ; version_info . dwOSVersionInfoSize = sizeof ( version_info ) ; if ( size == 0 ) return 0 ; if ( ( offset & ( va_granularity - 1 ) ) != 0 || size > pch_VA_max_size ) return - 1 ; GetVersionEx ( & version_info ) ; if ( version_info . dwMajorVersion > 4 ) { char local_object_name [ sizeof ( OBJECT_NAME_FMT ) + sizeof ( DWORD ) * 2 ] ; snprintf ( local_object_name , sizeof ( local_object_name ) , OBJECT_NAME_FMT "%lx" , GetCurrentProcessId ( ) ) ; object_name = local_object_name ; } mmap_handle = CreateFileMappingA ( ( HANDLE ) _get_osfhandle ( fd ) , NULL , PAGE_WRITECOPY | SEC_COMMIT , 0 , 0 , object_name ) ; if ( mmap_handle == NULL ) { w32_error ( __FUNCTION__ , __FILE__ , __LINE__ , "CreateFileMapping" ) ; return - 1 ; } for ( r = 0 ; r < 5 ; r ++ ) { mmap_addr = MapViewOfFileEx ( mmap_handle , FILE_MAP_COPY , 0 , offset , size , addr ) ; if ( mmap_addr == addr ) break ; if ( r != 4 ) Sleep ( 500 ) ; } if ( mmap_addr != addr ) { w32_error ( __FUNCTION__ , __FILE__ , __LINE__ , "MapViewOfFileEx" ) ; CloseHandle ( mmap_handle ) ; return - 1 ; } return 1 ; } </s>


<s> static size_t mingw32_gt_pch_alloc_granularity ( void ) { SYSTEM_INFO si ; GetSystemInfo ( & si ) ; va_granularity = ( size_t ) si . dwAllocationGranularity ; return va_granularity ; } </s>


<s> static void * mingw32_gt_pch_get_address ( size_t size , int ) { void * res ; size = ( size + va_granularity - 1 ) & ~ ( va_granularity - 1 ) ; if ( size > pch_VA_max_size ) return NULL ; res = VirtualAlloc ( NULL , pch_VA_max_size , MEM_RESERVE | MEM_TOP_DOWN , PAGE_NOACCESS ) ; if ( ! res ) w32_error ( __FUNCTION__ , __FILE__ , __LINE__ , "VirtualAlloc" ) ; else VirtualFree ( res , 0 , MEM_RELEASE ) ; return res ; } </s>


<s> static int mingw32_gt_pch_use_address ( void * & addr , size_t size , int fd , size_t offset ) { void * mmap_addr ; HANDLE mmap_handle ; char * object_name = NULL ; OSVERSIONINFO version_info ; int r ; version_info . dwOSVersionInfoSize = sizeof ( version_info ) ; if ( size == 0 ) return 0 ; if ( ( offset & ( va_granularity - 1 ) ) != 0 || size > pch_VA_max_size ) return - 1 ; GetVersionEx ( & version_info ) ; char local_object_name [ sizeof ( OBJECT_NAME_FMT ) + sizeof ( DWORD ) * 2 ] ; if ( version_info . dwMajorVersion > 4 ) { snprintf ( local_object_name , sizeof ( local_object_name ) , OBJECT_NAME_FMT "%lx" , GetCurrentProcessId ( ) ) ; object_name = local_object_name ; } mmap_handle = CreateFileMappingA ( ( HANDLE ) _get_osfhandle ( fd ) , NULL , PAGE_WRITECOPY | SEC_COMMIT , 0 , 0 , object_name ) ; if ( mmap_handle == NULL ) { w32_error ( __FUNCTION__ , __FILE__ , __LINE__ , "CreateFileMapping" ) ; return - 1 ; } for ( r = 0 ; r < 5 ; r ++ ) { mmap_addr = MapViewOfFileEx ( mmap_handle , FILE_MAP_COPY , 0 , offset , size , addr ) ; if ( mmap_addr == addr ) break ; if ( r != 4 ) Sleep ( 500 ) ; } if ( mmap_addr != addr ) { w32_error ( __FUNCTION__ , __FILE__ , __LINE__ , "MapViewOfFileEx" ) ; CloseHandle ( mmap_handle ) ; return - 1 ; } return 1 ; } </s>


"internal error in %s, at %s:%d: %s: %s\n"

Print out the GetLastError ( ) translation .


<s> static inline tree def_builtin ( HOST_WIDE_INT mask , HOST_WIDE_INT mask2 , const char * name , enum ix86_builtin_func_type tcode , enum ix86_builtins code ) { tree decl = NULL_TREE ; if ( ! ( mask & OPTION_MASK_ISA_64BIT ) || TARGET_64BIT ) { ix86_builtins_isa [ ( int ) code ] . isa = mask ; ix86_builtins_isa [ ( int ) code ] . isa2 = mask2 ; mask &= ~ OPTION_MASK_ISA_64BIT ; if ( ( mask & ix86_isa_flags & OPTION_MASK_ISA_AVX512VL ) && mask != OPTION_MASK_ISA_AVX512VL ) mask &= ~ OPTION_MASK_ISA_AVX512VL ; if ( ( mask & ix86_isa_flags & OPTION_MASK_ISA_AVX512BW ) && mask != OPTION_MASK_ISA_AVX512BW ) mask &= ~ OPTION_MASK_ISA_AVX512BW ; if ( ( ( mask2 == 0 || ( mask2 & ix86_isa_flags2 ) != 0 ) && ( mask == 0 || ( mask & ix86_isa_flags ) != 0 ) ) || ( ( mask & OPTION_MASK_ISA_MMX ) != 0 && TARGET_MMX_WITH_SSE ) || ( mask2 == OPTION_MASK_ISA2_AVXVNNI ) || ( mask2 == OPTION_MASK_ISA2_AVXIFMA ) || ( mask2 == ( OPTION_MASK_ISA2_AVXNECONVERT | OPTION_MASK_ISA2_AVX512BF16 ) ) || ( lang_hooks . builtin_function == lang_hooks . builtin_function_ext_scope ) ) { tree type = ix86_get_builtin_func_type ( tcode ) ; decl = add_builtin_function ( name , type , code , BUILT_IN_MD , NULL , NULL_TREE ) ; ix86_builtins [ ( int ) code ] = decl ; ix86_builtins_isa [ ( int ) code ] . set_and_not_built_p = false ; } else { deferred_isa_values |= mask ; deferred_isa_values2 |= mask2 ; ix86_builtins [ ( int ) code ] = NULL_TREE ; ix86_builtins_isa [ ( int ) code ] . tcode = tcode ; ix86_builtins_isa [ ( int ) code ] . name = name ; ix86_builtins_isa [ ( int ) code ] . const_p = false ; ix86_builtins_isa [ ( int ) code ] . pure_p = false ; ix86_builtins_isa [ ( int ) code ] . set_and_not_built_p = true ; } } return decl ; } </s>

Add a SPARC builtin function with NAME , ICODE , CODE and TYPE . Return the function decl or NULL_TREE if the builtin was not added .


<s> void ix86_add_new_builtins ( HOST_WIDE_INT isa , HOST_WIDE_INT isa2 ) { isa &= ~ OPTION_MASK_ISA_64BIT ; if ( ( isa & deferred_isa_values ) == 0 && ( isa2 & deferred_isa_values2 ) == 0 && ( ( deferred_isa_values & OPTION_MASK_ISA_MMX ) == 0 || ! ( TARGET_64BIT && ( isa & OPTION_MASK_ISA_SSE2 ) != 0 ) ) ) return ; deferred_isa_values &= ~ isa ; deferred_isa_values2 &= ~ isa2 ; if ( TARGET_64BIT && ( isa & OPTION_MASK_ISA_SSE2 ) != 0 ) deferred_isa_values &= ~ OPTION_MASK_ISA_MMX ; int i ; tree saved_current_target_pragma = current_target_pragma ; current_target_pragma = NULL_TREE ; for ( i = 0 ; i < ( int ) IX86_BUILTIN_MAX ; i ++ ) { if ( ( ( ix86_builtins_isa [ i ] . isa & isa ) != 0 || ( ix86_builtins_isa [ i ] . isa2 & isa2 ) != 0 || ( ( ix86_builtins_isa [ i ] . isa & OPTION_MASK_ISA_MMX ) != 0 && TARGET_64BIT && ( isa & OPTION_MASK_ISA_SSE2 ) != 0 ) ) && ix86_builtins_isa [ i ] . set_and_not_built_p ) { tree decl , type ; ix86_builtins_isa [ i ] . set_and_not_built_p = false ; type = ix86_get_builtin_func_type ( ix86_builtins_isa [ i ] . tcode ) ; decl = add_builtin_function_ext_scope ( ix86_builtins_isa [ i ] . name , type , i , BUILT_IN_MD , NULL , NULL_TREE ) ; ix86_builtins [ i ] = decl ; if ( ix86_builtins_isa [ i ] . const_p ) TREE_READONLY ( decl ) = 1 ; if ( ix86_builtins_isa [ i ] . pure_p ) DECL_PURE_P ( decl ) = 1 ; } } current_target_pragma = saved_current_target_pragma ; } </s>

Add any new builtin functions for a given ISA that may not have been declared . This saves a bit of space compared to adding all of the declarations to the tree , even if we did n't use them .


Returns a function decl for a vectorized version of the builtin function with builtin function code FN and the result vector type TYPE , or NULL_TREE if it is not available .


<s> static tree build_processor_model_struct ( void ) { const char * field_name [ ] = { "__cpu_vendor" , "__cpu_type" , "__cpu_subtype" , "__cpu_features" } ; tree field = NULL_TREE , field_chain = NULL_TREE ; int i ; tree type = make_node ( RECORD_TYPE ) ; for ( i = 0 ; i < 3 ; ++ i ) { field = build_decl ( UNKNOWN_LOCATION , FIELD_DECL , get_identifier ( field_name [ i ] ) , unsigned_type_node ) ; if ( field_chain != NULL_TREE ) DECL_CHAIN ( field ) = field_chain ; field_chain = field ; } field = build_decl ( UNKNOWN_LOCATION , FIELD_DECL , get_identifier ( field_name [ 3 ] ) , build_array_type ( unsigned_type_node , build_index_type ( size_one_node ) ) ) ; if ( field_chain != NULL_TREE ) DECL_CHAIN ( field ) = field_chain ; field_chain = field ; finish_builtin_struct ( type , "__processor_model" , field_chain , NULL_TREE ) ; return type ; } </s>

This builds the processor_model struct type defined in libgcc/config/i386/cpuinfo.c


<s> static inline tree def_builtin ( HOST_WIDE_INT mask , HOST_WIDE_INT mask2 , const char * name , enum ix86_builtin_func_type tcode , enum ix86_builtins code ) { tree decl = NULL_TREE ; if ( ! ( mask & OPTION_MASK_ISA_64BIT ) || TARGET_64BIT ) { ix86_builtins_isa [ ( int ) code ] . isa = mask ; ix86_builtins_isa [ ( int ) code ] . isa2 = mask2 ; mask &= ~ OPTION_MASK_ISA_64BIT ; if ( ( mask & ix86_isa_flags & OPTION_MASK_ISA_AVX512VL ) && mask != OPTION_MASK_ISA_AVX512VL ) mask &= ~ OPTION_MASK_ISA_AVX512VL ; if ( ( mask & ix86_isa_flags & OPTION_MASK_ISA_AVX512BW ) && mask != OPTION_MASK_ISA_AVX512BW ) mask &= ~ OPTION_MASK_ISA_AVX512BW ; if ( ( ( mask2 == 0 || ( mask2 & ix86_isa_flags2 ) != 0 ) && ( mask == 0 || ( mask & ix86_isa_flags ) != 0 ) ) || ( ( mask & OPTION_MASK_ISA_MMX ) != 0 && TARGET_MMX_WITH_SSE ) || ( mask2 == OPTION_MASK_ISA2_AVXVNNI ) || ( lang_hooks . builtin_function == lang_hooks . builtin_function_ext_scope ) ) { tree type = ix86_get_builtin_func_type ( tcode ) ; decl = add_builtin_function ( name , type , code , BUILT_IN_MD , NULL , NULL_TREE ) ; ix86_builtins [ ( int ) code ] = decl ; ix86_builtins_isa [ ( int ) code ] . set_and_not_built_p = false ; } else { deferred_isa_values |= mask ; deferred_isa_values2 |= mask2 ; ix86_builtins [ ( int ) code ] = NULL_TREE ; ix86_builtins_isa [ ( int ) code ] . tcode = tcode ; ix86_builtins_isa [ ( int ) code ] . name = name ; ix86_builtins_isa [ ( int ) code ] . const_p = false ; ix86_builtins_isa [ ( int ) code ] . pure_p = false ; ix86_builtins_isa [ ( int ) code ] . set_and_not_built_p = true ; } } return decl ; } </s>


<s> static inline tree def_builtin_const ( HOST_WIDE_INT mask , HOST_WIDE_INT mask2 , const char * name , enum ix86_builtin_func_type tcode , enum ix86_builtins code ) { tree decl = def_builtin ( mask , mask2 , name , tcode , code ) ; if ( decl ) TREE_READONLY ( decl ) = 1 ; else ix86_builtins_isa [ ( int ) code ] . const_p = true ; return decl ; } </s>

Likewise , but also marks the function as `` const '' .


<s> static inline tree def_builtin_pure ( HOST_WIDE_INT mask , HOST_WIDE_INT mask2 , const char * name , enum ix86_builtin_func_type tcode , enum ix86_builtins code ) { tree decl = def_builtin ( mask , mask2 , name , tcode , code ) ; if ( decl ) DECL_PURE_P ( decl ) = 1 ; else ix86_builtins_isa [ ( int ) code ] . pure_p = true ; return decl ; } </s>

Like def_builtin , but also marks the function decl `` pure '' .


<s> void ix86_add_new_builtins ( HOST_WIDE_INT isa , HOST_WIDE_INT isa2 ) { isa &= ~ OPTION_MASK_ISA_64BIT ; if ( ( isa & deferred_isa_values ) == 0 && ( isa2 & deferred_isa_values2 ) == 0 && ( ( deferred_isa_values & OPTION_MASK_ISA_MMX ) == 0 || ! ( TARGET_64BIT && ( isa & OPTION_MASK_ISA_SSE2 ) != 0 ) ) ) return ; deferred_isa_values &= ~ isa ; deferred_isa_values2 &= ~ isa2 ; if ( TARGET_64BIT && ( isa & OPTION_MASK_ISA_SSE2 ) != 0 ) deferred_isa_values &= ~ OPTION_MASK_ISA_MMX ; int i ; tree saved_current_target_pragma = current_target_pragma ; current_target_pragma = NULL_TREE ; for ( i = 0 ; i < ( int ) IX86_BUILTIN_MAX ; i ++ ) { if ( ( ( ix86_builtins_isa [ i ] . isa & isa ) != 0 || ( ix86_builtins_isa [ i ] . isa2 & isa2 ) != 0 || ( ( ix86_builtins_isa [ i ] . isa & OPTION_MASK_ISA_MMX ) != 0 && TARGET_64BIT && ( isa & OPTION_MASK_ISA_SSE2 ) != 0 ) ) && ix86_builtins_isa [ i ] . set_and_not_built_p ) { tree decl , type ; ix86_builtins_isa [ i ] . set_and_not_built_p = false ; type = ix86_get_builtin_func_type ( ix86_builtins_isa [ i ] . tcode ) ; decl = add_builtin_function_ext_scope ( ix86_builtins_isa [ i ] . name , type , i , BUILT_IN_MD , NULL , NULL_TREE ) ; ix86_builtins [ i ] = decl ; if ( ix86_builtins_isa [ i ] . const_p ) TREE_READONLY ( decl ) = 1 ; } } current_target_pragma = saved_current_target_pragma ; } </s>


<s> tree ix86_builtin_decl ( unsigned code , bool ) { if ( code >= IX86_BUILTIN_MAX ) return error_mark_node ; return ix86_builtins [ code ] ; } </s>


<s> tree ix86_builtin_reciprocal ( tree fndecl ) { enum ix86_builtins fn_code = ( enum ix86_builtins ) DECL_MD_FUNCTION_CODE ( fndecl ) ; switch ( fn_code ) { case IX86_BUILTIN_SQRTPS_NR : return ix86_get_builtin ( IX86_BUILTIN_RSQRTPS_NR ) ; case IX86_BUILTIN_SQRTPS_NR256 : return ix86_get_builtin ( IX86_BUILTIN_RSQRTPS_NR256 ) ; default : return NULL_TREE ; } } </s>

Returns a code for a target-specific builtin that implements reciprocal of the function , or NULL_TREE if not available .


<s> static tree ix86_get_builtin_func_type ( enum ix86_builtin_func_type tcode ) { tree type ; gcc_assert ( ( unsigned ) tcode < ARRAY_SIZE ( ix86_builtin_func_type_tab ) ) ; type = ix86_builtin_func_type_tab [ ( int ) tcode ] ; if ( type != NULL ) return type ; if ( tcode <= IX86_BT_LAST_FUNC ) { unsigned start = ix86_builtin_func_start [ ( int ) tcode ] ; unsigned after = ix86_builtin_func_start [ ( int ) tcode + 1 ] ; tree rtype , atype , args = void_list_node ; unsigned i ; rtype = ix86_get_builtin_type ( ix86_builtin_func_args [ start ] ) ; for ( i = after - 1 ; i > start ; -- i ) { atype = ix86_get_builtin_type ( ix86_builtin_func_args [ i ] ) ; args = tree_cons ( NULL , atype , args ) ; } type = build_function_type ( rtype , args ) ; } else { unsigned index = tcode - IX86_BT_LAST_FUNC - 1 ; enum ix86_builtin_func_type icode ; icode = ix86_builtin_func_alias_base [ index ] ; type = ix86_get_builtin_func_type ( icode ) ; } ix86_builtin_func_type_tab [ ( int ) tcode ] = type ; return type ; } </s>

Retrieve an element from the above table , building some of the types lazily .


<s> static tree ix86_get_builtin_type ( enum ix86_builtin_type tcode ) { unsigned int index ; tree type , itype ; gcc_assert ( ( unsigned ) tcode < ARRAY_SIZE ( ix86_builtin_type_tab ) ) ; type = ix86_builtin_type_tab [ ( int ) tcode ] ; if ( type != NULL ) return type ; gcc_assert ( tcode > IX86_BT_LAST_PRIM ) ; if ( tcode <= IX86_BT_LAST_VECT ) { machine_mode mode ; index = tcode - IX86_BT_LAST_PRIM - 1 ; itype = ix86_get_builtin_type ( ix86_builtin_type_vect_base [ index ] ) ; mode = ix86_builtin_type_vect_mode [ index ] ; type = build_vector_type_for_mode ( itype , mode ) ; } else { int quals ; index = tcode - IX86_BT_LAST_VECT - 1 ; if ( tcode <= IX86_BT_LAST_PTR ) quals = TYPE_UNQUALIFIED ; else quals = TYPE_QUAL_CONST ; itype = ix86_get_builtin_type ( ix86_builtin_type_ptr_base [ index ] ) ; if ( quals != TYPE_UNQUALIFIED ) itype = build_qualified_type ( itype , quals ) ; type = build_pointer_type ( itype ) ; } ix86_builtin_type_tab [ ( int ) tcode ] = type ; return type ; } </s>


Internal method for ix86_init_builtins .


<s> static void ix86_init_platform_type_builtins ( void ) { make_cpu_type_builtin ( "__builtin_cpu_init" , IX86_BUILTIN_CPU_INIT , INT_FTYPE_VOID , false ) ; make_cpu_type_builtin ( "__builtin_cpu_is" , IX86_BUILTIN_CPU_IS , INT_FTYPE_PCCHAR , true ) ; make_cpu_type_builtin ( "__builtin_cpu_supports" , IX86_BUILTIN_CPU_SUPPORTS , INT_FTYPE_PCCHAR , true ) ; } </s>

Make builtins to get CPU type and features supported . The created builtins are : __builtin_cpu_init ( ) , to detect cpu type and features , __builtin_cpu_is ( `` < CPUNAME > '' ) , to check if cpu is of type < CPUNAME > , __builtin_cpu_supports ( `` < FEATURE > '' ) , to check if cpu supports < FEATURE >


<s> static void ix86_init_tm_builtins ( void ) { enum ix86_builtin_func_type ftype ; const struct builtin_description * d ; size_t i ; tree decl ; tree attrs_load , attrs_type_load , attrs_store , attrs_type_store ; tree attrs_log , attrs_type_log ; if ( ! flag_tm ) return ; if ( ! builtin_decl_explicit_p ( BUILT_IN_TM_LOAD_1 ) ) return ; decl = builtin_decl_explicit ( BUILT_IN_TM_LOAD_1 ) ; attrs_load = DECL_ATTRIBUTES ( decl ) ; attrs_type_load = TYPE_ATTRIBUTES ( TREE_TYPE ( decl ) ) ; decl = builtin_decl_explicit ( BUILT_IN_TM_STORE_1 ) ; attrs_store = DECL_ATTRIBUTES ( decl ) ; attrs_type_store = TYPE_ATTRIBUTES ( TREE_TYPE ( decl ) ) ; decl = builtin_decl_explicit ( BUILT_IN_TM_LOG ) ; attrs_log = DECL_ATTRIBUTES ( decl ) ; attrs_type_log = TYPE_ATTRIBUTES ( TREE_TYPE ( decl ) ) ; for ( i = 0 , d = bdesc_tm ; i < ARRAY_SIZE ( bdesc_tm ) ; i ++ , d ++ ) { if ( ( d -> mask & ix86_isa_flags ) != 0 || ( ( d -> mask & OPTION_MASK_ISA_MMX ) != 0 && TARGET_MMX_WITH_SSE ) || ( lang_hooks . builtin_function == lang_hooks . builtin_function_ext_scope ) ) { tree type , attrs , attrs_type ; enum built_in_function code = ( enum built_in_function ) d -> code ; ftype = ( enum ix86_builtin_func_type ) d -> flag ; type = ix86_get_builtin_func_type ( ftype ) ; if ( BUILTIN_TM_LOAD_P ( code ) ) { attrs = attrs_load ; attrs_type = attrs_type_load ; } else if ( BUILTIN_TM_STORE_P ( code ) ) { attrs = attrs_store ; attrs_type = attrs_type_store ; } else { attrs = attrs_log ; attrs_type = attrs_type_log ; } decl = add_builtin_function ( d -> name , type , code , BUILT_IN_NORMAL , d -> name + strlen ( "__builtin_" ) , attrs ) ; decl_attributes ( & TREE_TYPE ( decl ) , attrs_type , ATTR_FLAG_BUILT_IN ) ; set_builtin_decl ( code , decl , false ) ; } } } </s>

Initialize the transactional memory vector load/store builtins .


<s> tree ix86_vectorize_builtin_gather ( const_tree mem_vectype , const_tree index_type , int scale ) { bool si ; enum ix86_builtins code ; if ( ! TARGET_AVX2 || ( known_eq ( TYPE_VECTOR_SUBPARTS ( mem_vectype ) , 2u ) ? ! TARGET_USE_GATHER_2PARTS : ( known_eq ( TYPE_VECTOR_SUBPARTS ( mem_vectype ) , 4u ) ? ! TARGET_USE_GATHER_4PARTS : ! TARGET_USE_GATHER ) ) ) return NULL_TREE ; if ( ( TREE_CODE ( index_type ) != INTEGER_TYPE && ! POINTER_TYPE_P ( index_type ) ) || ( TYPE_MODE ( index_type ) != SImode && TYPE_MODE ( index_type ) != DImode ) ) return NULL_TREE ; if ( TYPE_PRECISION ( index_type ) > POINTER_SIZE ) return NULL_TREE ; if ( TYPE_PRECISION ( index_type ) < POINTER_SIZE && TYPE_UNSIGNED ( index_type ) ) return NULL_TREE ; if ( scale <= 0 || scale > 8 || ( scale & ( scale - 1 ) ) != 0 ) return NULL_TREE ; si = TYPE_MODE ( index_type ) == SImode ; switch ( TYPE_MODE ( mem_vectype ) ) { case E_V2DFmode : if ( TARGET_AVX512VL ) code = si ? IX86_BUILTIN_GATHER3SIV2DF : IX86_BUILTIN_GATHER3DIV2DF ; else code = si ? IX86_BUILTIN_GATHERSIV2DF : IX86_BUILTIN_GATHERDIV2DF ; break ; case E_V4DFmode : if ( TARGET_AVX512VL ) code = si ? IX86_BUILTIN_GATHER3ALTSIV4DF : IX86_BUILTIN_GATHER3DIV4DF ; else code = si ? IX86_BUILTIN_GATHERALTSIV4DF : IX86_BUILTIN_GATHERDIV4DF ; break ; case E_V2DImode : if ( TARGET_AVX512VL ) code = si ? IX86_BUILTIN_GATHER3SIV2DI : IX86_BUILTIN_GATHER3DIV2DI ; else code = si ? IX86_BUILTIN_GATHERSIV2DI : IX86_BUILTIN_GATHERDIV2DI ; break ; case E_V4DImode : if ( TARGET_AVX512VL ) code = si ? IX86_BUILTIN_GATHER3ALTSIV4DI : IX86_BUILTIN_GATHER3DIV4DI ; else code = si ? IX86_BUILTIN_GATHERALTSIV4DI : IX86_BUILTIN_GATHERDIV4DI ; break ; case E_V4SFmode : if ( TARGET_AVX512VL ) code = si ? IX86_BUILTIN_GATHER3SIV4SF : IX86_BUILTIN_GATHER3DIV4SF ; else code = si ? IX86_BUILTIN_GATHERSIV4SF : IX86_BUILTIN_GATHERDIV4SF ; break ; case E_V8SFmode : if ( TARGET_AVX512VL ) code = si ? IX86_BUILTIN_GATHER3SIV8SF : IX86_BUILTIN_GATHER3ALTDIV8SF ; else code = si ? IX86_BUILTIN_GATHERSIV8SF : IX86_BUILTIN_GATHERALTDIV8SF ; break ; case E_V4SImode : if ( TARGET_AVX512VL ) code = si ? IX86_BUILTIN_GATHER3SIV4SI : IX86_BUILTIN_GATHER3DIV4SI ; else code = si ? IX86_BUILTIN_GATHERSIV4SI : IX86_BUILTIN_GATHERDIV4SI ; break ; case E_V8SImode : if ( TARGET_AVX512VL ) code = si ? IX86_BUILTIN_GATHER3SIV8SI : IX86_BUILTIN_GATHER3ALTDIV8SI ; else code = si ? IX86_BUILTIN_GATHERSIV8SI : IX86_BUILTIN_GATHERALTDIV8SI ; break ; case E_V8DFmode : if ( TARGET_AVX512F ) code = si ? IX86_BUILTIN_GATHER3ALTSIV8DF : IX86_BUILTIN_GATHER3DIV8DF ; else return NULL_TREE ; break ; case E_V8DImode : if ( TARGET_AVX512F ) code = si ? IX86_BUILTIN_GATHER3ALTSIV8DI : IX86_BUILTIN_GATHER3DIV8DI ; else return NULL_TREE ; break ; case E_V16SFmode : if ( TARGET_AVX512F ) code = si ? IX86_BUILTIN_GATHER3SIV16SF : IX86_BUILTIN_GATHER3ALTDIV16SF ; else return NULL_TREE ; break ; case E_V16SImode : if ( TARGET_AVX512F ) code = si ? IX86_BUILTIN_GATHER3SIV16SI : IX86_BUILTIN_GATHER3ALTDIV16SI ; else return NULL_TREE ; break ; default : return NULL_TREE ; } return ix86_get_builtin ( code ) ; } </s>

Returns a decl of a function that implements gather load with memory type MEM_VECTYPE and index type INDEX_VECTYPE and SCALE . Return NULL_TREE if it is not available .


<s> static void make_cpu_type_builtin ( const char * name , int code , enum ix86_builtin_func_type ftype , bool is_const ) { tree decl ; tree type ; type = ix86_get_builtin_func_type ( ftype ) ; decl = add_builtin_function ( name , type , code , BUILT_IN_MD , NULL , NULL_TREE ) ; gcc_assert ( decl != NULL_TREE ) ; ix86_builtins [ ( int ) code ] = decl ; TREE_READONLY ( decl ) = is_const ; } </s>

Make builtins to detect cpu type and features supported . NAME is the builtin name , CODE is the builtin code , and FTYPE is the function type of the builtin .


Returns a extern , comdat VAR_DECL of type TYPE and name NAME .


Hook to validate the current # pragma GCC target and set the state , and update the macros based on what was changed . If ARGS is NULL , then POP_TARGET is used to reset the options .


<s> void ix86_register_pragmas ( void ) { targetm . target_option . pragma_parse = ix86_pragma_target_parse ; REGISTER_SUBTARGET_PRAGMAS ( ) ; } </s>

Register target pragmas . We need to add the hook for parsing # pragma GCC option here rather than in i386.cc since it will pull in various preprocessor functions , and those are not present in languages like fortran without a preprocessor .


Function to tell the preprocessor about the defines for the current target .


<s> void ix86_register_pragmas ( void ) { targetm . target_option . pragma_parse = ix86_pragma_target_parse ; c_register_addr_space ( "__seg_fs" , ADDR_SPACE_SEG_FS ) ; c_register_addr_space ( "__seg_gs" , ADDR_SPACE_SEG_GS ) ; REGISTER_SUBTARGET_PRAGMAS ( ) ; } </s>


<s> static tree ix86_d_handle_target_float_abi ( void ) { const char * abi ; if ( ! ( TARGET_80387 || TARGET_FLOAT_RETURNS_IN_80387 ) ) abi = "soft" ; else abi = "hard" ; return build_string_literal ( strlen ( abi ) + 1 , abi ) ; } </s>

Handle a call to ` __traits ( getTargetInfo , `` floatAbi '' ) ' .


<s> static tree ix86_d_handle_target_object_format ( void ) { const char * objfmt = NULL ; if ( TARGET_MACHO ) objfmt = "macho" ; else if ( TARGET_COFF || TARGET_PECOFF ) objfmt = "coff" ; if ( objfmt == NULL ) return NULL_TREE ; return build_string_literal ( strlen ( objfmt ) + 1 , objfmt ) ; } </s>

Handle a call to ` __traits ( getTargetInfo , `` objectFormat '' ) ' .


<s> bool ix86_d_has_stdcall_convention ( unsigned int * link_system , unsigned int * link_windows ) { if ( ix86_abi == MS_ABI ) { * link_system = 1 ; * link_windows = ( ! TARGET_64BIT ) ? 1 : 0 ; } else { * link_system = 0 ; * link_windows = 0 ; } return true ; } </s>

Implement TARGET_D_HAS_STDCALL_CONVENTION for x86 targets .


<s> void ix86_d_register_target_info ( void ) { const struct d_target_info_spec handlers [ ] = { { "floatAbi" , ix86_d_handle_target_float_abi } , { "objectFormat" , ix86_d_handle_target_object_format } , { NULL , NULL } , } ; d_add_target_info_handlers ( handlers ) ; } </s>


<s> void ix86_d_target_versions ( void ) { if ( TARGET_64BIT ) { d_add_builtin_version ( "X86_64" ) ; if ( TARGET_X32 ) d_add_builtin_version ( "D_X32" ) ; } else d_add_builtin_version ( "X86" ) ; if ( TARGET_80387 ) d_add_builtin_version ( "D_HardFloat" ) ; else d_add_builtin_version ( "D_SoftFloat" ) ; } </s>

Implement TARGET_D_CPU_VERSIONS for x86 targets .


<s> static bool expand_vec_perm_movs ( struct expand_vec_perm_d * d ) { machine_mode vmode = d -> vmode ; unsigned i , nelt = d -> nelt ; rtx x ; if ( d -> one_operand_p ) return false ; if ( ! ( TARGET_SSE && ( vmode == V4SFmode || vmode == V4SImode ) ) && ! ( TARGET_MMX_WITH_SSE && ( vmode == V2SFmode || vmode == V2SImode ) ) && ! ( TARGET_SSE2 && ( vmode == V2DFmode || vmode == V2DImode ) ) ) return false ; if ( d -> perm [ 0 ] != nelt && d -> perm [ 0 ] != 0 ) return false ; for ( i = 1 ; i < nelt ; ++ i ) if ( d -> perm [ i ] != i + nelt - d -> perm [ 0 ] ) return false ; if ( d -> testing_p ) return true ; if ( d -> perm [ 0 ] == nelt ) x = gen_rtx_VEC_MERGE ( vmode , d -> op1 , d -> op0 , GEN_INT ( 1 ) ) ; else x = gen_rtx_VEC_MERGE ( vmode , d -> op0 , d -> op1 , GEN_INT ( 1 ) ) ; emit_insn ( gen_rtx_SET ( d -> target , x ) ) ; return true ; } </s>

A subroutine of ix86_expand_vec_perm_builtin_1 . Try to implement D using movss or movsd .


<s> static bool expand_vec_perm_pblendv ( struct expand_vec_perm_d * d ) { unsigned i , which , nelt = d -> nelt ; struct expand_vec_perm_d dcopy , dcopy1 ; machine_mode vmode = d -> vmode ; bool ok ; if ( d -> one_operand_p ) return false ; if ( TARGET_AVX2 && GET_MODE_SIZE ( vmode ) == 32 ) ; else if ( TARGET_AVX && ( vmode == V4DFmode || vmode == V8SFmode ) ) ; else if ( TARGET_SSE4_1 && ( GET_MODE_SIZE ( vmode ) == 16 || ( TARGET_MMX_WITH_SSE && GET_MODE_SIZE ( vmode ) == 8 ) || GET_MODE_SIZE ( vmode ) == 4 ) ) ; else return false ; for ( i = 0 , which = 0 ; i < nelt ; ++ i ) { unsigned e = d -> perm [ i ] ; if ( e != i ) which |= ( e < nelt ? 1 : 2 ) ; } if ( which != 1 && which != 2 ) return false ; if ( d -> testing_p && GET_MODE_SIZE ( vmode ) == 16 ) return true ; dcopy = * d ; if ( which == 2 ) dcopy . op0 = dcopy . op1 = d -> op1 ; else dcopy . op0 = dcopy . op1 = d -> op0 ; if ( ! d -> testing_p ) dcopy . target = gen_reg_rtx ( vmode ) ; dcopy . one_operand_p = true ; for ( i = 0 ; i < nelt ; ++ i ) dcopy . perm [ i ] = d -> perm [ i ] & ( nelt - 1 ) ; ok = expand_vec_perm_1 ( & dcopy ) ; if ( GET_MODE_SIZE ( vmode ) != 16 && ! ok ) return false ; else gcc_assert ( ok ) ; if ( d -> testing_p ) return true ; dcopy1 = * d ; if ( which == 2 ) dcopy1 . op1 = dcopy . target ; else dcopy1 . op0 = dcopy . target ; for ( i = 0 ; i < nelt ; ++ i ) dcopy1 . perm [ i ] = ( ( d -> perm [ i ] >= nelt ) ? ( nelt + i ) : i ) ; ok = expand_vec_perm_blend ( & dcopy1 ) ; gcc_assert ( ok ) ; return true ; } </s>

A subroutine of ix86_expand_vec_perm_const_1 . Try to simplify the permutation using the SSE4_1 pblendv instruction . Potentially reduces permutation from 2 pshufb and or to 1 pshufb and pblendv .


<s> static machine_mode get_mode_wider_vector ( machine_mode o ) { machine_mode n = GET_MODE_NEXT_MODE ( o ) . require ( ) ; gcc_assert ( GET_MODE_NUNITS ( o ) == GET_MODE_NUNITS ( n ) * 2 ) ; gcc_assert ( GET_MODE_SIZE ( o ) == GET_MODE_SIZE ( n ) ) ; return n ; } </s>

Get a vector mode of the same size as the original but with elements twice as wide . This is only guaranteed to apply to integral vectors .


<s> static void ix86_avx256_split_vector_move_misalign ( rtx op0 , rtx op1 ) { rtx m ; rtx ( * extract ) ( rtx , rtx , rtx ) ; machine_mode mode ; if ( ( MEM_P ( op1 ) && ! TARGET_AVX256_SPLIT_UNALIGNED_LOAD ) || ( MEM_P ( op0 ) && ! TARGET_AVX256_SPLIT_UNALIGNED_STORE ) ) { emit_insn ( gen_rtx_SET ( op0 , op1 ) ) ; return ; } rtx orig_op0 = NULL_RTX ; mode = GET_MODE ( op0 ) ; switch ( GET_MODE_CLASS ( mode ) ) { case MODE_VECTOR_INT : case MODE_INT : if ( mode != V32QImode ) { if ( ! MEM_P ( op0 ) ) { orig_op0 = op0 ; op0 = gen_reg_rtx ( V32QImode ) ; } else op0 = gen_lowpart ( V32QImode , op0 ) ; op1 = gen_lowpart ( V32QImode , op1 ) ; mode = V32QImode ; } break ; case MODE_VECTOR_FLOAT : break ; default : gcc_unreachable ( ) ; } switch ( mode ) { default : gcc_unreachable ( ) ; case E_V32QImode : extract = gen_avx_vextractf128v32qi ; mode = V16QImode ; break ; case E_V16BFmode : extract = gen_avx_vextractf128v16bf ; mode = V8BFmode ; break ; case E_V16HFmode : extract = gen_avx_vextractf128v16hf ; mode = V8HFmode ; break ; case E_V8SFmode : extract = gen_avx_vextractf128v8sf ; mode = V4SFmode ; break ; case E_V4DFmode : extract = gen_avx_vextractf128v4df ; mode = V2DFmode ; break ; } if ( MEM_P ( op1 ) ) { rtx r = gen_reg_rtx ( mode ) ; m = adjust_address ( op1 , mode , 0 ) ; emit_move_insn ( r , m ) ; m = adjust_address ( op1 , mode , 16 ) ; r = gen_rtx_VEC_CONCAT ( GET_MODE ( op0 ) , r , m ) ; emit_move_insn ( op0 , r ) ; } else if ( MEM_P ( op0 ) ) { m = adjust_address ( op0 , mode , 0 ) ; emit_insn ( extract ( m , op1 , const0_rtx ) ) ; m = adjust_address ( op0 , mode , 16 ) ; emit_insn ( extract ( m , copy_rtx ( op1 ) , const1_rtx ) ) ; } else gcc_unreachable ( ) ; if ( orig_op0 ) emit_move_insn ( orig_op0 , gen_lowpart ( GET_MODE ( orig_op0 ) , op0 ) ) ; } </s>

Split 32-byte AVX unaligned load and store if needed .


<s> bool ix86_check_builtin_isa_match ( unsigned int fcode , HOST_WIDE_INT * pbisa , HOST_WIDE_INT * pbisa2 ) { HOST_WIDE_INT isa = ix86_isa_flags ; HOST_WIDE_INT isa2 = ix86_isa_flags2 ; HOST_WIDE_INT bisa = ix86_builtins_isa [ fcode ] . isa ; HOST_WIDE_INT bisa2 = ix86_builtins_isa [ fcode ] . isa2 ; if ( ( ( bisa & ( OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A ) ) == ( OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A ) ) && ( isa & ( OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A ) ) != 0 ) isa |= ( OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A ) ; if ( ( ( bisa & ( OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32 ) ) == ( OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32 ) ) && ( isa & ( OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32 ) ) != 0 ) isa |= ( OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32 ) ; if ( ( ( bisa & ( OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4 ) ) == ( OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4 ) ) && ( isa & ( OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4 ) ) != 0 ) isa |= ( OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4 ) ; if ( ( ( ( bisa & ( OPTION_MASK_ISA_AVX512VNNI | OPTION_MASK_ISA_AVX512VL ) ) == ( OPTION_MASK_ISA_AVX512VNNI | OPTION_MASK_ISA_AVX512VL ) ) || ( bisa2 & OPTION_MASK_ISA2_AVXVNNI ) != 0 ) && ( ( ( isa & ( OPTION_MASK_ISA_AVX512VNNI | OPTION_MASK_ISA_AVX512VL ) ) == ( OPTION_MASK_ISA_AVX512VNNI | OPTION_MASK_ISA_AVX512VL ) ) || ( isa2 & OPTION_MASK_ISA2_AVXVNNI ) != 0 ) ) { isa |= OPTION_MASK_ISA_AVX512VNNI | OPTION_MASK_ISA_AVX512VL ; isa2 |= OPTION_MASK_ISA2_AVXVNNI ; } if ( ( ( ( bisa & ( OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL ) ) == ( OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL ) ) || ( bisa2 & OPTION_MASK_ISA2_AVXIFMA ) != 0 ) && ( ( ( isa & ( OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL ) ) == ( OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL ) ) || ( isa2 & OPTION_MASK_ISA2_AVXIFMA ) != 0 ) ) { isa |= OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL ; isa2 |= OPTION_MASK_ISA2_AVXIFMA ; } if ( ( ( ( bisa & OPTION_MASK_ISA_AVX512VL ) != 0 && ( bisa2 & OPTION_MASK_ISA2_AVX512BF16 ) != 0 ) && ( bisa2 & OPTION_MASK_ISA2_AVXNECONVERT ) != 0 ) && ( ( ( isa & OPTION_MASK_ISA_AVX512VL ) != 0 && ( isa2 & OPTION_MASK_ISA2_AVX512BF16 ) != 0 ) || ( isa2 & OPTION_MASK_ISA2_AVXNECONVERT ) != 0 ) ) { isa |= OPTION_MASK_ISA_AVX512VL ; isa2 |= OPTION_MASK_ISA2_AVXNECONVERT | OPTION_MASK_ISA2_AVX512BF16 ; } if ( ( bisa & OPTION_MASK_ISA_MMX ) && ! TARGET_MMX && TARGET_MMX_WITH_SSE && fcode != IX86_BUILTIN_MASKMOVQ ) { bisa &= ~ OPTION_MASK_ISA_MMX ; bisa |= OPTION_MASK_ISA_SSE2 ; } if ( pbisa ) * pbisa = bisa ; if ( pbisa2 ) * pbisa2 = bisa2 ; return ( bisa & isa ) == bisa && ( bisa2 & isa2 ) == bisa2 ; } </s>

Return true if the necessary isa options for this builtin exist , else false . fcode = DECL_MD_FUNCTION_CODE ( fndecl ) ;


Convert the CONST_WIDE_INT operand OP to broadcast in MODE .


<s> rtx ix86_expand_fast_convert_bf_to_sf ( rtx val ) { rtx op = gen_lowpart ( HImode , val ) , ret ; if ( CONST_INT_P ( op ) ) { ret = simplify_const_unary_operation ( FLOAT_EXTEND , SFmode , val , BFmode ) ; if ( ret ) return ret ; ret = gen_reg_rtx ( SImode ) ; emit_move_insn ( ret , GEN_INT ( INTVAL ( op ) & 0xffff ) ) ; emit_insn ( gen_ashlsi3 ( ret , ret , GEN_INT ( 16 ) ) ) ; return gen_lowpart ( SFmode , ret ) ; } ret = gen_reg_rtx ( SFmode ) ; emit_insn ( gen_extendbfsf2_1 ( ret , force_reg ( BFmode , val ) ) ) ; return ret ; } </s>

Convert a BFmode VAL to SFmode without signaling sNaNs . This is done by returning SF SUBREG of ( ( HI SUBREG ) ( VAL ) ) < < 16 .


<s> void ix86_expand_fp_absneg_operator ( enum rtx_code code , machine_mode mode , rtx operands [ ] ) { rtx set , dst , src ; bool use_sse = false ; bool vector_mode = VECTOR_MODE_P ( mode ) ; machine_mode vmode = mode ; rtvec par ; if ( vector_mode || mode == TFmode ) use_sse = true ; else if ( TARGET_SSE_MATH ) { use_sse = SSE_FLOAT_MODE_P ( mode ) ; if ( mode == SFmode ) vmode = V4SFmode ; else if ( mode == DFmode ) vmode = V2DFmode ; } dst = operands [ 0 ] ; src = operands [ 1 ] ; set = gen_rtx_fmt_e ( code , mode , src ) ; set = gen_rtx_SET ( dst , set ) ; if ( use_sse ) { rtx mask , use , clob ; mask = ix86_build_signbit_mask ( vmode , vector_mode , code == ABS ) ; use = gen_rtx_USE ( VOIDmode , mask ) ; if ( vector_mode || mode == TFmode ) par = gen_rtvec ( 2 , set , use ) ; else { clob = gen_rtx_CLOBBER ( VOIDmode , gen_rtx_REG ( CCmode , FLAGS_REG ) ) ; par = gen_rtvec ( 3 , set , use , clob ) ; } } else { rtx clob ; clob = gen_rtx_CLOBBER ( VOIDmode , gen_rtx_REG ( CCmode , FLAGS_REG ) ) ; par = gen_rtvec ( 2 , set , clob ) ; } emit_insn ( gen_rtx_PARALLEL ( VOIDmode , par ) ) ; } </s>

Generate code for floating point ABS or NEG .


<s> bool ix86_expand_fp_movcc ( rtx operands [ ] ) { machine_mode mode = GET_MODE ( operands [ 0 ] ) ; enum rtx_code code = GET_CODE ( operands [ 1 ] ) ; rtx tmp , compare_op ; rtx op0 = XEXP ( operands [ 1 ] , 0 ) ; rtx op1 = XEXP ( operands [ 1 ] , 1 ) ; if ( GET_MODE ( op0 ) == BFmode && ! ix86_fp_comparison_operator ( operands [ 1 ] , VOIDmode ) ) return false ; if ( SSE_FLOAT_MODE_SSEMATH_OR_HF_P ( mode ) ) { machine_mode cmode ; cmode = GET_MODE ( op0 ) ; if ( cmode == VOIDmode ) cmode = GET_MODE ( op1 ) ; if ( cmode != mode ) return false ; code = ix86_prepare_sse_fp_compare_args ( operands [ 0 ] , code , & op0 , & op1 ) ; if ( code == UNKNOWN ) return false ; if ( ix86_expand_sse_fp_minmax ( operands [ 0 ] , code , op0 , op1 , operands [ 2 ] , operands [ 3 ] ) ) return true ; tmp = ix86_expand_sse_cmp ( operands [ 0 ] , code , op0 , op1 , operands [ 2 ] , operands [ 3 ] ) ; ix86_expand_sse_movcc ( operands [ 0 ] , tmp , operands [ 2 ] , operands [ 3 ] ) ; return true ; } if ( GET_MODE ( op0 ) == TImode || ( GET_MODE ( op0 ) == DImode && ! TARGET_64BIT ) ) return false ; compare_op = ix86_expand_compare ( code , op0 , op1 ) ; if ( ! fcmov_comparison_operator ( compare_op , VOIDmode ) ) { tmp = gen_reg_rtx ( QImode ) ; ix86_expand_setcc ( tmp , code , op0 , op1 ) ; compare_op = ix86_expand_compare ( NE , tmp , const0_rtx ) ; } emit_insn ( gen_rtx_SET ( operands [ 0 ] , gen_rtx_IF_THEN_ELSE ( mode , compare_op , operands [ 2 ] , operands [ 3 ] ) ) ) ; return true ; } </s>

Expand a floating-point conditional move . Return true if successful .


<s> static rtx ix86_expand_sse_cmp ( rtx dest , enum rtx_code code , rtx cmp_op0 , rtx cmp_op1 , rtx op_true , rtx op_false ) { machine_mode mode = GET_MODE ( dest ) ; machine_mode cmp_ops_mode = GET_MODE ( cmp_op0 ) ; machine_mode cmp_mode ; bool maskcmp = false ; rtx x ; if ( ix86_use_mask_cmp_p ( mode , cmp_ops_mode , op_true , op_false ) ) { unsigned int nbits = GET_MODE_NUNITS ( cmp_ops_mode ) ; maskcmp = true ; cmp_mode = nbits > 8 ? int_mode_for_size ( nbits , 0 ) . require ( ) : E_QImode ; } else cmp_mode = cmp_ops_mode ; cmp_op0 = force_reg ( cmp_ops_mode , cmp_op0 ) ; int ( * op1_predicate ) ( rtx , machine_mode ) = VECTOR_MODE_P ( cmp_ops_mode ) ? vector_operand : nonimmediate_operand ; if ( ! op1_predicate ( cmp_op1 , cmp_ops_mode ) ) cmp_op1 = force_reg ( cmp_ops_mode , cmp_op1 ) ; if ( optimize || ( maskcmp && cmp_mode != mode ) || ( op_true && reg_overlap_mentioned_p ( dest , op_true ) ) || ( op_false && reg_overlap_mentioned_p ( dest , op_false ) ) ) dest = gen_reg_rtx ( maskcmp ? cmp_mode : mode ) ; if ( maskcmp ) { bool ok = ix86_expand_mask_vec_cmp ( dest , code , cmp_op0 , cmp_op1 ) ; gcc_assert ( ok ) ; return dest ; } x = gen_rtx_fmt_ee ( code , cmp_mode , cmp_op0 , cmp_op1 ) ; if ( cmp_mode != mode ) { x = force_reg ( cmp_ops_mode , x ) ; convert_move ( dest , x , false ) ; } else emit_insn ( gen_rtx_SET ( dest , x ) ) ; return dest ; } </s>

Expand an sse vector comparison . Return the register with the result .


<s> static rtx ix86_expand_sse_comi ( const struct builtin_description * d , tree exp , rtx target ) { rtx pat , set_dst ; tree arg0 = CALL_EXPR_ARG ( exp , 0 ) ; tree arg1 = CALL_EXPR_ARG ( exp , 1 ) ; rtx op0 = expand_normal ( arg0 ) ; rtx op1 = expand_normal ( arg1 ) ; enum insn_code icode = d -> icode ; const struct insn_data_d * insn_p = & insn_data [ icode ] ; machine_mode mode0 = insn_p -> operand [ 0 ] . mode ; machine_mode mode1 = insn_p -> operand [ 1 ] . mode ; if ( VECTOR_MODE_P ( mode0 ) ) op0 = safe_vector_operand ( op0 , mode0 ) ; if ( VECTOR_MODE_P ( mode1 ) ) op1 = safe_vector_operand ( op1 , mode1 ) ; enum rtx_code comparison = d -> comparison ; rtx const_val = const0_rtx ; bool check_unordered = false ; machine_mode mode = CCFPmode ; switch ( comparison ) { case LE : case LT : std :: swap ( op0 , op1 ) ; comparison = swap_condition ( comparison ) ; case GT : case GE : break ; case EQ : check_unordered = true ; mode = CCZmode ; break ; case NE : check_unordered = true ; mode = CCZmode ; const_val = const1_rtx ; break ; default : gcc_unreachable ( ) ; } target = gen_reg_rtx ( SImode ) ; emit_move_insn ( target , const_val ) ; target = gen_rtx_SUBREG ( QImode , target , 0 ) ; if ( ( optimize && ! register_operand ( op0 , mode0 ) ) || ! insn_p -> operand [ 0 ] . predicate ( op0 , mode0 ) ) op0 = copy_to_mode_reg ( mode0 , op0 ) ; if ( ( optimize && ! register_operand ( op1 , mode1 ) ) || ! insn_p -> operand [ 1 ] . predicate ( op1 , mode1 ) ) op1 = copy_to_mode_reg ( mode1 , op1 ) ; pat = GEN_FCN ( icode ) ( op0 , op1 ) ; if ( ! pat ) return 0 ; set_dst = SET_DEST ( pat ) ; emit_insn ( pat ) ; return ix86_ssecom_setcc ( comparison , check_unordered , mode , set_dst , target ) ; } </s>

Subroutine of ix86_expand_builtin to take care of comi insns .


<s> bool ix86_expand_vecmul_qihi ( rtx dest , rtx op1 , rtx op2 ) { machine_mode himode , qimode = GET_MODE ( dest ) ; rtx hop1 , hop2 , hdest ; rtx ( * gen_extend ) ( rtx , rtx ) ; rtx ( * gen_truncate ) ( rtx , rtx ) ; if ( qimode == E_V64QImode ) return false ; if ( ! TARGET_AVX512BW ) return false ; if ( ( qimode == V8QImode || qimode == V16QImode ) && ! TARGET_AVX512VL ) return false ; if ( qimode == V32QImode && ( TARGET_PREFER_AVX128 || TARGET_PREFER_AVX256 ) ) return false ; switch ( qimode ) { case E_V8QImode : himode = V8HImode ; gen_extend = gen_zero_extendv8qiv8hi2 ; gen_truncate = gen_truncv8hiv8qi2 ; break ; case E_V16QImode : himode = V16HImode ; gen_extend = gen_zero_extendv16qiv16hi2 ; gen_truncate = gen_truncv16hiv16qi2 ; break ; case E_V32QImode : himode = V32HImode ; gen_extend = gen_zero_extendv32qiv32hi2 ; gen_truncate = gen_truncv32hiv32qi2 ; break ; default : gcc_unreachable ( ) ; } hop1 = gen_reg_rtx ( himode ) ; hop2 = gen_reg_rtx ( himode ) ; hdest = gen_reg_rtx ( himode ) ; emit_insn ( gen_extend ( hop1 , op1 ) ) ; emit_insn ( gen_extend ( hop2 , op2 ) ) ; emit_insn ( gen_rtx_SET ( hdest , simplify_gen_binary ( MULT , himode , hop1 , hop2 ) ) ) ; emit_insn ( gen_truncate ( dest , hdest ) ) ; return true ; } </s>

Optimize vector MUL generation for V8QI , V16QI and V32QI under TARGET_AVX512BW . i.e . for v16qi a * b , it has vpmovzxbw ymm2 , xmm0 vpmovzxbw ymm3 , xmm1 vpmullw ymm4 , ymm2 , ymm3 vpmovwb xmm0 , ymm4 it would take less instructions than ix86_expand_vecop_qihi . Return true if success .


<s> void ix86_expand_vector_init ( bool mmx_ok , rtx target , rtx vals ) { machine_mode mode = GET_MODE ( target ) ; machine_mode inner_mode = GET_MODE_INNER ( mode ) ; int n_elts = GET_MODE_NUNITS ( mode ) ; int n_var = 0 , one_var = - 1 ; bool all_same = true , all_const_zero = true ; int i ; rtx x ; if ( n_elts != XVECLEN ( vals , 0 ) ) { rtx subtarget = target ; x = XVECEXP ( vals , 0 , 0 ) ; gcc_assert ( GET_MODE_INNER ( GET_MODE ( x ) ) == inner_mode ) ; if ( GET_MODE_NUNITS ( GET_MODE ( x ) ) * 2 == n_elts ) { rtx ops [ 2 ] = { XVECEXP ( vals , 0 , 0 ) , XVECEXP ( vals , 0 , 1 ) } ; if ( inner_mode == QImode || inner_mode == HImode || inner_mode == TImode || inner_mode == HFmode || inner_mode == BFmode ) { unsigned int n_bits = n_elts * GET_MODE_SIZE ( inner_mode ) ; scalar_mode elt_mode = inner_mode == TImode ? DImode : SImode ; n_bits /= GET_MODE_SIZE ( elt_mode ) ; mode = mode_for_vector ( elt_mode , n_bits ) . require ( ) ; inner_mode = mode_for_vector ( elt_mode , n_bits / 2 ) . require ( ) ; ops [ 0 ] = gen_lowpart ( inner_mode , ops [ 0 ] ) ; ops [ 1 ] = gen_lowpart ( inner_mode , ops [ 1 ] ) ; subtarget = gen_reg_rtx ( mode ) ; } ix86_expand_vector_init_concat ( mode , subtarget , ops , 2 ) ; if ( subtarget != target ) emit_move_insn ( target , gen_lowpart ( GET_MODE ( target ) , subtarget ) ) ; return ; } gcc_unreachable ( ) ; } for ( i = 0 ; i < n_elts ; ++ i ) { x = XVECEXP ( vals , 0 , i ) ; if ( ! ( CONST_SCALAR_INT_P ( x ) || CONST_DOUBLE_P ( x ) || CONST_FIXED_P ( x ) ) ) n_var ++ , one_var = i ; else if ( x != CONST0_RTX ( inner_mode ) ) all_const_zero = false ; if ( i > 0 && ! rtx_equal_p ( x , XVECEXP ( vals , 0 , 0 ) ) ) all_same = false ; } if ( n_var == 0 ) { emit_move_insn ( target , gen_rtx_CONST_VECTOR ( mode , XVEC ( vals , 0 ) ) ) ; return ; } if ( all_same && ix86_expand_vector_init_duplicate ( mmx_ok , mode , target , XVECEXP ( vals , 0 , 0 ) ) ) return ; if ( n_var == 1 ) { if ( all_const_zero && ix86_expand_vector_init_one_nonzero ( mmx_ok , mode , target , XVECEXP ( vals , 0 , one_var ) , one_var ) ) return ; if ( ix86_expand_vector_init_one_var ( mmx_ok , mode , target , vals , one_var ) ) return ; } ix86_expand_vector_init_general ( mmx_ok , mode , target , vals ) ; } </s>

Initialize vector TARGET via VALS . Suppress the use of MMX instructions unless MMX_OK is true .


<s> static bool ix86_expand_vector_init_one_var ( bool mmx_ok , machine_mode mode , rtx target , rtx vals , int one_var ) { rtx var = XVECEXP ( vals , 0 , one_var ) ; machine_mode wmode ; rtx const_vec , x ; const_vec = copy_rtx ( vals ) ; XVECEXP ( const_vec , 0 , one_var ) = CONST0_RTX ( GET_MODE_INNER ( mode ) ) ; const_vec = gen_rtx_CONST_VECTOR ( mode , XVEC ( const_vec , 0 ) ) ; switch ( mode ) { case E_V2DFmode : case E_V2DImode : case E_V2SFmode : case E_V2SImode : return false ; case E_V4DImode : if ( ! TARGET_64BIT ) return false ; case E_V8HFmode : case E_V16HFmode : case E_V8BFmode : case E_V16BFmode : case E_V4DFmode : case E_V8SFmode : case E_V8SImode : case E_V16HImode : case E_V32QImode : case E_V4SFmode : case E_V4SImode : case E_V8HImode : case E_V4HImode : break ; case E_V16QImode : if ( TARGET_SSE4_1 ) break ; wmode = V8HImode ; goto widen ; case E_V8QImode : if ( TARGET_MMX_WITH_SSE && TARGET_SSE4_1 ) break ; wmode = V4HImode ; goto widen ; case E_V4QImode : if ( TARGET_SSE4_1 ) break ; wmode = V2HImode ; widen : x = XVECEXP ( vals , 0 , one_var ^ 1 ) ; if ( one_var & 1 ) { var = convert_modes ( HImode , QImode , var , true ) ; var = expand_simple_binop ( HImode , ASHIFT , var , GEN_INT ( 8 ) , NULL_RTX , 1 , OPTAB_LIB_WIDEN ) ; x = GEN_INT ( INTVAL ( x ) & 0xff ) ; } else { var = convert_modes ( HImode , QImode , var , true ) ; x = gen_int_mode ( UINTVAL ( x ) << 8 , HImode ) ; } if ( x != const0_rtx ) var = expand_simple_binop ( HImode , IOR , var , x , var , 1 , OPTAB_LIB_WIDEN ) ; x = gen_reg_rtx ( wmode ) ; emit_move_insn ( x , gen_lowpart ( wmode , const_vec ) ) ; ix86_expand_vector_set ( mmx_ok , x , var , one_var >> 1 ) ; emit_move_insn ( target , gen_lowpart ( mode , x ) ) ; return true ; default : return false ; } emit_move_insn ( target , const_vec ) ; ix86_expand_vector_set ( mmx_ok , target , var , one_var ) ; return true ; } </s>

A subroutine of ix86_expand_vector_init . Store into TARGET a vector consisting of the values in VALS . It is known that all elements except ONE_VAR are constants . Return true if successful .


The guts of ix86_expand_vec_perm_const , also used by the ok hook . With all of the interface bits taken care of , perform the expansion in D and return true on success .


<s> static rtx ix86_expand_vec_set_builtin ( tree exp ) { machine_mode tmode , mode1 ; tree arg0 , arg1 , arg2 ; int elt ; rtx op0 , op1 , target ; arg0 = CALL_EXPR_ARG ( exp , 0 ) ; arg1 = CALL_EXPR_ARG ( exp , 1 ) ; arg2 = CALL_EXPR_ARG ( exp , 2 ) ; tmode = TYPE_MODE ( TREE_TYPE ( arg0 ) ) ; mode1 = TYPE_MODE ( TREE_TYPE ( TREE_TYPE ( arg0 ) ) ) ; gcc_assert ( VECTOR_MODE_P ( tmode ) ) ; op0 = expand_expr ( arg0 , NULL_RTX , tmode , EXPAND_NORMAL ) ; op1 = expand_expr ( arg1 , NULL_RTX , mode1 , EXPAND_NORMAL ) ; elt = get_element_number ( TREE_TYPE ( arg0 ) , arg2 ) ; if ( GET_MODE ( op1 ) != mode1 ) op1 = convert_modes ( mode1 , GET_MODE ( op1 ) , op1 , true ) ; op0 = force_reg ( tmode , op0 ) ; op1 = force_reg ( mode1 , op1 ) ; target = gen_reg_rtx ( tmode ) ; emit_move_insn ( target , op0 ) ; ix86_expand_vector_set ( true , target , op1 , elt ) ; return target ; } </s>

A subroutine of ix86_expand_builtin . These builtins are a wrapper around ix86_expand_vector_set . They would be redundant ( for non-MMX ) if we had a language-level syntax for referencing vector elements .


<s> bool ix86_expand_vec_shift_qihi_constant ( enum rtx_code code , rtx dest , rtx op1 , rtx op2 ) { machine_mode qimode , himode ; HOST_WIDE_INT and_constant , xor_constant ; HOST_WIDE_INT shift_amount ; rtx vec_const_and , vec_const_xor ; rtx tmp , op1_subreg ; rtx ( * gen_shift ) ( rtx , rtx , rtx ) ; rtx ( * gen_and ) ( rtx , rtx , rtx ) ; rtx ( * gen_xor ) ( rtx , rtx , rtx ) ; rtx ( * gen_sub ) ( rtx , rtx , rtx ) ; if ( ! CONST_INT_P ( op2 ) ) return false ; qimode = GET_MODE ( dest ) ; shift_amount = INTVAL ( op2 ) ; if ( shift_amount > 7 ) return false ; gcc_assert ( code == ASHIFT || code == ASHIFTRT || code == LSHIFTRT ) ; xor_constant = 1 << ( 8 - shift_amount - 1 ) ; and_constant = ( code == ASHIFT ? 256 - ( 1 << shift_amount ) : ( 1 << ( 8 - shift_amount ) ) - 1 ) ; switch ( qimode ) { case V16QImode : himode = V8HImode ; gen_shift = ( ( code == ASHIFT ) ? gen_ashlv8hi3 : ( code == ASHIFTRT ) ? gen_ashrv8hi3 : gen_lshrv8hi3 ) ; gen_and = gen_andv16qi3 ; gen_xor = gen_xorv16qi3 ; gen_sub = gen_subv16qi3 ; break ; case V32QImode : himode = V16HImode ; gen_shift = ( ( code == ASHIFT ) ? gen_ashlv16hi3 : ( code == ASHIFTRT ) ? gen_ashrv16hi3 : gen_lshrv16hi3 ) ; gen_and = gen_andv32qi3 ; gen_xor = gen_xorv32qi3 ; gen_sub = gen_subv32qi3 ; break ; case V64QImode : himode = V32HImode ; gen_shift = ( ( code == ASHIFT ) ? gen_ashlv32hi3 : ( code == ASHIFTRT ) ? gen_ashrv32hi3 : gen_lshrv32hi3 ) ; gen_and = gen_andv64qi3 ; gen_xor = gen_xorv64qi3 ; gen_sub = gen_subv64qi3 ; break ; default : gcc_unreachable ( ) ; } tmp = gen_reg_rtx ( himode ) ; vec_const_and = gen_reg_rtx ( qimode ) ; op1_subreg = lowpart_subreg ( himode , op1 , qimode ) ; emit_insn ( gen_shift ( tmp , op1_subreg , op2 ) ) ; emit_move_insn ( dest , simplify_gen_subreg ( qimode , tmp , himode , 0 ) ) ; emit_move_insn ( vec_const_and , ix86_build_const_vector ( qimode , true , gen_int_mode ( and_constant , QImode ) ) ) ; emit_insn ( gen_and ( dest , dest , vec_const_and ) ) ; if ( code == ASHIFTRT ) { vec_const_xor = gen_reg_rtx ( qimode ) ; emit_move_insn ( vec_const_xor , ix86_build_const_vector ( qimode , true , gen_int_mode ( xor_constant , QImode ) ) ) ; emit_insn ( gen_xor ( dest , dest , vec_const_xor ) ) ; emit_insn ( gen_sub ( dest , dest , vec_const_xor ) ) ; } return true ; } </s>

Expand a vector operation shift by constant for a V * QImode in terms of the same operation on V * HImode . Return true if success .


<s> rtx ix86_replace_reg_with_reg ( rtx x , rtx from , rtx to ) { gcc_checking_assert ( REG_P ( from ) && REG_P ( to ) && GET_MODE ( from ) == GET_MODE ( to ) ) ; if ( ! reg_overlap_mentioned_p ( from , x ) ) return x ; rtx ret = copy_rtx ( x ) ; subrtx_ptr_iterator :: array_type array ; FOR_EACH_SUBRTX_PTR ( iter , array , & ret , NONCONST ) { rtx * loc = * iter ; x = * loc ; if ( REG_P ( x ) && REGNO ( x ) == REGNO ( from ) ) { if ( x == from ) * loc = to ; else { gcc_checking_assert ( REG_NREGS ( x ) == 1 ) ; * loc = gen_rtx_REG ( GET_MODE ( x ) , REGNO ( to ) ) ; } } } return ret ; } </s>

Replace all occurrences of REG FROM with REG TO in X , including occurrences with different modes .


<s> void ix86_split_convert_uns_si_sse ( rtx operands [ ] ) { machine_mode vecmode ; rtx value , large , zero_or_two31 , input , two31 , x ; large = operands [ 1 ] ; zero_or_two31 = operands [ 2 ] ; input = operands [ 3 ] ; two31 = operands [ 4 ] ; vecmode = GET_MODE ( large ) ; value = gen_rtx_REG ( vecmode , REGNO ( operands [ 0 ] ) ) ; if ( MEM_P ( input ) ) { if ( vecmode == V4SFmode ) emit_insn ( gen_vec_setv4sf_0 ( value , CONST0_RTX ( V4SFmode ) , input ) ) ; else emit_insn ( gen_sse2_loadlpd ( value , CONST0_RTX ( V2DFmode ) , input ) ) ; } else { input = gen_rtx_REG ( vecmode , REGNO ( input ) ) ; emit_move_insn ( value , CONST0_RTX ( vecmode ) ) ; if ( vecmode == V4SFmode ) emit_insn ( gen_sse_movss_v4sf ( value , value , input ) ) ; else emit_insn ( gen_sse2_movsd_v2df ( value , value , input ) ) ; } emit_move_insn ( large , two31 ) ; emit_move_insn ( zero_or_two31 , MEM_P ( two31 ) ? large : two31 ) ; x = gen_rtx_fmt_ee ( LE , vecmode , large , value ) ; emit_insn ( gen_rtx_SET ( large , x ) ) ; x = gen_rtx_AND ( vecmode , zero_or_two31 , large ) ; emit_insn ( gen_rtx_SET ( zero_or_two31 , x ) ) ; x = gen_rtx_MINUS ( vecmode , value , zero_or_two31 ) ; emit_insn ( gen_rtx_SET ( value , x ) ) ; large = gen_rtx_REG ( V4SImode , REGNO ( large ) ) ; emit_insn ( gen_ashlv4si3 ( large , large , GEN_INT ( 31 ) ) ) ; x = gen_rtx_REG ( V4SImode , REGNO ( value ) ) ; if ( vecmode == V4SFmode ) emit_insn ( gen_fix_truncv4sfv4si2 ( x , value ) ) ; else emit_insn ( gen_sse2_cvttpd2dq ( x , value ) ) ; value = x ; emit_insn ( gen_xorv4si3 ( value , value , large ) ) ; } </s>

Post-reload splitter for converting an SF or DFmode value in an SSE register into an unsigned SImode .


<s> static rtx ix86_ssecom_setcc ( const enum rtx_code comparison , bool check_unordered , machine_mode mode , rtx set_dst , rtx target ) { rtx_code_label * label = NULL ; if ( check_unordered ) { gcc_assert ( comparison == EQ || comparison == NE ) ; rtx flag = gen_rtx_REG ( CCFPmode , FLAGS_REG ) ; label = gen_label_rtx ( ) ; rtx tmp = gen_rtx_fmt_ee ( UNORDERED , VOIDmode , flag , const0_rtx ) ; tmp = gen_rtx_IF_THEN_ELSE ( VOIDmode , tmp , gen_rtx_LABEL_REF ( VOIDmode , label ) , pc_rtx ) ; emit_jump_insn ( gen_rtx_SET ( pc_rtx , tmp ) ) ; } if ( GET_MODE ( set_dst ) != mode ) { gcc_assert ( mode == CCAmode || mode == CCCmode || mode == CCOmode || mode == CCPmode || mode == CCSmode || mode == CCZmode ) ; set_dst = gen_rtx_REG ( mode , FLAGS_REG ) ; } emit_insn ( gen_rtx_SET ( gen_rtx_STRICT_LOW_PART ( VOIDmode , target ) , gen_rtx_fmt_ee ( comparison , QImode , set_dst , const0_rtx ) ) ) ; if ( label ) emit_label ( label ) ; return SUBREG_REG ( target ) ; } </s>

Subroutine of ix86_sse_comi and ix86_sse_comi_round to take care of * ordered EQ or unordered NE , generate PF jump .


<s> static bool ix86_use_mask_cmp_p ( machine_mode mode , machine_mode cmp_mode , rtx op_true , rtx op_false ) { if ( GET_MODE_SIZE ( mode ) == 64 ) return true ; gcc_assert ( ! op_true == ! op_false ) ; if ( ! op_true || ! ix86_valid_mask_cmp_mode ( cmp_mode ) ) return false ; if ( op_false == CONST0_RTX ( mode ) || op_true == CONST0_RTX ( mode ) || ( INTEGRAL_MODE_P ( mode ) && ( op_true == CONSTM1_RTX ( mode ) || op_false == CONSTM1_RTX ( mode ) ) ) ) return false ; return true ; } </s>

Return true if integer mask comparison should be used .


<s> static bool ix86_vector_duplicate_value ( machine_mode mode , rtx target , rtx val ) { bool ok ; rtx_insn * insn ; rtx dup ; recog_data_d recog_data_save = recog_data ; dup = gen_vec_duplicate ( mode , val ) ; insn = emit_insn ( gen_rtx_SET ( target , dup ) ) ; if ( recog_memoized ( insn ) < 0 ) { rtx_insn * seq ; machine_mode innermode = GET_MODE_INNER ( mode ) ; rtx reg ; start_sequence ( ) ; reg = force_reg ( innermode , val ) ; if ( GET_MODE ( reg ) != innermode ) reg = gen_lowpart ( innermode , reg ) ; SET_SRC ( PATTERN ( insn ) ) = gen_vec_duplicate ( mode , reg ) ; seq = get_insns ( ) ; end_sequence ( ) ; if ( seq ) emit_insn_before ( seq , insn ) ; ok = recog_memoized ( insn ) >= 0 ; gcc_assert ( ok ) ; } recog_data = recog_data_save ; return true ; } </s>

A subroutine of ix86_expand_vector_init_duplicate . Tries to fill target with val via vec_duplicate .


<s> static void ix86_vec_perm_index_canon ( struct expand_vec_perm_d * d ) { unsigned nelt = d -> nelt ; if ( d -> perm [ 0 ] < nelt ) return ; for ( unsigned i = 0 ; i != nelt ; i ++ ) d -> perm [ i ] = ( d -> perm [ i ] + nelt ) % ( 2 * nelt ) ; std :: swap ( d -> op0 , d -> op1 ) ; return ; } </s>

Canonicalize vec_perm index to make the first index always comes from the first vector .


<s> void split_double_concat ( machine_mode mode , rtx dst , rtx lo , rtx hi ) { rtx dlo , dhi ; int deleted_move_count = 0 ; split_double_mode ( mode , & dst , 1 , & dlo , & dhi ) ; if ( MEM_P ( lo ) && rtx_equal_p ( dlo , hi ) && reg_overlap_mentioned_p ( dhi , lo ) ) { emit_move_insn ( dhi , lo ) ; lo = dhi ; } else if ( MEM_P ( hi ) && ! MEM_P ( lo ) && ! rtx_equal_p ( dlo , lo ) && reg_overlap_mentioned_p ( dlo , hi ) ) { if ( rtx_equal_p ( dhi , lo ) ) { emit_move_insn ( dlo , hi ) ; hi = dlo ; } else { emit_move_insn ( dhi , hi ) ; hi = dhi ; } } if ( ! rtx_equal_p ( dlo , hi ) ) { if ( ! rtx_equal_p ( dlo , lo ) ) emit_move_insn ( dlo , lo ) ; else deleted_move_count ++ ; if ( ! rtx_equal_p ( dhi , hi ) ) emit_move_insn ( dhi , hi ) ; else deleted_move_count ++ ; } else if ( ! rtx_equal_p ( lo , dhi ) ) { if ( ! rtx_equal_p ( dhi , hi ) ) emit_move_insn ( dhi , hi ) ; else deleted_move_count ++ ; if ( ! rtx_equal_p ( dlo , lo ) ) emit_move_insn ( dlo , lo ) ; else deleted_move_count ++ ; } else if ( mode == TImode ) emit_insn ( gen_swapdi ( dlo , dhi ) ) ; else emit_insn ( gen_swapsi ( dlo , dhi ) ) ; if ( deleted_move_count == 2 ) emit_note ( NOTE_INSN_DELETED ) ; } </s>

Emit the double word assignment DST = { LO , HI } .


<s> void split_double_mode ( machine_mode mode , rtx operands [ ] , int num , rtx lo_half [ ] , rtx hi_half [ ] ) { machine_mode half_mode ; unsigned int byte ; rtx mem_op = NULL_RTX ; int mem_num = 0 ; switch ( mode ) { case E_TImode : half_mode = DImode ; break ; case E_DImode : half_mode = SImode ; break ; case E_P2HImode : half_mode = HImode ; break ; case E_P2QImode : half_mode = QImode ; break ; default : gcc_unreachable ( ) ; } byte = GET_MODE_SIZE ( half_mode ) ; while ( num -- ) { rtx op = operands [ num ] ; if ( MEM_P ( op ) ) { if ( mem_op && rtx_equal_p ( op , mem_op ) ) { lo_half [ num ] = lo_half [ mem_num ] ; hi_half [ num ] = hi_half [ mem_num ] ; } else { mem_op = op ; mem_num = num ; lo_half [ num ] = adjust_address ( op , half_mode , 0 ) ; hi_half [ num ] = adjust_address ( op , half_mode , byte ) ; } } else { lo_half [ num ] = simplify_gen_subreg ( half_mode , op , GET_MODE ( op ) == VOIDmode ? mode : GET_MODE ( op ) , 0 ) ; hi_half [ num ] = simplify_gen_subreg ( half_mode , op , GET_MODE ( op ) == VOIDmode ? mode : GET_MODE ( op ) , byte ) ; } } } </s>

Split one or more double-mode RTL references into pairs of half-mode references . The RTL can be REG , offsettable MEM , integer constant , or CONST_DOUBLE . `` operands '' is a pointer to an array of double-mode RTLs to split and `` num '' is its length . lo_half and hi_half are output arrays that parallel `` operands '' .


<s> static bool alg_usable_p ( enum stringop_alg alg , bool memset , bool have_as ) { if ( alg == no_stringop ) return false ; if ( alg == vector_loop ) return TARGET_SSE || TARGET_AVX ; if ( alg == rep_prefix_1_byte || alg == rep_prefix_4_byte || alg == rep_prefix_8_byte ) { if ( have_as ) return false ; if ( fixed_regs [ CX_REG ] || fixed_regs [ DI_REG ] || ( memset ? fixed_regs [ AX_REG ] : fixed_regs [ SI_REG ] ) ) return false ; } return true ; } </s>

Return true if ALG can be used in current context . Assume we expand memset if MEMSET is true .


<s> static bool canonicalize_perm ( struct expand_vec_perm_d * d ) { int i , which , nelt = d -> nelt ; for ( i = which = 0 ; i < nelt ; ++ i ) which |= ( d -> perm [ i ] < nelt ? 1 : 2 ) ; d -> one_operand_p = true ; switch ( which ) { default : gcc_unreachable ( ) ; case 3 : if ( ! rtx_equal_p ( d -> op0 , d -> op1 ) ) { d -> one_operand_p = false ; break ; } case 2 : for ( i = 0 ; i < nelt ; ++ i ) d -> perm [ i ] &= nelt - 1 ; d -> op0 = d -> op1 ; break ; case 1 : d -> op1 = d -> op0 ; break ; } return ( which == 3 ) ; } </s>

If a permutation only uses one operand , make it clear . Returns true if the permutation references both operands .


<s> static bool canonicalize_vector_int_perm ( const struct expand_vec_perm_d * d , struct expand_vec_perm_d * nd ) { int i ; machine_mode mode = VOIDmode ; switch ( d -> vmode ) { case E_V8QImode : mode = V4HImode ; break ; case E_V16QImode : mode = V8HImode ; break ; case E_V32QImode : mode = V16HImode ; break ; case E_V64QImode : mode = V32HImode ; break ; case E_V4HImode : mode = V2SImode ; break ; case E_V8HImode : mode = V4SImode ; break ; case E_V16HImode : mode = V8SImode ; break ; case E_V32HImode : mode = V16SImode ; break ; case E_V4SImode : mode = V2DImode ; break ; case E_V8SImode : mode = V4DImode ; break ; case E_V16SImode : mode = V8DImode ; break ; default : return false ; } for ( i = 0 ; i < d -> nelt ; i += 2 ) if ( ( d -> perm [ i ] & 1 ) || d -> perm [ i + 1 ] != d -> perm [ i ] + 1 ) return false ; nd -> vmode = mode ; nd -> nelt = d -> nelt / 2 ; for ( i = 0 ; i < nd -> nelt ; i ++ ) nd -> perm [ i ] = d -> perm [ 2 * i ] / 2 ; if ( GET_MODE_INNER ( mode ) != DImode ) canonicalize_vector_int_perm ( nd , nd ) ; if ( nd != d ) { nd -> one_operand_p = d -> one_operand_p ; nd -> testing_p = d -> testing_p ; if ( d -> op0 == d -> op1 ) nd -> op0 = nd -> op1 = gen_lowpart ( nd -> vmode , d -> op0 ) ; else { nd -> op0 = gen_lowpart ( nd -> vmode , d -> op0 ) ; nd -> op1 = gen_lowpart ( nd -> vmode , d -> op1 ) ; } if ( d -> testing_p ) nd -> target = gen_raw_REG ( nd -> vmode , LAST_VIRTUAL_REGISTER + 1 ) ; else nd -> target = gen_reg_rtx ( nd -> vmode ) ; } return true ; } </s>

For V * [ QHS ] Imode permutations , check if the same permutation ca n't be performed in a 2x , 4x or 8x wider inner mode .


<s> static rtx construct_plt_address ( rtx symbol ) { rtx tmp , unspec ; gcc_assert ( GET_CODE ( symbol ) == SYMBOL_REF ) ; gcc_assert ( ix86_cmodel == CM_LARGE_PIC && ! TARGET_PECOFF ) ; gcc_assert ( Pmode == DImode ) ; tmp = gen_reg_rtx ( Pmode ) ; unspec = gen_rtx_UNSPEC ( Pmode , gen_rtvec ( 1 , symbol ) , UNSPEC_PLTOFF ) ; emit_move_insn ( tmp , gen_rtx_CONST ( Pmode , unspec ) ) ; emit_insn ( gen_add2_insn ( tmp , pic_offset_table_rtx ) ) ; return tmp ; } </s>

For given symbol ( function ) construct code to compute address of it 's PLT entry in large x86-64 PIC model .


<s> static bool const_vector_equal_evenodd_p ( rtx op ) { machine_mode mode = GET_MODE ( op ) ; int i , nunits = GET_MODE_NUNITS ( mode ) ; if ( GET_CODE ( op ) != CONST_VECTOR || nunits != CONST_VECTOR_NUNITS ( op ) ) return false ; for ( i = 0 ; i < nunits ; i += 2 ) if ( CONST_VECTOR_ELT ( op , i ) != CONST_VECTOR_ELT ( op , i + 1 ) ) return false ; return true ; } </s>

Helper function of ix86_expand_mul_widen_evenodd . Return true if op is CONST_VECTOR with all odd elements equal to their preceding element .


<s> static machine_mode counter_mode ( rtx count_exp ) { if ( GET_MODE ( count_exp ) != VOIDmode ) return GET_MODE ( count_exp ) ; if ( ! CONST_INT_P ( count_exp ) ) return Pmode ; if ( TARGET_64BIT && ( INTVAL ( count_exp ) & ~ 0xffffffff ) ) return DImode ; return SImode ; } </s>

Return mode for the memcpy/memset loop counter . Prefer SImode over DImode for constant loop counts .


<s> static int decide_alignment ( int align , enum stringop_alg alg , int expected_size , machine_mode move_mode ) { int desired_align = 0 ; gcc_assert ( alg != no_stringop ) ; if ( alg == libcall ) return 0 ; if ( move_mode == VOIDmode ) return 0 ; desired_align = GET_MODE_SIZE ( move_mode ) ; if ( TARGET_CPU_P ( PENTIUMPRO ) && ( alg == rep_prefix_4_byte || alg == rep_prefix_1_byte ) ) desired_align = 8 ; if ( optimize_size ) desired_align = 1 ; if ( desired_align < align ) desired_align = align ; if ( expected_size != - 1 && expected_size < 4 ) desired_align = align ; return desired_align ; } </s>

Decide on alignment . We know that the operand is already aligned to ALIGN ( ALIGN can be based on profile feedback and thus it is not 100 % guaranteed ) .


<s> static rtx emit_memmov ( rtx destmem , rtx * srcmem , rtx destptr , rtx srcptr , HOST_WIDE_INT size_to_move ) { rtx dst = destmem , src = * srcmem , tempreg ; enum insn_code code ; machine_mode move_mode ; int piece_size , i ; piece_size = 1 << floor_log2 ( size_to_move ) ; while ( ! int_mode_for_size ( piece_size * BITS_PER_UNIT , 0 ) . exists ( & move_mode ) || ( code = optab_handler ( mov_optab , move_mode ) ) == CODE_FOR_nothing ) { gcc_assert ( piece_size > 1 ) ; piece_size >>= 1 ; } if ( GET_MODE_SIZE ( move_mode ) > GET_MODE_SIZE ( word_mode ) ) { int nunits = GET_MODE_SIZE ( move_mode ) / GET_MODE_SIZE ( word_mode ) ; if ( ! mode_for_vector ( word_mode , nunits ) . exists ( & move_mode ) || ( code = optab_handler ( mov_optab , move_mode ) ) == CODE_FOR_nothing ) { move_mode = word_mode ; piece_size = GET_MODE_SIZE ( move_mode ) ; code = optab_handler ( mov_optab , move_mode ) ; } } gcc_assert ( code != CODE_FOR_nothing ) ; dst = adjust_automodify_address_nv ( dst , move_mode , destptr , 0 ) ; src = adjust_automodify_address_nv ( src , move_mode , srcptr , 0 ) ; gcc_assert ( size_to_move % piece_size == 0 ) ; for ( i = 0 ; i < size_to_move ; i += piece_size ) { tempreg = gen_reg_rtx ( move_mode ) ; emit_insn ( GEN_FCN ( code ) ( tempreg , src ) ) ; emit_insn ( GEN_FCN ( code ) ( dst , tempreg ) ) ; emit_move_insn ( destptr , plus_constant ( Pmode , copy_rtx ( destptr ) , piece_size ) ) ; emit_move_insn ( srcptr , plus_constant ( Pmode , copy_rtx ( srcptr ) , piece_size ) ) ; dst = adjust_automodify_address_nv ( dst , move_mode , destptr , piece_size ) ; src = adjust_automodify_address_nv ( src , move_mode , srcptr , piece_size ) ; } * srcmem = src ; return dst ; } </s>

This function emits moves to copy SIZE_TO_MOVE bytes from SRCMEM to DESTMEM . SRC is passed by pointer to be updated on return . Return value is updated DST .


<s> static rtx emit_memset ( rtx destmem , rtx destptr , rtx promoted_val , HOST_WIDE_INT size_to_move ) { rtx dst = destmem ; enum insn_code code ; machine_mode move_mode ; int piece_size , i ; move_mode = GET_MODE ( promoted_val ) ; if ( move_mode == VOIDmode ) move_mode = QImode ; if ( size_to_move < GET_MODE_SIZE ( move_mode ) ) { unsigned int move_bits = size_to_move * BITS_PER_UNIT ; move_mode = int_mode_for_size ( move_bits , 0 ) . require ( ) ; promoted_val = gen_lowpart ( move_mode , promoted_val ) ; } piece_size = GET_MODE_SIZE ( move_mode ) ; code = optab_handler ( mov_optab , move_mode ) ; gcc_assert ( code != CODE_FOR_nothing && promoted_val != NULL_RTX ) ; dst = adjust_automodify_address_nv ( dst , move_mode , destptr , 0 ) ; gcc_assert ( size_to_move % piece_size == 0 ) ; for ( i = 0 ; i < size_to_move ; i += piece_size ) { if ( piece_size <= GET_MODE_SIZE ( word_mode ) ) { emit_insn ( gen_strset ( destptr , dst , promoted_val ) ) ; dst = adjust_automodify_address_nv ( dst , move_mode , destptr , piece_size ) ; continue ; } emit_insn ( GEN_FCN ( code ) ( dst , promoted_val ) ) ; emit_move_insn ( destptr , plus_constant ( Pmode , copy_rtx ( destptr ) , piece_size ) ) ; dst = adjust_automodify_address_nv ( dst , move_mode , destptr , piece_size ) ; } return dst ; } </s>

This function emits moves to fill SIZE_TO_MOVE bytes starting from DESTMEM with value PROMOTED_VAL . SRC is passed by pointer to be updated on return . Return value is updated DST .


<s> static void expand_setmem_epilogue_via_loop ( rtx destmem , rtx destptr , rtx value , rtx count , int max_size ) { count = expand_simple_binop ( counter_mode ( count ) , AND , count , GEN_INT ( max_size - 1 ) , count , 1 , OPTAB_DIRECT ) ; expand_set_or_cpymem_via_loop ( destmem , NULL , destptr , NULL , gen_lowpart ( QImode , value ) , count , QImode , 1 , max_size / 2 , true ) ; } </s>

Output code to set at most count & ( max_size - 1 ) bytes starting by DEST .


<s> static rtx expand_set_or_cpymem_constant_prologue ( rtx dst , rtx * srcp , rtx destreg , rtx srcreg , rtx value , rtx vec_value , int desired_align , int align_bytes , bool issetmem ) { rtx src = NULL ; rtx orig_dst = dst ; rtx orig_src = NULL ; int piece_size = 1 ; int copied_bytes = 0 ; if ( ! issetmem ) { gcc_assert ( srcp != NULL ) ; src = * srcp ; orig_src = src ; } for ( piece_size = 1 ; piece_size <= desired_align && copied_bytes < align_bytes ; piece_size <<= 1 ) { if ( align_bytes & piece_size ) { if ( issetmem ) { if ( vec_value && piece_size > GET_MODE_SIZE ( GET_MODE ( value ) ) ) dst = emit_memset ( dst , destreg , vec_value , piece_size ) ; else dst = emit_memset ( dst , destreg , value , piece_size ) ; } else dst = emit_memmov ( dst , & src , destreg , srcreg , piece_size ) ; copied_bytes += piece_size ; } } if ( MEM_ALIGN ( dst ) < ( unsigned int ) desired_align * BITS_PER_UNIT ) set_mem_align ( dst , desired_align * BITS_PER_UNIT ) ; if ( MEM_SIZE_KNOWN_P ( orig_dst ) ) set_mem_size ( dst , MEM_SIZE ( orig_dst ) - align_bytes ) ; if ( ! issetmem ) { int src_align_bytes = get_mem_align_offset ( src , desired_align * BITS_PER_UNIT ) ; if ( src_align_bytes >= 0 ) src_align_bytes = desired_align - src_align_bytes ; if ( src_align_bytes >= 0 ) { unsigned int src_align ; for ( src_align = desired_align ; src_align >= 2 ; src_align >>= 1 ) { if ( ( src_align_bytes & ( src_align - 1 ) ) == ( align_bytes & ( src_align - 1 ) ) ) break ; } if ( src_align > ( unsigned int ) desired_align ) src_align = desired_align ; if ( MEM_ALIGN ( src ) < src_align * BITS_PER_UNIT ) set_mem_align ( src , src_align * BITS_PER_UNIT ) ; } if ( MEM_SIZE_KNOWN_P ( orig_src ) ) set_mem_size ( src , MEM_SIZE ( orig_src ) - align_bytes ) ; * srcp = src ; } return dst ; } </s>

This function is like the previous one , except here we know how many bytes need to be copied . That allows us to update alignment not only of DST , which is returned , but also of SRC , which is passed as a pointer for that reason .


<s> static rtx expand_set_or_cpymem_prologue ( rtx destmem , rtx srcmem , rtx destptr , rtx srcptr , rtx value , rtx vec_value , rtx count , int align , int desired_alignment , bool issetmem ) { int i ; for ( i = 1 ; i < desired_alignment ; i <<= 1 ) { if ( align <= i ) { rtx_code_label * label = ix86_expand_aligntest ( destptr , i , false ) ; if ( issetmem ) { if ( vec_value && i > GET_MODE_SIZE ( GET_MODE ( value ) ) ) destmem = emit_memset ( destmem , destptr , vec_value , i ) ; else destmem = emit_memset ( destmem , destptr , value , i ) ; } else destmem = emit_memmov ( destmem , & srcmem , destptr , srcptr , i ) ; ix86_adjust_counter ( count , i ) ; emit_label ( label ) ; LABEL_NUSES ( label ) = 1 ; set_mem_align ( destmem , i * 2 * BITS_PER_UNIT ) ; } } return destmem ; } </s>

Depending on ISSETMEM , copy enough from SRCMEM to DESTMEM or set enough to DESTMEM to align it to DESIRED_ALIGNMENT . Original alignment is ALIGN . Depending on ISSETMEM , either arguments SRCMEM/SRCPTR or VALUE/VEC_VALUE are ignored . Return value is updated DESTMEM .


<s> static void expand_set_or_cpymem_via_rep ( rtx destmem , rtx srcmem , rtx destptr , rtx srcptr , rtx value , rtx orig_value , rtx count , machine_mode mode , bool issetmem ) { rtx destexp ; rtx srcexp ; rtx countreg ; HOST_WIDE_INT rounded_count ; if ( mode == QImode && CONST_INT_P ( count ) && ! ( INTVAL ( count ) & 3 ) && ! TARGET_PREFER_KNOWN_REP_MOVSB_STOSB && ( ! issetmem || orig_value == const0_rtx ) ) mode = SImode ; if ( destptr != XEXP ( destmem , 0 ) || GET_MODE ( destmem ) != BLKmode ) destmem = adjust_automodify_address_nv ( destmem , BLKmode , destptr , 0 ) ; countreg = ix86_zero_extend_to_Pmode ( scale_counter ( count , GET_MODE_SIZE ( mode ) ) ) ; if ( mode != QImode ) { destexp = gen_rtx_ASHIFT ( Pmode , countreg , GEN_INT ( exact_log2 ( GET_MODE_SIZE ( mode ) ) ) ) ; destexp = gen_rtx_PLUS ( Pmode , destexp , destptr ) ; } else destexp = gen_rtx_PLUS ( Pmode , destptr , countreg ) ; if ( ( ! issetmem || orig_value == const0_rtx ) && CONST_INT_P ( count ) ) { rounded_count = ROUND_DOWN ( INTVAL ( count ) , ( HOST_WIDE_INT ) GET_MODE_SIZE ( mode ) ) ; destmem = shallow_copy_rtx ( destmem ) ; set_mem_size ( destmem , rounded_count ) ; } else if ( MEM_SIZE_KNOWN_P ( destmem ) ) clear_mem_size ( destmem ) ; if ( issetmem ) { value = force_reg ( mode , gen_lowpart ( mode , value ) ) ; emit_insn ( gen_rep_stos ( destptr , countreg , destmem , value , destexp ) ) ; } else { if ( srcptr != XEXP ( srcmem , 0 ) || GET_MODE ( srcmem ) != BLKmode ) srcmem = adjust_automodify_address_nv ( srcmem , BLKmode , srcptr , 0 ) ; if ( mode != QImode ) { srcexp = gen_rtx_ASHIFT ( Pmode , countreg , GEN_INT ( exact_log2 ( GET_MODE_SIZE ( mode ) ) ) ) ; srcexp = gen_rtx_PLUS ( Pmode , srcexp , srcptr ) ; } else srcexp = gen_rtx_PLUS ( Pmode , srcptr , countreg ) ; if ( CONST_INT_P ( count ) ) { rounded_count = ROUND_DOWN ( INTVAL ( count ) , ( HOST_WIDE_INT ) GET_MODE_SIZE ( mode ) ) ; srcmem = shallow_copy_rtx ( srcmem ) ; set_mem_size ( srcmem , rounded_count ) ; } else { if ( MEM_SIZE_KNOWN_P ( srcmem ) ) clear_mem_size ( srcmem ) ; } emit_insn ( gen_rep_mov ( destptr , destmem , srcptr , srcmem , countreg , destexp , srcexp ) ) ; } } </s>

Output `` rep ; mov '' or `` rep ; stos '' instruction depending on ISSETMEM argument . When ISSETMEM is true , arguments SRCMEM and SRCPTR are ignored . When ISSETMEM is false , arguments VALUE and ORIG_VALUE are ignored . For setmem case , VALUE is a promoted to a wider size ORIG_VALUE . ORIG_VALUE is the original value passed to memset to fill the memory with . Other arguments have same meaning as for previous function .


<s> static void expand_small_cpymem_or_setmem ( rtx destmem , rtx srcmem , rtx destptr , rtx srcptr , rtx value , rtx vec_value , rtx count , int size , rtx done_label , bool issetmem ) { rtx_code_label * label = ix86_expand_aligntest ( count , size , false ) ; machine_mode mode = int_mode_for_size ( size * BITS_PER_UNIT , 1 ) . else_blk ( ) ; rtx modesize ; int n ; if ( issetmem ) { if ( ! vec_value ) { if ( GET_MODE ( value ) == VOIDmode && size > 8 ) mode = Pmode ; else if ( GET_MODE_SIZE ( mode ) > GET_MODE_SIZE ( GET_MODE ( value ) ) ) mode = GET_MODE ( value ) ; } else mode = GET_MODE ( vec_value ) , value = vec_value ; } else { if ( size >= 32 ) mode = TARGET_AVX ? V32QImode : TARGET_SSE ? V16QImode : DImode ; else if ( size >= 16 ) mode = TARGET_SSE ? V16QImode : DImode ; srcmem = change_address ( srcmem , mode , srcptr ) ; } destmem = change_address ( destmem , mode , destptr ) ; modesize = GEN_INT ( GET_MODE_SIZE ( mode ) ) ; gcc_assert ( GET_MODE_SIZE ( mode ) <= size ) ; for ( n = 0 ; n * GET_MODE_SIZE ( mode ) < size ; n ++ ) { if ( issetmem ) emit_move_insn ( destmem , gen_lowpart ( mode , value ) ) ; else { emit_move_insn ( destmem , srcmem ) ; srcmem = offset_address ( srcmem , modesize , GET_MODE_SIZE ( mode ) ) ; } destmem = offset_address ( destmem , modesize , GET_MODE_SIZE ( mode ) ) ; } destmem = offset_address ( destmem , count , 1 ) ; destmem = offset_address ( destmem , GEN_INT ( - 2 * size ) , GET_MODE_SIZE ( mode ) ) ; if ( ! issetmem ) { srcmem = offset_address ( srcmem , count , 1 ) ; srcmem = offset_address ( srcmem , GEN_INT ( - 2 * size ) , GET_MODE_SIZE ( mode ) ) ; } for ( n = 0 ; n * GET_MODE_SIZE ( mode ) < size ; n ++ ) { if ( issetmem ) emit_move_insn ( destmem , gen_lowpart ( mode , value ) ) ; else { emit_move_insn ( destmem , srcmem ) ; srcmem = offset_address ( srcmem , modesize , GET_MODE_SIZE ( mode ) ) ; } destmem = offset_address ( destmem , modesize , GET_MODE_SIZE ( mode ) ) ; } emit_jump_insn ( gen_jump ( done_label ) ) ; emit_barrier ( ) ; emit_label ( label ) ; LABEL_NUSES ( label ) = 1 ; } </s>

Test if COUNT & SIZE is nonzero and if so , expand movme or setmem sequence that is valid for SIZE .. 2 * SIZE-1 bytes and jump to DONE_LABEL .


<s> static bool expand_vec_perm_2vperm2f128_vshuf ( struct expand_vec_perm_d * d ) { struct expand_vec_perm_d dfirst , dsecond , dthird ; bool ok ; if ( ! TARGET_AVX || ( d -> vmode != V4DFmode ) ) return false ; if ( d -> testing_p ) return true ; dfirst = * d ; dsecond = * d ; dthird = * d ; dfirst . perm [ 0 ] = ( d -> perm [ 0 ] & ~ 1 ) ; dfirst . perm [ 1 ] = ( d -> perm [ 0 ] & ~ 1 ) + 1 ; dfirst . perm [ 2 ] = ( d -> perm [ 2 ] & ~ 1 ) ; dfirst . perm [ 3 ] = ( d -> perm [ 2 ] & ~ 1 ) + 1 ; dsecond . perm [ 0 ] = ( d -> perm [ 1 ] & ~ 1 ) ; dsecond . perm [ 1 ] = ( d -> perm [ 1 ] & ~ 1 ) + 1 ; dsecond . perm [ 2 ] = ( d -> perm [ 3 ] & ~ 1 ) ; dsecond . perm [ 3 ] = ( d -> perm [ 3 ] & ~ 1 ) + 1 ; dthird . perm [ 0 ] = ( d -> perm [ 0 ] % 2 ) ; dthird . perm [ 1 ] = ( d -> perm [ 1 ] % 2 ) + 4 ; dthird . perm [ 2 ] = ( d -> perm [ 2 ] % 2 ) + 2 ; dthird . perm [ 3 ] = ( d -> perm [ 3 ] % 2 ) + 6 ; dfirst . target = gen_reg_rtx ( dfirst . vmode ) ; dsecond . target = gen_reg_rtx ( dsecond . vmode ) ; dthird . op0 = dfirst . target ; dthird . op1 = dsecond . target ; dthird . one_operand_p = false ; canonicalize_perm ( & dfirst ) ; canonicalize_perm ( & dsecond ) ; ok = expand_vec_perm_1 ( & dfirst ) && expand_vec_perm_1 ( & dsecond ) && expand_vec_perm_1 ( & dthird ) ; gcc_assert ( ok ) ; return true ; } </s>

A subroutine of ix86_expand_vec_perm_builtin_1 . Implement a V4DF permutation using two vperm2f128 , followed by a vshufpd insn blending the two vectors together .


<s> static bool expand_vec_perm_broadcast ( struct expand_vec_perm_d * d ) { unsigned i , elt , nelt = d -> nelt ; if ( ! d -> one_operand_p ) return false ; elt = d -> perm [ 0 ] ; for ( i = 1 ; i < nelt ; ++ i ) if ( d -> perm [ i ] != elt ) return false ; return expand_vec_perm_broadcast_1 ( d ) ; } </s>


<s> static bool expand_vec_perm_even_odd ( struct expand_vec_perm_d * d ) { unsigned i , odd , nelt = d -> nelt ; odd = d -> perm [ 0 ] ; if ( odd != 0 && odd != 1 ) return false ; for ( i = 1 ; i < nelt ; ++ i ) if ( d -> perm [ i ] != 2 * i + odd ) return false ; if ( d -> vmode == E_V32HImode && d -> testing_p && ! TARGET_AVX512BW ) return false ; return expand_vec_perm_even_odd_1 ( d , odd ) ; } </s>

A subroutine of ix86_expand_vec_perm_builtin_1 . Pattern match extract-even and extract-odd permutations .


<s> static bool expand_vec_perm_even_odd_trunc ( struct expand_vec_perm_d * d ) { rtx t1 , t2 , t3 , t4 ; unsigned i , odd , nelt = d -> nelt ; if ( ! TARGET_AVX512BW || d -> one_operand_p || d -> vmode != V64QImode ) return false ; odd = d -> perm [ 0 ] ; if ( odd > 1 ) return false ; for ( i = 1 ; i < nelt ; ++ i ) if ( d -> perm [ i ] != 2 * i + odd ) return false ; if ( d -> testing_p ) return true ; if ( odd ) { t1 = gen_reg_rtx ( V32HImode ) ; t2 = gen_reg_rtx ( V32HImode ) ; emit_insn ( gen_lshrv32hi3 ( t1 , gen_lowpart ( V32HImode , d -> op0 ) , GEN_INT ( 8 ) ) ) ; emit_insn ( gen_lshrv32hi3 ( t2 , gen_lowpart ( V32HImode , d -> op1 ) , GEN_INT ( 8 ) ) ) ; } else { t1 = gen_lowpart ( V32HImode , d -> op0 ) ; t2 = gen_lowpart ( V32HImode , d -> op1 ) ; } t3 = gen_reg_rtx ( V32QImode ) ; t4 = gen_reg_rtx ( V32QImode ) ; emit_insn ( gen_avx512bw_truncatev32hiv32qi2 ( t3 , t1 ) ) ; emit_insn ( gen_avx512bw_truncatev32hiv32qi2 ( t4 , t2 ) ) ; emit_insn ( gen_avx_vec_concatv64qi ( d -> target , t3 , t4 ) ) ; return true ; } </s>

A subroutine of expand_vec_perm_even_odd_1 . Implement extract-even and extract-odd permutations of two V64QI operands with two `` shifts '' , two `` truncs '' and one `` concat '' insns for `` odd '' and two `` truncs '' and one concat insn for `` even . '' Have already failed all two instruction sequences .


<s> static bool expand_vec_perm_interleave3 ( struct expand_vec_perm_d * d ) { unsigned i , nelt ; rtx ( * gen ) ( rtx , rtx , rtx ) ; if ( d -> one_operand_p ) return false ; if ( TARGET_AVX2 && GET_MODE_SIZE ( d -> vmode ) == 32 ) ; else if ( TARGET_AVX && ( d -> vmode == V8SFmode || d -> vmode == V4DFmode ) ) ; else return false ; nelt = d -> nelt ; if ( d -> perm [ 0 ] != 0 && d -> perm [ 0 ] != nelt / 2 ) return false ; for ( i = 0 ; i < nelt ; i += 2 ) if ( d -> perm [ i ] != d -> perm [ 0 ] + i / 2 || d -> perm [ i + 1 ] != d -> perm [ 0 ] + i / 2 + nelt ) return false ; if ( d -> testing_p ) return true ; switch ( d -> vmode ) { case E_V32QImode : if ( d -> perm [ 0 ] ) gen = gen_vec_interleave_highv32qi ; else gen = gen_vec_interleave_lowv32qi ; break ; case E_V16HImode : if ( d -> perm [ 0 ] ) gen = gen_vec_interleave_highv16hi ; else gen = gen_vec_interleave_lowv16hi ; break ; case E_V8SImode : if ( d -> perm [ 0 ] ) gen = gen_vec_interleave_highv8si ; else gen = gen_vec_interleave_lowv8si ; break ; case E_V4DImode : if ( d -> perm [ 0 ] ) gen = gen_vec_interleave_highv4di ; else gen = gen_vec_interleave_lowv4di ; break ; case E_V8SFmode : if ( d -> perm [ 0 ] ) gen = gen_vec_interleave_highv8sf ; else gen = gen_vec_interleave_lowv8sf ; break ; case E_V4DFmode : if ( d -> perm [ 0 ] ) gen = gen_vec_interleave_highv4df ; else gen = gen_vec_interleave_lowv4df ; break ; default : gcc_unreachable ( ) ; } emit_insn ( gen ( d -> target , d -> op0 , d -> op1 ) ) ; return true ; } </s>

A subroutine of ix86_expand_vec_perm_builtin_1 . Try to simplify a two vector permutation using 2 intra-lane interleave insns and cross-lane shuffle for 32-byte vectors .


<s> static bool expand_vec_perm_movs ( struct expand_vec_perm_d * d ) { machine_mode vmode = d -> vmode ; unsigned i , nelt = d -> nelt ; rtx x ; if ( d -> one_operand_p ) return false ; if ( ! ( TARGET_SSE && vmode == V4SFmode ) && ! ( TARGET_MMX_WITH_SSE && vmode == V2SFmode ) && ! ( TARGET_SSE2 && vmode == V2DFmode ) ) return false ; if ( d -> perm [ 0 ] != nelt && d -> perm [ 0 ] != 0 ) return false ; for ( i = 1 ; i < nelt ; ++ i ) if ( d -> perm [ i ] != i + nelt - d -> perm [ 0 ] ) return false ; if ( d -> testing_p ) return true ; if ( d -> perm [ 0 ] == nelt ) x = gen_rtx_VEC_MERGE ( vmode , d -> op1 , d -> op0 , GEN_INT ( 1 ) ) ; else x = gen_rtx_VEC_MERGE ( vmode , d -> op0 , d -> op1 , GEN_INT ( 1 ) ) ; emit_insn ( gen_rtx_SET ( d -> target , x ) ) ; return true ; } </s>

A subroutine of ix86_expand_vec_perm_builtin_1 . Try to implement D using movss or movsd .


<s> static bool expand_vec_perm_pblendv ( struct expand_vec_perm_d * d ) { unsigned i , which , nelt = d -> nelt ; struct expand_vec_perm_d dcopy , dcopy1 ; machine_mode vmode = d -> vmode ; bool ok ; if ( d -> one_operand_p ) return false ; if ( TARGET_AVX2 && GET_MODE_SIZE ( vmode ) == 32 ) ; else if ( TARGET_AVX && ( vmode == V4DFmode || vmode == V8SFmode ) ) ; else if ( TARGET_SSE4_1 && ( GET_MODE_SIZE ( vmode ) == 4 || GET_MODE_SIZE ( vmode ) == 8 || GET_MODE_SIZE ( vmode ) == 16 ) ) ; else return false ; for ( i = 0 , which = 0 ; i < nelt ; ++ i ) { unsigned e = d -> perm [ i ] ; if ( e != i ) which |= ( e < nelt ? 1 : 2 ) ; } if ( which != 1 && which != 2 ) return false ; if ( d -> testing_p && GET_MODE_SIZE ( vmode ) == 16 ) return true ; dcopy = * d ; if ( which == 2 ) dcopy . op0 = dcopy . op1 = d -> op1 ; else dcopy . op0 = dcopy . op1 = d -> op0 ; if ( ! d -> testing_p ) dcopy . target = gen_reg_rtx ( vmode ) ; dcopy . one_operand_p = true ; for ( i = 0 ; i < nelt ; ++ i ) dcopy . perm [ i ] = d -> perm [ i ] & ( nelt - 1 ) ; ok = expand_vec_perm_1 ( & dcopy ) ; if ( GET_MODE_SIZE ( vmode ) != 16 && ! ok ) return false ; else gcc_assert ( ok ) ; if ( d -> testing_p ) return true ; dcopy1 = * d ; if ( which == 2 ) dcopy1 . op1 = dcopy . target ; else dcopy1 . op0 = dcopy . target ; for ( i = 0 ; i < nelt ; ++ i ) dcopy1 . perm [ i ] = ( ( d -> perm [ i ] >= nelt ) ? ( nelt + i ) : i ) ; ok = expand_vec_perm_blend ( & dcopy1 ) ; gcc_assert ( ok ) ; return true ; } </s>

A subroutine of ix86_expand_vec_perm_const_1 . Try to simplify the permutation using the SSE4_1 pblendv instruction . Potentially reduces permutation from 2 pshufb and or to 1 pshufb and pblendv .


<s> static bool expand_vec_perm_pshufb2 ( struct expand_vec_perm_d * d ) { rtx rperm [ 2 ] [ 16 ] , vperm , l , h , op , m128 ; unsigned int i , nelt , eltsz ; machine_mode mode ; rtx ( * gen ) ( rtx , rtx , rtx ) ; if ( ! TARGET_SSSE3 || ( GET_MODE_SIZE ( d -> vmode ) != 16 && GET_MODE_SIZE ( d -> vmode ) != 8 && GET_MODE_SIZE ( d -> vmode ) != 4 ) ) return false ; gcc_assert ( ! d -> one_operand_p ) ; if ( d -> testing_p ) return true ; switch ( GET_MODE_SIZE ( d -> vmode ) ) { case 4 : mode = V4QImode ; gen = gen_mmx_pshufbv4qi3 ; break ; case 8 : mode = V8QImode ; gen = gen_mmx_pshufbv8qi3 ; break ; case 16 : mode = V16QImode ; gen = gen_ssse3_pshufbv16qi3 ; break ; default : gcc_unreachable ( ) ; } nelt = d -> nelt ; eltsz = GET_MODE_UNIT_SIZE ( d -> vmode ) ; m128 = GEN_INT ( - 128 ) ; for ( i = 0 ; i < nelt ; ++ i ) { unsigned j , k , e = d -> perm [ i ] ; unsigned which = ( e >= nelt ) ; if ( e >= nelt ) e -= nelt ; for ( j = 0 ; j < eltsz ; ++ j ) { rperm [ which ] [ i * eltsz + j ] = GEN_INT ( e * eltsz + j ) ; rperm [ 1 - which ] [ i * eltsz + j ] = m128 ; } for ( k = i * eltsz + j ; k < 16 ; ++ k ) rperm [ 0 ] [ k ] = rperm [ 1 ] [ k ] = m128 ; } vperm = gen_rtx_CONST_VECTOR ( V16QImode , gen_rtvec_v ( 16 , rperm [ 0 ] ) ) ; vperm = force_reg ( V16QImode , vperm ) ; l = gen_reg_rtx ( mode ) ; op = gen_lowpart ( mode , d -> op0 ) ; emit_insn ( gen ( l , op , vperm ) ) ; vperm = gen_rtx_CONST_VECTOR ( V16QImode , gen_rtvec_v ( 16 , rperm [ 1 ] ) ) ; vperm = force_reg ( V16QImode , vperm ) ; h = gen_reg_rtx ( mode ) ; op = gen_lowpart ( mode , d -> op1 ) ; emit_insn ( gen ( h , op , vperm ) ) ; op = d -> target ; if ( d -> vmode != mode ) op = gen_reg_rtx ( mode ) ; ix86_emit_vec_binop ( IOR , mode , op , l , h ) ; if ( op != d -> target ) emit_move_insn ( d -> target , gen_lowpart ( d -> vmode , op ) ) ; return true ; } </s>

A subroutine of expand_vec_perm_even_odd_1 . Implement the double-word permutation with two pshufb insns and an ior . We should have already failed all two instruction sequences .


<s> static bool expand_vec_perm_pshuflw_pshufhw ( struct expand_vec_perm_d * d ) { unsigned char perm2 [ MAX_VECT_LEN ] ; unsigned i ; bool ok ; if ( d -> vmode != V8HImode || ! d -> one_operand_p ) return false ; for ( i = 0 ; i < 4 ; ++ i ) if ( d -> perm [ i ] >= 4 ) return false ; for ( i = 4 ; i < 8 ; ++ i ) if ( d -> perm [ i ] < 4 ) return false ; if ( d -> testing_p ) return true ; memcpy ( perm2 , d -> perm , 4 ) ; for ( i = 4 ; i < 8 ; ++ i ) perm2 [ i ] = i ; ok = expand_vselect ( d -> target , d -> op0 , perm2 , 8 , d -> testing_p ) ; gcc_assert ( ok ) ; memcpy ( perm2 + 4 , d -> perm + 4 , 4 ) ; for ( i = 0 ; i < 4 ; ++ i ) perm2 [ i ] = i ; ok = expand_vselect ( d -> target , d -> target , perm2 , 8 , d -> testing_p ) ; gcc_assert ( ok ) ; return true ; } </s>

A subroutine of ix86_expand_vec_perm_builtin_1 . Try to implement D in terms of a pair of pshuflw + pshufhw instructions .


<s> static bool expand_vec_perm_vperm2f128_vblend ( struct expand_vec_perm_d * d ) { struct expand_vec_perm_d dfirst , dsecond ; unsigned i , j , msk , nelt = d -> nelt , nelt2 = nelt / 2 ; rtx_insn * seq ; bool ok ; rtx ( * blend ) ( rtx , rtx , rtx , rtx ) = NULL ; if ( ! TARGET_AVX || TARGET_AVX2 || ( d -> vmode != V8SFmode && d -> vmode != V4DFmode ) || ! d -> one_operand_p ) return false ; dfirst = * d ; for ( i = 0 ; i < nelt ; i ++ ) dfirst . perm [ i ] = 0xff ; for ( i = 0 , msk = 0 ; i < nelt ; i ++ ) { j = ( d -> perm [ i ] & nelt2 ) ? i | nelt2 : i & ~ nelt2 ; if ( dfirst . perm [ j ] != 0xff && dfirst . perm [ j ] != d -> perm [ i ] ) return false ; dfirst . perm [ j ] = d -> perm [ i ] ; if ( j != i ) msk |= ( 1 << i ) ; } for ( i = 0 ; i < nelt ; i ++ ) if ( dfirst . perm [ i ] == 0xff ) dfirst . perm [ i ] = i ; if ( ! d -> testing_p ) dfirst . target = gen_reg_rtx ( dfirst . vmode ) ; start_sequence ( ) ; ok = expand_vec_perm_1 ( & dfirst ) ; seq = get_insns ( ) ; end_sequence ( ) ; if ( ! ok ) return false ; if ( d -> testing_p ) return true ; emit_insn ( seq ) ; dsecond = * d ; dsecond . op0 = dfirst . target ; dsecond . op1 = dfirst . target ; dsecond . one_operand_p = true ; dsecond . target = gen_reg_rtx ( dsecond . vmode ) ; for ( i = 0 ; i < nelt ; i ++ ) dsecond . perm [ i ] = i ^ nelt2 ; ok = expand_vec_perm_1 ( & dsecond ) ; gcc_assert ( ok ) ; blend = d -> vmode == V8SFmode ? gen_avx_blendps256 : gen_avx_blendpd256 ; emit_insn ( blend ( d -> target , dfirst . target , dsecond . target , GEN_INT ( msk ) ) ) ; return true ; } </s>

A subroutine of ix86_expand_vec_perm_builtin_1 . Try to implement a single vector permutation using a single intra-lane vector permutation , vperm2f128 swapping the lanes and vblend * insn blending the non-swapped and swapped vectors together .


<s> static bool expand_vec_perm_vpermil ( struct expand_vec_perm_d * d ) { rtx rperm [ 8 ] , vperm ; unsigned i ; if ( ! TARGET_AVX || d -> vmode != V8SFmode || ! d -> one_operand_p ) return false ; for ( i = 0 ; i < 8 ; ++ i ) { unsigned e = d -> perm [ i ] ; if ( i < 4 ? e >= 4 : e < 4 ) return false ; } if ( d -> testing_p ) return true ; for ( i = 0 ; i < 8 ; ++ i ) { unsigned e = d -> perm [ i ] ; if ( e >= 8 + 4 ) e -= 8 ; else if ( e >= 4 ) e -= 4 ; rperm [ i ] = GEN_INT ( e ) ; } vperm = gen_rtx_CONST_VECTOR ( V8SImode , gen_rtvec_v ( 8 , rperm ) ) ; vperm = force_reg ( V8SImode , vperm ) ; emit_insn ( gen_avx_vpermilvarv8sf3 ( d -> target , d -> op0 , vperm ) ) ; return true ; } </s>

A subroutine of ix86_expand_vec_perm_builtin_1 . Try to implement D in terms of the variable form of vpermilps . Note that we will have already failed the immediate input vpermilps , which requires that the high and low part shuffle be identical ; the variable form does n't require that .


<s> static bool expand_vec_perm_vpermt2_vpshub2 ( struct expand_vec_perm_d * d ) { if ( ! TARGET_AVX512BW || ! ( d -> vmode == V64QImode ) ) return false ; if ( d -> testing_p ) return true ; struct expand_vec_perm_d ds [ 2 ] ; rtx rperm [ 128 ] , vperm , target0 , target1 ; unsigned int i , nelt ; machine_mode vmode ; nelt = d -> nelt ; vmode = V64QImode ; for ( i = 0 ; i < 2 ; i ++ ) { ds [ i ] = * d ; ds [ i ] . vmode = V32HImode ; ds [ i ] . nelt = 32 ; ds [ i ] . target = gen_reg_rtx ( V32HImode ) ; ds [ i ] . op0 = gen_lowpart ( V32HImode , d -> op0 ) ; ds [ i ] . op1 = gen_lowpart ( V32HImode , d -> op1 ) ; } for ( i = 0 ; i < nelt ; i ++ ) { ds [ i & 1 ] . perm [ i / 2 ] = d -> perm [ i ] / 2 ; if ( i & 1 ) { rperm [ i ] = constm1_rtx ; rperm [ i + 64 ] = GEN_INT ( ( i & 14 ) + ( d -> perm [ i ] & 1 ) ) ; } else { rperm [ i ] = GEN_INT ( ( i & 14 ) + ( d -> perm [ i ] & 1 ) ) ; rperm [ i + 64 ] = constm1_rtx ; } } bool ok = expand_vec_perm_1 ( & ds [ 0 ] ) ; gcc_assert ( ok ) ; ds [ 0 ] . target = gen_lowpart ( V64QImode , ds [ 0 ] . target ) ; ok = expand_vec_perm_1 ( & ds [ 1 ] ) ; gcc_assert ( ok ) ; ds [ 1 ] . target = gen_lowpart ( V64QImode , ds [ 1 ] . target ) ; vperm = gen_rtx_CONST_VECTOR ( V64QImode , gen_rtvec_v ( 64 , rperm ) ) ; vperm = force_reg ( vmode , vperm ) ; target0 = gen_reg_rtx ( V64QImode ) ; emit_insn ( gen_avx512bw_pshufbv64qi3 ( target0 , ds [ 0 ] . target , vperm ) ) ; vperm = gen_rtx_CONST_VECTOR ( V64QImode , gen_rtvec_v ( 64 , rperm + 64 ) ) ; vperm = force_reg ( vmode , vperm ) ; target1 = gen_reg_rtx ( V64QImode ) ; emit_insn ( gen_avx512bw_pshufbv64qi3 ( target1 , ds [ 1 ] . target , vperm ) ) ; emit_insn ( gen_iorv64qi3 ( d -> target , target0 , target1 ) ) ; return true ; } </s>

Implement arbitrary permutations of two V64QImode operands with 2 vperm [ it ] 2w , 2 vpshufb and one vpor instruction .


<s> static bool expand_vec_perm_vpshufb2_vpermq ( struct expand_vec_perm_d * d ) { rtx rperm [ 2 ] [ 32 ] , vperm , l , h , hp , op , m128 ; unsigned int i , nelt , eltsz ; if ( ! TARGET_AVX2 || ! d -> one_operand_p || ( d -> vmode != V32QImode && d -> vmode != V16HImode ) ) return false ; if ( d -> testing_p ) return true ; nelt = d -> nelt ; eltsz = GET_MODE_UNIT_SIZE ( d -> vmode ) ; m128 = GEN_INT ( - 128 ) ; for ( i = 0 ; i < nelt ; ++ i ) { unsigned j , e = d -> perm [ i ] & ( nelt / 2 - 1 ) ; unsigned which = ( ( d -> perm [ i ] ^ i ) & ( nelt / 2 ) ) * eltsz ; for ( j = 0 ; j < eltsz ; ++ j ) { rperm [ ! ! which ] [ ( i * eltsz + j ) ^ which ] = GEN_INT ( e * eltsz + j ) ; rperm [ ! which ] [ ( i * eltsz + j ) ^ ( which ^ 16 ) ] = m128 ; } } vperm = gen_rtx_CONST_VECTOR ( V32QImode , gen_rtvec_v ( 32 , rperm [ 1 ] ) ) ; vperm = force_reg ( V32QImode , vperm ) ; h = gen_reg_rtx ( V32QImode ) ; op = gen_lowpart ( V32QImode , d -> op0 ) ; emit_insn ( gen_avx2_pshufbv32qi3 ( h , op , vperm ) ) ; hp = gen_reg_rtx ( V4DImode ) ; op = gen_lowpart ( V4DImode , h ) ; emit_insn ( gen_avx2_permv4di_1 ( hp , op , const2_rtx , GEN_INT ( 3 ) , const0_rtx , const1_rtx ) ) ; vperm = gen_rtx_CONST_VECTOR ( V32QImode , gen_rtvec_v ( 32 , rperm [ 0 ] ) ) ; vperm = force_reg ( V32QImode , vperm ) ; l = gen_reg_rtx ( V32QImode ) ; op = gen_lowpart ( V32QImode , d -> op0 ) ; emit_insn ( gen_avx2_pshufbv32qi3 ( l , op , vperm ) ) ; op = d -> target ; if ( d -> vmode != V32QImode ) op = gen_reg_rtx ( V32QImode ) ; emit_insn ( gen_iorv32qi3 ( op , l , gen_lowpart ( V32QImode , hp ) ) ) ; if ( op != d -> target ) emit_move_insn ( d -> target , gen_lowpart ( d -> vmode , op ) ) ; return true ; } </s>

Implement arbitrary permutation of one V32QImode and V16QImode operand with two vpshufb insns , vpermq and vpor . We should have already failed all two or three instruction sequences .


A subroutine of expand_vec_perm_even_odd_1 . Implement extract-even and extract-odd permutations of two V32QImode and V16QImode operand with two vpshufb insns , vpor and vpermq . We should have already failed all two or three instruction sequences .