<s> static void register_builtin_types ( ) { scalar_types [ VECTOR_TYPE_ ## ACLE_NAME ] = SCALAR_TYPE ; for ( unsigned int i = 0 ; i < NUM_VECTOR_TYPES ; ++ i ) { tree eltype = scalar_types [ i ] ; tree vectype ; unsigned int num_zr = 0 , num_pr = 0 ; if ( eltype == boolean_type_node ) { vectype = build_truth_vector_type_for_mode ( BYTES_PER_SVE_VECTOR , VNx16BImode ) ; gcc_assert ( TYPE_MODE ( vectype ) == VNx16BImode && TYPE_MODE ( vectype ) == TYPE_MODE_RAW ( vectype ) && TYPE_ALIGN ( vectype ) == 16 && known_eq ( wi :: to_poly_offset ( TYPE_SIZE ( vectype ) ) , BYTES_PER_SVE_VECTOR ) ) ; num_pr = 1 ; } else { scalar_mode elmode = SCALAR_TYPE_MODE ( eltype ) ; unsigned int elbytes = GET_MODE_SIZE ( elmode ) ; poly_uint64 nunits = exact_div ( BYTES_PER_SVE_VECTOR , elbytes ) ; machine_mode mode = aarch64_sve_data_mode ( elmode , nunits ) . require ( ) ; vectype = build_vector_type_for_mode ( eltype , mode ) ; gcc_assert ( VECTOR_MODE_P ( TYPE_MODE ( vectype ) ) && TYPE_MODE ( vectype ) == mode && TYPE_MODE_RAW ( vectype ) == mode && TYPE_ALIGN ( vectype ) == 128 && known_eq ( wi :: to_poly_offset ( TYPE_SIZE ( vectype ) ) , BITS_PER_SVE_VECTOR ) ) ; num_zr = 1 ; } vectype = build_distinct_type_copy ( vectype ) ; gcc_assert ( vectype == TYPE_MAIN_VARIANT ( vectype ) ) ; SET_TYPE_STRUCTURAL_EQUALITY ( vectype ) ; TYPE_ARTIFICIAL ( vectype ) = 1 ; TYPE_INDIVISIBLE_P ( vectype ) = 1 ; add_sve_type_attribute ( vectype , num_zr , num_pr , vector_types [ i ] . mangled_name , vector_types [ i ] . acle_name ) ; make_type_sizeless ( vectype ) ; abi_vector_types [ i ] = vectype ; lang_hooks . types . register_builtin_type ( vectype , vector_types [ i ] . abi_name ) ; } } </s>

Register the built-in RVV ABI types , such as __rvv_int32m1_t .


<s> static void register_tuple_type ( unsigned int num_vectors , vector_type_index type ) { tree tuple_type = lang_hooks . types . make_type ( RECORD_TYPE ) ; char buffer [ sizeof ( "svbfloat16x4_t" ) ] ; const char * vector_type_name = vector_types [ type ] . acle_name ; snprintf ( buffer , sizeof ( buffer ) , "%.*sx%d_t" , ( int ) strlen ( vector_type_name ) - 2 , vector_type_name , num_vectors ) ; tree vector_type = acle_vector_types [ 0 ] [ type ] ; tree array_type = build_array_type_nelts ( vector_type , num_vectors ) ; gcc_assert ( VECTOR_MODE_P ( TYPE_MODE ( array_type ) ) && TYPE_MODE_RAW ( array_type ) == TYPE_MODE ( array_type ) && TYPE_ALIGN ( array_type ) == 128 ) ; tree field = build_decl ( input_location , FIELD_DECL , get_identifier ( "__val" ) , array_type ) ; DECL_FIELD_CONTEXT ( field ) = tuple_type ; TYPE_FIELDS ( tuple_type ) = field ; add_sve_type_attribute ( tuple_type , num_vectors , 0 , NULL , buffer ) ; make_type_sizeless ( tuple_type ) ; layout_type ( tuple_type ) ; gcc_assert ( VECTOR_MODE_P ( TYPE_MODE ( tuple_type ) ) && TYPE_MODE_RAW ( tuple_type ) == TYPE_MODE ( tuple_type ) && TYPE_ALIGN ( tuple_type ) == 128 ) ; tree decl = build_decl ( input_location , TYPE_DECL , get_identifier ( buffer ) , tuple_type ) ; TYPE_NAME ( tuple_type ) = decl ; TYPE_STUB_DECL ( tuple_type ) = decl ; lang_hooks . decls . pushdecl ( decl ) ; DECL_ORIGINAL_TYPE ( decl ) = NULL_TREE ; acle_vector_types [ num_vectors - 1 ] [ type ] = tuple_type ; } </s>

Register the tuple type that contains NUM_VECTORS vectors of type TYPE .


"passing %qT to argument %d of %qE, which"

Require argument ARGNO to have vector type TYPE , in cases where this requirement holds for all uses of the function . Return true if the argument has the right form , otherwise report an appropriate error .


" expects a single SVE vector rather than a tuple"

"passing single vector %qT to argument %d"

" of %qE, which expects a tuple of %d vectors"

Require argument ARGNO to be a single vector or a tuple of NUM_VECTORS vectors ; NUM_VECTORS is 1 for the former . Return the associated type suffix on success , using TYPE_SUFFIX_b for predicates . Report an error and return NUM_TYPE_SUFFIXES on failure .


<s> static void register_svpattern ( ) { auto_vec < string_int_pair , 32 > values ; values . quick_push ( string_int_pair ( "SV_" # UPPER , VALUE ) ) ; AARCH64_FOR_SVPATTERN ( PUSH ) acle_svpattern = lang_hooks . types . simulate_enum_decl ( input_location , "svpattern" , & values ) ; } </s>


<s> static void register_svprfop ( ) { auto_vec < string_int_pair , 16 > values ; values . quick_push ( string_int_pair ( "SV_" # UPPER , VALUE ) ) ; AARCH64_FOR_SVPRFOP ( PUSH ) acle_svprfop = lang_hooks . types . simulate_enum_decl ( input_location , "svprfop" , & values ) ; } </s>


<s> static void register_vector_type ( vector_type_index type ) { tree vectype = abi_vector_types [ type ] ; tree id = get_identifier ( vector_types [ type ] . acle_name ) ; tree decl = build_decl ( input_location , TYPE_DECL , id , vectype ) ; decl = lang_hooks . decls . pushdecl ( decl ) ; if ( decl && TREE_CODE ( decl ) == TYPE_DECL && TREE_TYPE ( decl ) != error_mark_node && TYPE_MAIN_VARIANT ( TREE_TYPE ( decl ) ) == vectype ) vectype = TREE_TYPE ( decl ) ; acle_vector_types [ 0 ] [ type ] = vectype ; } </s>

Register vector type TYPE under its risv_vector.h name .


<s> registered_function & function_builder :: add_function ( const function_instance & instance , const char * name , tree fntype , tree attrs , aarch64_feature_flags required_extensions , bool overloaded_p , bool placeholder_p ) { unsigned int code = vec_safe_length ( registered_functions ) ; code = ( code << AARCH64_BUILTIN_SHIFT ) | AARCH64_BUILTIN_SVE ; tree decl = placeholder_p ? integer_zero_node : simulate_builtin_function_decl ( input_location , name , fntype , code , NULL , attrs ) ; registered_function & rfn = * ggc_alloc < registered_function > ( ) ; rfn . instance = instance ; rfn . decl = decl ; rfn . required_extensions = required_extensions ; rfn . overloaded_p = overloaded_p ; vec_safe_push ( registered_functions , & rfn ) ; return rfn ; } </s>

Add a function called NAME with type FNTYPE and attributes ATTRS . INSTANCE describes what the function does .


<s> void function_builder :: add_overloaded_function ( const function_instance & instance , aarch64_feature_flags required_extensions ) { char * name = get_name ( instance , true ) ; if ( registered_function * * map_value = m_overload_names . get ( name ) ) { gcc_assert ( ( * map_value ) -> instance == instance && ( ( * map_value ) -> required_extensions & ~ required_extensions ) == 0 ) ; obstack_free ( & m_string_obstack , name ) ; } else { registered_function & rfn = add_function ( instance , name , m_overload_type , NULL_TREE , required_extensions , true , m_direct_overloads ) ; m_overload_names . put ( name , & rfn ) ; } } </s>

Add one function decl for INSTANCE , to be used with manual overload resolution . REQUIRED_EXTENSIONS are the set of architecture extensions that the function requires . For simplicity , deal with duplicate attempts to add the same function , including cases in which the new function requires more features than the original one did . In that case we 'll check whether the required features are available as part of resolving the function to the relevant unique function .


<s> void function_builder :: add_unique_function ( const function_instance & instance , tree return_type , vec < tree > & argument_types , aarch64_feature_flags required_extensions , bool force_direct_overloads ) { char * name = get_name ( instance , false ) ; tree fntype = build_function_type_array ( return_type , argument_types . length ( ) , argument_types . address ( ) ) ; tree attrs = get_attributes ( instance ) ; registered_function & rfn = add_function ( instance , name , fntype , attrs , required_extensions , false , false ) ; hashval_t hash = instance . hash ( ) ; registered_function * * rfn_slot = function_table -> find_slot_with_hash ( instance , hash , INSERT ) ; gcc_assert ( ! * rfn_slot ) ; * rfn_slot = & rfn ; char * overload_name = get_name ( instance , true ) ; if ( strcmp ( name , overload_name ) != 0 ) { tree attrs = get_attributes ( instance ) ; bool placeholder_p = ! ( m_direct_overloads || force_direct_overloads ) ; add_function ( instance , overload_name , fntype , attrs , required_extensions , false , placeholder_p ) ; } obstack_free ( & m_string_obstack , name ) ; } </s>

Add a built-in function for INSTANCE , with the argument types given by ARGUMENT_TYPES and the return type given by RETURN_TYPE . NAME is the `` full '' name for C function . OVERLOAD_NAME is the `` short '' name for C++ overloaded function . OVERLOAD_NAME can be nullptr because some instance does n't have C++ overloaded function .


<s> static bool check_required_extensions ( location_t location , tree fndecl , aarch64_feature_flags required_extensions ) { auto missing_extensions = required_extensions & ~ aarch64_asm_isa_flags ; if ( missing_extensions == 0 ) return check_required_registers ( location , fndecl ) ; static const struct { aarch64_feature_flags flag ; const char * name ; } extensions [ ] = { { AARCH64_FL_ ## IDENT , EXT_NAME } , } ; for ( unsigned int i = 0 ; i < ARRAY_SIZE ( extensions ) ; ++ i ) if ( missing_extensions & extensions [ i ] . flag ) { report_missing_extension ( location , fndecl , extensions [ i ] . name ) ; return false ; } gcc_unreachable ( ) ; } </s>

Check whether all the RVV_REQUIRE_ * values in REQUIRED_EXTENSIONS are enabled .


<s> gimple * gimple_folder :: fold_to_cstu ( poly_uint64 val ) { return gimple_build_assign ( lhs , build_int_cstu ( TREE_TYPE ( lhs ) , val ) ) ; } </s>


<s> void function_expander :: add_address_operand ( rtx x ) { m_ops . safe_grow ( m_ops . length ( ) + 1 ) ; create_address_operand ( & m_ops . last ( ) , x ) ; } </s>

Add an address operand with value X . The static operand data says what mode and form the address must have .


<s> static tree add_attribute ( const char * name , tree attrs ) { return tree_cons ( get_identifier ( name ) , NULL_TREE , attrs ) ; } </s>


<s> void function_expander :: add_fixed_operand ( rtx x ) { m_ops . safe_grow ( m_ops . length ( ) + 1 ) ; create_fixed_operand ( & m_ops . last ( ) , x ) ; } </s>

Add an operand that must be X . The only way of legitimizing an invalid X is to reload the address of a MEM .


<s> registered_function & function_builder :: add_function ( const function_instance & instance , const char * name , tree fntype , tree attrs , uint64_t required_extensions , bool overloaded_p ) { unsigned int code = vec_safe_length ( registered_functions ) ; code = ( code << AARCH64_BUILTIN_SHIFT ) | AARCH64_BUILTIN_SVE ; tree decl = simulate_builtin_function_decl ( input_location , name , fntype , code , NULL , attrs ) ; registered_function & rfn = * ggc_alloc < registered_function > ( ) ; rfn . instance = instance ; rfn . decl = decl ; rfn . required_extensions = required_extensions ; rfn . overloaded_p = overloaded_p ; vec_safe_push ( registered_functions , & rfn ) ; return rfn ; } </s>


<s> void function_expander :: add_input_operand ( insn_code icode , rtx x ) { unsigned int opno = m_ops . length ( ) ; const insn_operand_data & operand = insn_data [ icode ] . operand [ opno ] ; machine_mode mode = operand . mode ; if ( mode == VOIDmode ) { gcc_assert ( operand . predicate == aarch64_any_register_operand ) ; mode = GET_MODE ( x ) ; } else if ( ! VECTOR_MODE_P ( GET_MODE ( x ) ) && VECTOR_MODE_P ( mode ) ) x = expand_vector_broadcast ( mode , x ) ; else if ( GET_MODE_CLASS ( mode ) == MODE_VECTOR_BOOL ) { gcc_assert ( GET_MODE ( x ) == VNx16BImode ) ; x = gen_lowpart ( mode , x ) ; } m_ops . safe_grow ( m_ops . length ( ) + 1 ) ; create_input_operand ( & m_ops . last ( ) , x , mode ) ; } </s>

Create op and add it into M_OPS and increase OPNO .


<s> void function_expander :: add_integer_operand ( HOST_WIDE_INT x ) { m_ops . safe_grow ( m_ops . length ( ) + 1 ) ; create_integer_operand ( & m_ops . last ( ) , x ) ; } </s>


<s> void function_expander :: add_mem_operand ( machine_mode mode , rtx addr ) { gcc_assert ( VECTOR_MODE_P ( mode ) || mode == OImode ) ; rtx mem = gen_rtx_MEM ( mode , memory_address ( mode , addr ) ) ; set_mem_align ( mem , GET_MODE_ALIGNMENT ( GET_MODE_INNER ( mode ) ) ) ; add_fixed_operand ( mem ) ; } </s>

Add a memory operand with mode MODE and address ADDR .


<s> void function_expander :: add_output_operand ( insn_code icode ) { unsigned int opno = m_ops . length ( ) ; machine_mode mode = insn_data [ icode ] . operand [ opno ] . mode ; m_ops . safe_grow ( opno + 1 ) ; create_output_operand ( & m_ops . last ( ) , possible_target , mode ) ; } </s>

Create output and add it into M_OPS and increase OPNO .


<s> void function_builder :: add_overloaded_function ( const function_instance & instance , uint64_t required_extensions ) { char * name = get_name ( instance , true ) ; if ( registered_function * * map_value = m_overload_names . get ( name ) ) gcc_assert ( ( * map_value ) -> instance == instance && ( ( * map_value ) -> required_extensions & ~ required_extensions ) == 0 ) ; else { registered_function & rfn = add_function ( instance , name , m_overload_type , NULL_TREE , required_extensions , true ) ; const char * permanent_name = IDENTIFIER_POINTER ( DECL_NAME ( rfn . decl ) ) ; m_overload_names . put ( permanent_name , & rfn ) ; } obstack_free ( & m_string_obstack , name ) ; } </s>


<s> void function_builder :: add_overloaded_functions ( const function_group_info & group , mode_suffix_index mode ) { if ( m_direct_overloads ) return ; unsigned int explicit_type0 = ( * group . shape ) -> explicit_type_suffix_p ( 0 ) ; unsigned int explicit_type1 = ( * group . shape ) -> explicit_type_suffix_p ( 1 ) ; for ( unsigned int pi = 0 ; group . preds [ pi ] != NUM_PREDS ; ++ pi ) { if ( ! explicit_type0 && ! explicit_type1 ) { function_instance instance ( group . base_name , * group . base , * group . shape , mode , types_none [ 0 ] , group . preds [ pi ] ) ; add_overloaded_function ( instance , group . required_extensions ) ; } else for ( unsigned int ti = 0 ; group . types [ ti ] [ 0 ] != NUM_TYPE_SUFFIXES ; ++ ti ) { type_suffix_pair types = { explicit_type0 ? group . types [ ti ] [ 0 ] : NUM_TYPE_SUFFIXES , explicit_type1 ? group . types [ ti ] [ 1 ] : NUM_TYPE_SUFFIXES } ; function_instance instance ( group . base_name , * group . base , * group . shape , mode , types , group . preds [ pi ] ) ; add_overloaded_function ( instance , group . required_extensions ) ; } } } </s>

If we are using manual overload resolution , add one function decl for each overloaded function in GROUP . Take the function base name from GROUP and the mode from MODE .


<s> void function_expander :: add_ptrue_hint ( unsigned int argno , machine_mode pred_mode ) { rtx pred = gen_lowpart ( pred_mode , args [ argno ] ) ; int hint = ( pred == CONSTM1_RTX ( pred_mode ) ? SVE_KNOWN_PTRUE : SVE_MAYBE_NOT_PTRUE ) ; args . quick_insert ( argno + 1 , gen_int_mode ( hint , SImode ) ) ; } </s>

Add a dummy argument to indicate whether predicate argument ARGNO is all-true when interpreted in mode PRED_MODE . The hint goes immediately after ARGNO .


<s>
static
void
add_sve_type_attribute
(
tree
type
,
unsigned
int
num_zr
,
unsigned
int
num_pr
,
const
char
*
mangled_name
)
{
tree
mangled_name_tree
=
(
mangled_name
get_identifier
(
mangled_name
)
:
NULL_TREE
)
;
tree
value
=
tree_cons
(
NULL_TREE
,
mangled_name_tree
,
NULL_TREE
)
;
value
=
tree_cons
(
NULL_TREE
,
size_int
(
num_pr
)
,
value
)
;
value
=
tree_cons
(
NULL_TREE
,
size_int
(
num_zr
)
,
value
)
;
TYPE_ATTRIBUTES
(
type
)
=
tree_cons
(
get_identifier
(
"SVE type"
)
,
value
,
TYPE_ATTRIBUTES
(
type
)
)
;
}
</s>

Record that TYPE is an ABI-defined SVE type that contains NUM_ZR SVE vectors and NUM_PR SVE predicates . MANGLED_NAME , if nonnull , is the ABI-defined mangling of the type . ACLE_NAME is the < arm_sve.h > name of the type .


<s> void function_builder :: add_unique_function ( const function_instance & instance , tree return_type , vec < tree > & argument_types , uint64_t required_extensions , bool force_direct_overloads ) { char * name = get_name ( instance , false ) ; tree fntype = build_function_type_array ( return_type , argument_types . length ( ) , argument_types . address ( ) ) ; tree attrs = get_attributes ( instance ) ; registered_function & rfn = add_function ( instance , name , fntype , attrs , required_extensions , false ) ; hashval_t hash = instance . hash ( ) ; registered_function * * rfn_slot = function_table -> find_slot_with_hash ( instance , hash , INSERT ) ; gcc_assert ( ! * rfn_slot ) ; * rfn_slot = & rfn ; if ( m_direct_overloads || force_direct_overloads ) { char * overload_name = get_name ( instance , true ) ; if ( strcmp ( name , overload_name ) != 0 ) { tree attrs = get_attributes ( instance ) ; add_function ( instance , overload_name , fntype , attrs , required_extensions , false ) ; } } obstack_free ( & m_string_obstack , name ) ; } </s>


<s> void function_builder :: append_name ( const char * name ) { obstack_grow ( & m_string_obstack , name , strlen ( name ) ) ; } </s>

Add NAME to the end of the function name being built .


<s> bool function_checker :: argument_exists_p ( unsigned int argno ) { gcc_assert ( argno < ( unsigned int ) type_num_arguments ( m_fntype ) ) ; return argno < m_nargs ; } </s>

Return true if argument ARGNO exists . which it might not for erroneous calls . It is safe to wave through checks if this function returns false .


<s> inline tree function_instance :: base_vector_type ( ) const { return acle_vector_types [ 0 ] [ mode_suffix ( ) . base_vector_type ] ; } </s>

Return the type of the function 's vector base address argument , or null it does n't have a vector base address .


<s> tree builtin_decl ( unsigned int code , bool ) { if ( code >= vec_safe_length ( registered_functions ) ) return error_mark_node ; return ( * registered_functions ) [ code ] -> decl ; } </s>

Return the function decl with RVV function subcode CODE , or error_mark_node if no such function exists .


<s> bool builtin_type_p ( const_tree type , unsigned int * num_zr , unsigned int * num_pr ) { if ( tree attr = lookup_sve_type_attribute ( type ) ) { tree num_zr_node = TREE_VALUE ( attr ) ; tree num_pr_node = TREE_CHAIN ( num_zr_node ) ; * num_zr = tree_to_uhwi ( TREE_VALUE ( num_zr_node ) ) ; * num_pr = tree_to_uhwi ( TREE_VALUE ( num_pr_node ) ) ; return true ; } return false ; } </s>

Return true if TYPE is a built-in SVE type defined by the ABI or ACLE . If so , store the number of constituent SVE vectors in * NUM_ZR and the number of constituent SVE predicates in * NUM_PR .


<s> inline unsigned int function_base :: call_properties ( const function_instance & instance ) const { unsigned int flags = 0 ; if ( instance . type_suffix ( 0 ) . float_p || instance . type_suffix ( 1 ) . float_p ) flags |= CP_READ_FPCR | CP_RAISE_FP_EXCEPTIONS ; return flags ; } </s>

Default implementation of function_base : :call_properties , with conservatively correct behavior for floating-point instructions .


<s> virtual bool check ( function_checker & ) const { return true ; } </s>

Perform semantic checks on the call . Return true if the call is valid , otherwise report a suitable error .


<s> bool check_builtin_call ( location_t location , vec < location_t > , unsigned int code , tree fndecl , unsigned int nargs , tree * args ) { const registered_function & rfn = * ( * registered_functions ) [ code ] ; if ( ! check_required_extensions ( location , rfn . decl , rfn . required_extensions ) ) return false ; return function_checker ( location , rfn . instance , fndecl , TREE_TYPE ( rfn . decl ) , nargs , args ) . check ( ) ; } </s>

Perform any semantic checks needed for a call to the SVE function with subcode CODE , such as testing for integer constant expressions . The call occurs at location LOCATION and has NARGS arguments , given by ARGS . FNDECL is the original function decl , before overload resolution . Return true if the call is valid , otherwise report a suitable error .


<s> bool function_resolver :: check_gp_argument ( unsigned int nops , unsigned int & i , unsigned int & nargs ) { i = 0 ; if ( pred != PRED_none ) { gcc_assert ( nops != 1 || pred != PRED_m ) ; nargs = nops + 1 ; if ( ! check_num_arguments ( nargs ) || ! require_vector_type ( i , VECTOR_TYPE_svbool_t ) ) return false ; i += 1 ; } else { nargs = nops ; if ( ! check_num_arguments ( nargs ) ) return false ; } return true ; } </s>

If the function is predicated , check that the first argument is a suitable governing predicate . Also check that there are NOPS further arguments after any governing predicate , but do n't check what they are . Return true on success , otherwise report a suitable error . When returning true : - set I to the number of the first unchecked argument . - set NARGS to the total number of arguments .


<s>
bool
function_resolver
::
check_num_arguments
(
unsigned
int
expected
)
{
if
(
m_arglist
.
length
(
)
<
expected
)
error_at
(
location
,
"too few arguments to function %qE"
,
fndecl
)
;
else
if
(
m_arglist
.
length
(
)
>
expected
)
error_at
(
location
,
"too many arguments to function %qE"
,
fndecl
)
;
return
m_arglist
.
length
(
)
==
expected
;
}
</s>

Require the function to have exactly EXPECTED arguments . Return true if it does , otherwise report an appropriate error .

aarch64
"too few arguments to function %qE"
"too many arguments to function %qE"


<s> static bool check_required_extensions ( location_t location , tree fndecl , uint64_t required_extensions ) { uint64_t missing_extensions = required_extensions & ~ aarch64_isa_flags ; if ( missing_extensions == 0 ) return check_required_registers ( location , fndecl ) ; static const struct { uint64_t flag ; const char * name ; } extensions [ ] = { SYNTHETIC , FEATURE_STRING ) \ { FLAG_CANONICAL , EXT_NAME } , } ; for ( unsigned int i = 0 ; i < ARRAY_SIZE ( extensions ) ; ++ i ) if ( missing_extensions & extensions [ i ] . flag ) { report_missing_extension ( location , fndecl , extensions [ i ] . name ) ; return false ; } gcc_unreachable ( ) ; } </s>

Check whether all the RVV_REQUIRE_ * values in REQUIRED_EXTENSIONS are enabled .


"ACLE function %qD is incompatible with the use of %qs"

Check whether the registers required by SVE function fndecl are available . Report an error against LOCATION and return false if not .


<s> tree gimple_folder :: convert_pred ( gimple_seq & stmts , tree vectype , unsigned int argno ) { tree pred = gimple_call_arg ( call , argno ) ; if ( known_eq ( TYPE_VECTOR_SUBPARTS ( TREE_TYPE ( pred ) ) , TYPE_VECTOR_SUBPARTS ( vectype ) ) ) return pred ; return gimple_build ( & stmts , VIEW_CONVERT_EXPR , truth_type_for ( vectype ) , pred ) ; } </s>

Convert predicate argument ARGNO so that it has the type appropriate for an operation on VECTYPE . Add any new statements to STMTS .


<s> rtx function_expander :: convert_to_pmode ( rtx x ) { if ( ptr_mode == SImode ) x = simplify_gen_unary ( ZERO_EXTEND , DImode , x , SImode ) ; return x ; } </s>


<s> bool function_instance :: could_trap_p ( ) const { unsigned int flags = call_properties ( ) ; if ( flags & CP_RAISE_FP_EXCEPTIONS ) return true ; if ( flags & ( CP_READ_MEMORY | CP_WRITE_MEMORY ) ) return true ; return false ; } </s>

Return true if calls to the function could raise a signal .


<s> insn_code function_expander :: direct_optab_handler ( optab op , unsigned int suffix_i ) { return :: direct_optab_handler ( op , vector_mode ( suffix_i ) ) ; } </s>

Return the handler of direct optab OP for type suffix SUFFIX_I .


<s> insn_code function_expander :: direct_optab_handler_for_sign ( optab signed_op , optab unsigned_op , unsigned int suffix_i , machine_mode mode ) { if ( mode == VOIDmode ) mode = vector_mode ( suffix_i ) ; optab op = type_suffix ( suffix_i ) . unsigned_p ? unsigned_op : signed_op ; return :: direct_optab_handler ( op , mode ) ; } </s>

Choose between signed and unsigned direct optabs SIGNED_OP and UNSIGNED_OP based on the signedness of type suffix SUFFIX_I , then pick the appropriate optab handler for the mode . Use MODE as the mode if given , otherwise use the mode of type suffix SUFFIX_I .


<s> inline units_index function_instance :: displacement_units ( ) const { return mode_suffix ( ) . displacement_units ; } </s>

If the function takes a vector or scalar displacement , return the units in which the displacement is measured , otherwise return UNITS_none .


<s> inline tree function_instance :: displacement_vector_type ( ) const { return acle_vector_types [ 0 ] [ mode_suffix ( ) . displacement_vector_type ] ; } </s>

Return the type of the function 's vector index or offset argument , or null if does n't have a vector index or offset argument .


<s> inline unsigned int function_instance :: elements_per_vq ( unsigned int i ) const { return 128 / type_suffix ( i ) . element_bits ; } </s>

Return the number of elements of type suffix I that fit within a 128-bit block .


<s> inline bool registered_function_hasher :: equal ( value_type value , const compare_type & key ) { return value -> instance == key ; } </s>


<s> rtx function_expander :: expand ( ) { unsigned int nargs = call_expr_nargs ( call_expr ) ; args . reserve ( nargs ) ; for ( unsigned int i = 0 ; i < nargs ; ++ i ) args . quick_push ( expand_normal ( CALL_EXPR_ARG ( call_expr , i ) ) ) ; return base -> expand ( * this ) ; } </s>


<s> rtx expand_builtin ( unsigned int code , tree exp , rtx target ) { registered_function & rfn = * ( * registered_functions ) [ code ] ; if ( ! check_required_extensions ( EXPR_LOCATION ( exp ) , rfn . decl , rfn . required_extensions ) ) return target ; return function_expander ( rfn . instance , rfn . decl , exp , target ) . expand ( ) ; } </s>

Expand a call to the RVV function with subcode CODE . EXP is the call expression and TARGET is the preferred location for the result . Return the value of the lhs .


Try to find a mode with the given mode_suffix_info fields . Return the mode on success or MODE_none on failure .


<s> inline type_suffix_index find_type_suffix ( type_class_index tclass , unsigned int element_bits ) { for ( unsigned int i = 0 ; i < NUM_TYPE_SUFFIXES ; ++ i ) if ( type_suffixes [ i ] . tclass == tclass && type_suffixes [ i ] . element_bits == element_bits ) return type_suffix_index ( i ) ; gcc_unreachable ( ) ; } </s>

Return the type suffix associated with ELEMENT_BITS-bit elements of type class TCLASS .


<s> static type_suffix_index find_type_suffix_for_scalar_type ( const_tree type ) { for ( unsigned int suffix_i = 0 ; suffix_i < NUM_TYPE_SUFFIXES ; ++ suffix_i ) if ( ! type_suffixes [ suffix_i ] . bool_p ) { vector_type_index vector_i = type_suffixes [ suffix_i ] . vector_type ; if ( matches_type_p ( scalar_types [ vector_i ] , type ) ) return type_suffix_index ( suffix_i ) ; } return NUM_TYPE_SUFFIXES ; } </s>

If TYPE is a valid SVE element type , return the corresponding type suffix , otherwise return NUM_TYPE_SUFFIXES .


<s> char * function_builder :: finish_name ( ) { obstack_1grow ( & m_string_obstack , 0 ) ; return ( char * ) obstack_finish ( & m_string_obstack ) ; } </s>

Zero-terminate and complete the function name being built .


"passing %qT to argument %d of %qE, but its"

Finish resolving a function whose final argument can be a vector or a scalar , with the function having an implicit `` _n '' suffix in the latter case . This `` _n '' form might only exist for certain type suffixes . ARGNO is the index of the final argument . The inferred type suffix was obtained from argument FIRST_ARGNO , which has type FIRST_TYPE . EXPECTED_TCLASS and EXPECTED_BITS describe the expected properties of the final vector or scalar argument , in the same way as for require_derived_vector_type . INFERRED_TYPE is the inferred type suffix itself , or NUM_TYPE_SUFFIXES if it 's the same as FIRST_TYPE . Return the function decl of the resolved function on success , otherwise report a suitable error and return error_mark_node .


<s> virtual gimple * fold ( gimple_folder & ) const { return NULL ; } </s>

Try to fold the call . Return the new statement on success and null on failure .


<s> tree gimple_folder :: fold_contiguous_base ( gimple_seq & stmts , tree vectype ) { tree base = gimple_call_arg ( call , 1 ) ; if ( mode_suffix_id == MODE_vnum ) { tree offset = gimple_call_arg ( call , 2 ) ; offset = gimple_convert ( & stmts , sizetype , offset ) ; offset = gimple_build ( & stmts , MULT_EXPR , sizetype , offset , TYPE_SIZE_UNIT ( vectype ) ) ; base = gimple_build ( & stmts , POINTER_PLUS_EXPR , TREE_TYPE ( base ) , base , offset ) ; } return base ; } </s>

Return a pointer to the address in a contiguous load or store , given that each memory vector has type VECTYPE . Add any new statements to STMTS .


<s> gimple * gimple_folder :: fold_to_pfalse ( ) { return gimple_build_assign ( lhs , build_zero_cst ( TREE_TYPE ( lhs ) ) ) ; } </s>


<s> gimple * gimple_folder :: fold_to_ptrue ( ) { tree svbool_type = TREE_TYPE ( lhs ) ; tree bool_type = TREE_TYPE ( svbool_type ) ; unsigned int element_bytes = type_suffix ( 0 ) . element_bytes ; tree_vector_builder builder ( svbool_type , element_bytes , 1 ) ; builder . quick_push ( build_all_ones_cst ( bool_type ) ) ; for ( unsigned int i = 1 ; i < element_bytes ; ++ i ) builder . quick_push ( build_zero_cst ( bool_type ) ) ; return gimple_build_assign ( lhs , builder . build ( ) ) ; } </s>

Fold the call to a PTRUE , taking the element size from type suffix 0 .


<s> gimple * gimple_folder :: fold_to_vl_pred ( unsigned int vl ) { tree vectype = TREE_TYPE ( lhs ) ; tree element_type = TREE_TYPE ( vectype ) ; tree minus_one = build_all_ones_cst ( element_type ) ; tree zero = build_zero_cst ( element_type ) ; unsigned int element_bytes = type_suffix ( 0 ) . element_bytes ; unsigned int count = constant_lower_bound ( TYPE_VECTOR_SUBPARTS ( vectype ) ) ; gcc_assert ( vl * element_bytes <= count ) ; tree_vector_builder builder ( vectype , count , 2 ) ; for ( unsigned int i = 0 ; i < count * 2 ; ++ i ) { bool bit = ( i & ( element_bytes - 1 ) ) == 0 && i < vl * element_bytes ; builder . quick_push ( bit ? minus_one : zero ) ; } return gimple_build_assign ( lhs , builder . build ( ) ) ; } </s>

Fold an operation to a constant predicate in which the first VL elements are set and the rest are clear . Take the element size from type suffix 0 .


<s> tree gimple_folder :: force_vector ( gimple_seq & stmts , tree vectype , tree value ) { if ( ! VECTOR_TYPE_P ( TREE_TYPE ( value ) ) ) value = gimple_build_vector_from_val ( & stmts , vectype , value ) ; return value ; } </s>

VALUE might be a vector of type VECTYPE or a single scalar element . Duplicate it into a vector of type VECTYPE in the latter case , adding any new statements to STMTS .


<s> inline bool function_call_info :: function_returns_void_p ( ) { return TREE_TYPE ( TREE_TYPE ( fndecl ) ) == void_type_node ; } </s>

Return true if the function has no return value .


<s> rtx function_expander :: generate_insn ( insn_code icode ) { expand_insn ( icode , m_ops . length ( ) , m_ops . address ( ) ) ; return function_returns_void_p ( ) ? const0_rtx : m_ops [ 0 ] . value ; } </s>

Generate instruction ICODE , given that its operands have already been added to M_OPS . Return the value of the first operand .


<s> tree function_resolver :: get_argument_type ( unsigned int i ) { tree arg = m_arglist [ i ] ; return arg == error_mark_node ? arg : TREE_TYPE ( arg ) ; } </s>

Return the type of argument I , or error_mark_node if it is n't well-formed .


<s> tree function_builder :: get_attributes ( const function_instance & instance ) { tree attrs = NULL_TREE ; if ( ! instance . modifies_global_state_p ( ) ) { if ( instance . reads_global_state_p ( ) ) attrs = add_attribute ( "pure" , attrs ) ; else attrs = add_attribute ( "const" , attrs ) ; } if ( ! flag_non_call_exceptions || ! instance . could_trap_p ( ) ) attrs = add_attribute ( "nothrow" , attrs ) ; return add_attribute ( "leaf" , attrs ) ; } </s>

Return the appropriate function attributes for INSTANCE .


<s> rtx function_expander :: get_contiguous_base ( machine_mode mem_mode ) { rtx base = convert_to_pmode ( args [ 1 ] ) ; if ( mode_suffix_id == MODE_vnum ) { poly_int64 size = ordered_min ( GET_MODE_SIZE ( mem_mode ) , BYTES_PER_SVE_VECTOR ) ; rtx offset = gen_int_mode ( size , Pmode ) ; offset = simplify_gen_binary ( MULT , Pmode , args [ 2 ] , offset ) ; base = simplify_gen_binary ( PLUS , Pmode , base , offset ) ; } return base ; } </s>

Return the base address for a contiguous load or store function . MEM_MODE is the mode of the addressed memory .


<s> char * function_builder :: get_name ( const function_instance & instance , bool overloaded_p ) { append_name ( instance . base_name ) ; if ( overloaded_p ) switch ( instance . displacement_units ( ) ) { case UNITS_none : break ; case UNITS_bytes : append_name ( "_offset" ) ; break ; case UNITS_elements : append_name ( "_index" ) ; break ; case UNITS_vectors : append_name ( "_vnum" ) ; break ; } else append_name ( instance . mode_suffix ( ) . string ) ; for ( unsigned int i = 0 ; i < 2 ; ++ i ) if ( ! overloaded_p || instance . shape -> explicit_type_suffix_p ( i ) ) append_name ( instance . type_suffix ( i ) . string ) ; append_name ( pred_suffixes [ instance . pred ] ) ; return finish_name ( ) ; } </s>

Return the overloaded or full function name for INSTANCE ; OVERLOADED_P selects which . Allocate the string on m_string_obstack ; the caller must use obstack_free to free it after use .


<s> rtx function_expander :: get_nonoverlapping_reg_target ( ) { if ( possible_target && overlaps_input_p ( possible_target ) ) possible_target = NULL_RTX ; return get_reg_target ( ) ; } </s>

As for get_reg_target , but make sure that the returned REG does not overlap any inputs .


<s> rtx function_expander :: get_reg_target ( ) { machine_mode target_mode = TYPE_MODE ( TREE_TYPE ( TREE_TYPE ( fndecl ) ) ) ; if ( ! possible_target || GET_MODE ( possible_target ) != target_mode ) possible_target = gen_reg_rtx ( target_mode ) ; return possible_target ; } </s>

Return a REG rtx that can be used for the result of the function , using the preferred target if suitable .


<s> const char * function_resolver :: get_scalar_type_name ( type_suffix_index type ) { return vector_types [ type_suffixes [ type ] . vector_type ] . acle_name + 2 ; } </s>

Return the < stdint.h > name associated with TYPE . Using the < stdint.h > name should be more user-friendly than the underlying canonical type , since it makes the signedness and bitwidth explicit .


<s> inline tree get_svbool_t ( void ) { return acle_vector_types [ 0 ] [ VECTOR_TYPE_svbool_t ] ; } </s>


<s> tree function_resolver :: get_vector_type ( type_suffix_index type ) { return acle_vector_types [ 0 ] [ type_suffixes [ type ] . vector_type ] ; } </s>

Return the vector type associated with type suffix TYPE .


<s> gimple * gimple_fold_builtin ( unsigned int code , gimple_stmt_iterator * gsi , gcall * stmt ) { registered_function & rfn = * ( * registered_functions ) [ code ] ; return gimple_folder ( rfn . instance , rfn . decl , gsi , stmt ) . fold ( ) ; } </s>

Attempt to fold STMT , given that it 's a call to the SVE function with subcode CODE . Return the new statement on success and null on failure . Insert any other new statements at GSI .


<s> inline machine_mode function_instance :: gp_mode ( unsigned int i ) const { return aarch64_sve_pred_mode ( type_suffix ( i ) . element_bytes ) . require ( ) ; } </s>

Return the mode of the governing predicate to use when operating on type suffix I .


<s>
void
handle_arm_sve_h
(
)
{
if
(
function_table
)
{
error
(
"duplicate definition of %qs"
,
"arm_sve.h"
)
;
return
;
}
sve_switcher
sve
;
for
(
unsigned
int
type_i
=
0
;
type_i
<
NUM_VECTOR_TYPES
;
++
type_i
)
{
vector_type_index
type
=
vector_type_index
(
type_i
)
;
register_vector_type
(
type
)
;
if
(
type
!=
VECTOR_TYPE_svbool_t
)
for
(
unsigned
int
count
=
2
;
count
<=
MAX_TUPLE_SIZE
;
++
count
)
register_tuple_type
(
count
,
type
)
;
}
register_svpattern
(
)
;
register_svprfop
(
)
;
function_table
=
new
hash_table
<
registered_function_hasher
>
(
1023
)
;
function_builder
builder
;
for
(
unsigned
int
i
=
0
;
i
<
ARRAY_SIZE
(
function_groups
)
;
++
i
)
builder
.
register_function_group
(
function_groups
[
i
]
)
;
}
</s>

Implement # pragma GCC aarch64 `` arm_sve.h '' .


"%qs requires an integer constant expression"

An attribute callback for the `` arm_sve_vector_bits '' attribute .


<s> inline hashval_t registered_function_hasher :: hash ( value_type value ) { return value -> instance . hash ( ) ; } </s>

Hash function for builtin functions with up to 3 arguments and a return type .


"passing %qT to argument %d of %qE, which expects"

Require argument ARGNO to be a 32-bit or 64-bit scalar integer type . Return the associated type suffix on success , otherwise report an error and return NUM_TYPE_SUFFIXES .


"passing %qT to argument %d of %qE, which"

Like infer_vector_type , but also require the type to be integral .


" when using a vector of base addresses"

"passing %qT to argument %d of %qE, but %qT is not"

" expects a pointer to 32-bit or 64-bit elements"

Require argument ARGNO to be a pointer to a scalar type that has a corresponding type suffix . Return that type suffix on success , otherwise report an error and return NUM_TYPE_SUFFIXES . GATHER_SCATTER_P is true if the function is a gather/scatter operation , and so requires a pointer to 32-bit or 64-bit data .


" expects a vector of 32-bit or 64-bit elements"

Like infer_vector_type , but also require the element size to be 32 or 64 bits .


<s> type_suffix_index function_resolver :: infer_tuple_type ( unsigned int argno ) { return infer_vector_or_tuple_type ( argno , vectors_per_tuple ( ) ) ; } </s>

If the function operates on tuples of vectors , require argument ARGNO to be a tuple with the appropriate number of vectors , otherwise require it to be a single vector . Return the associated type suffix on success , using TYPE_SUFFIX_b for predicates . Report an error and return NUM_TYPE_SUFFIXES on failure .


" expects a vector of unsigned integers"

Like infer_vector_type , but also require the type to be an unsigned integer .


Require argument ARGNO to be a vector base in a gather-style address . Return its type on success , otherwise return NUM_VECTOR_TYPES .


" expects a vector of 32-bit or 64-bit integers"

Require argument ARGNO to be a vector displacement in a gather-style address . Return its type on success , otherwise return NUM_VECTOR_TYPES .


" expects a single SVE vector rather than a tuple"

"passing single vector %qT to argument %d"

" of %qE, which expects a tuple of %d vectors"

Require argument ARGNO to be a single vector or a tuple of NUM_VECTORS vectors ; NUM_VECTORS is 1 for the former . Return the associated type suffix on success , using TYPE_SUFFIX_b for predicates . Report an error and return NUM_TYPE_SUFFIXES on failure .


<s> type_suffix_index function_resolver :: infer_vector_type ( unsigned int argno ) { return infer_vector_or_tuple_type ( argno , 1 ) ; } </s>

Require argument ARGNO to have some form of vector type . Return the associated type suffix on success , using TYPE_SUFFIX_b for predicates . Report an error and return NUM_TYPE_SUFFIXES on failure .


<s> void init_builtins ( ) { sve_switcher sve ; register_builtin_types ( ) ; if ( in_lto_p ) handle_arm_sve_h ( ) ; } </s>

Initialize all compiler built-ins related to RVV that should be defined at start-up .


<s> tree gimple_folder :: load_store_cookie ( tree type ) { return build_int_cst ( build_pointer_type ( type ) , TYPE_ALIGN_UNIT ( type ) ) ; } </s>

Return the alignment and TBAA argument to an internal load or store function like IFN_MASK_LOAD or IFN_MASK_STORE , given that it accesses memory elements of type TYPE .


<s> tree function_resolver :: lookup_form ( mode_suffix_index mode , type_suffix_index type0 , type_suffix_index type1 ) { type_suffix_pair types = { type0 , type1 } ; function_instance instance ( base_name , base , shape , mode , types , pred ) ; registered_function * rfn = function_table -> find_with_hash ( instance , instance . hash ( ) ) ; return rfn ? rfn -> decl : NULL_TREE ; } </s>

Silently check whether there is an instance of the function with the mode suffix given by MODE and the type suffixes given by TYPE0 and TYPE1 . Return its function decl if so , otherwise return null .


<s>
static
tree
lookup_sve_type_attribute
(
const_tree
type
)
{
if
(
type
==
error_mark_node
)
return
NULL_TREE
;
return
lookup_attribute
(
"SVE type"
,
TYPE_ATTRIBUTES
(
type
)
)
;
}
</s>

If TYPE is an ABI-defined SVE type , return its attribute descriptor , otherwise return null .


<s>
static
void
make_type_sizeless
(
tree
type
)
{
TYPE_ATTRIBUTES
(
type
)
=
tree_cons
(
get_identifier
(
"SVE sizeless type"
)
,
NULL_TREE
,
TYPE_ATTRIBUTES
(
type
)
)
;
}
</s>


<s> const char * mangle_builtin_type ( const_tree type ) { if ( TYPE_NAME ( type ) && TREE_CODE ( TYPE_NAME ( type ) ) == TYPE_DECL ) type = TREE_TYPE ( TYPE_NAME ( type ) ) ; if ( tree attr = lookup_sve_type_attribute ( type ) ) if ( tree id = TREE_VALUE ( chain_index ( 2 , TREE_VALUE ( attr ) ) ) ) return IDENTIFIER_POINTER ( id ) ; return NULL ; } </s>

If TYPE is a built-in type defined by the RVV ABI , return the mangled name , otherwise return NULL .


<s> rtx function_expander :: map_to_rtx_codes ( rtx_code code_for_sint , rtx_code code_for_uint , int unspec_for_fp , unsigned int merge_argno ) { machine_mode mode = vector_mode ( 0 ) ; rtx_code code = ( type_suffix ( 0 ) . unsigned_p ? code_for_uint : code_for_sint ) ; insn_code icode ; if ( type_suffix ( 0 ) . tclass == TYPE_bool ) { gcc_assert ( pred == PRED_z && code_for_uint == code_for_sint ) ; return use_exact_insn ( code_for_aarch64_pred_z ( code , mode ) ) ; } if ( pred == PRED_x ) { if ( type_suffix ( 0 ) . integer_p ) icode = maybe_code_for_aarch64_pred ( code , mode ) ; else icode = maybe_code_for_aarch64_pred ( unspec_for_fp , mode ) ; if ( icode != CODE_FOR_nothing ) return use_pred_x_insn ( icode ) ; } if ( pred == PRED_none || pred == PRED_x ) { icode = direct_optab_handler ( code_to_optab ( code ) , 0 ) ; if ( icode == CODE_FOR_nothing ) icode = code_for_aarch64_sve ( code , mode ) ; return use_unpred_insn ( icode ) ; } if ( type_suffix ( 0 ) . integer_p ) icode = code_for_cond ( code , mode ) ; else icode = code_for_cond ( unspec_for_fp , mode ) ; return use_cond_insn ( icode , merge_argno ) ; } </s>

Implement the call using one of the following strategies , chosen in order : ( 1 ) `` aarch64_pred_ < optab > < mode > _z '' for PRED_z predicate functions ( 2 ) `` aarch64_pred_ < optab > < mode > '' for PRED_x functions ( 3 ) a normal unpredicated optab for PRED_none and PRED_x functions , dropping the predicate in the latter case ( 4 ) an unpredicated `` aarch64_sve_ < code_optab > < mode > '' for PRED_none and PRED_x functions , again dropping the predicate for PRED_x ( 5 ) `` cond_ < optab > < mode > '' otherwise where < optab > corresponds to : - CODE_FOR_SINT for signed integers - CODE_FOR_UINT for unsigned integers - UNSPEC_FOR_FP for floating-point values and where < code_optab > is like < optab > , but uses CODE_FOR_SINT instead of UNSPEC_FOR_FP for floating-point values . MERGE_ARGNO is the argument that provides the values of inactive lanes for _m functions , or DEFAULT_MERGE_ARGNO if we should apply the usual rules .


<s> rtx function_expander :: map_to_unspecs ( int unspec_for_sint , int unspec_for_uint , int unspec_for_fp , unsigned int merge_argno ) { machine_mode mode = vector_mode ( 0 ) ; int unspec = ( ! type_suffix ( 0 ) . integer_p ? unspec_for_fp : type_suffix ( 0 ) . unsigned_p ? unspec_for_uint : unspec_for_sint ) ; if ( pred == PRED_x ) { insn_code icode = maybe_code_for_aarch64_pred ( unspec , mode ) ; if ( icode != CODE_FOR_nothing ) return use_pred_x_insn ( icode ) ; } if ( pred == PRED_none || pred == PRED_x ) { insn_code icode = maybe_code_for_aarch64_sve ( unspec , mode ) ; if ( icode != CODE_FOR_nothing ) return use_unpred_insn ( icode ) ; } insn_code icode = code_for_cond ( unspec , vector_mode ( 0 ) ) ; return use_cond_insn ( icode , merge_argno ) ; } </s>

Implement the call using one of the following strategies , chosen in order : ( 1 ) `` aarch64_pred_ < optab > < mode > '' for PRED_x functions ; this is a predicated pattern ( 2 ) `` aarch64_sve_ < optab > < mode > '' for PRED_none and PRED_x functions ; this is an unpredicated pattern ( 3 ) `` cond_ < optab > < mode > '' otherwise where < optab > corresponds to : - UNSPEC_FOR_SINT for signed integers - UNSPEC_FOR_UINT for unsigned integers - UNSPEC_FOR_FP for floating-point values MERGE_ARGNO is the argument that provides the values of inactive lanes for _m functions , or DEFAULT_MERGE_ARGNO if we should apply the usual rules .


<s> static bool matches_type_p ( const_tree model_type , const_tree candidate ) { if ( VECTOR_TYPE_P ( model_type ) ) { if ( ! VECTOR_TYPE_P ( candidate ) || maybe_ne ( TYPE_VECTOR_SUBPARTS ( model_type ) , TYPE_VECTOR_SUBPARTS ( candidate ) ) || TYPE_MODE ( model_type ) != TYPE_MODE ( candidate ) ) return false ; model_type = TREE_TYPE ( model_type ) ; candidate = TREE_TYPE ( candidate ) ; } return ( candidate != error_mark_node && TYPE_MAIN_VARIANT ( model_type ) == TYPE_MAIN_VARIANT ( candidate ) ) ; } </s>

Return true if CANDIDATE is equivalent to MODEL_TYPE for overloading purposes .


<s> inline tree function_instance :: memory_scalar_type ( ) const { return base -> memory_scalar_type ( * this ) ; } </s>

If the function addresses memory , return the type of a single scalar memory element .


<s> inline machine_mode function_instance :: memory_vector_mode ( ) const { return base -> memory_vector_mode ( * this ) ; } </s>

If the function addresses memory , return a vector mode whose GET_MODE_NUNITS is the number of elements addressed and whose GET_MODE_INNER is the mode of a single scalar memory element .


<s> inline const mode_suffix_info & function_instance :: mode_suffix ( ) const { return mode_suffixes [ mode_suffix_id ] ; } </s>

Return information about the function 's mode suffix .


<s> bool function_instance :: modifies_global_state_p ( ) const { unsigned int flags = call_properties ( ) ; if ( flags & CP_RAISE_FP_EXCEPTIONS ) return true ; if ( flags & CP_PREFETCH_MEMORY ) return true ; return flags & ( CP_WRITE_MEMORY | CP_WRITE_FFR ) ; } </s>

Return true if calls to the function could modify some form of global state .


<s> bool function_expander :: overlaps_input_p ( rtx x ) { for ( unsigned int i = 0 ; i < args . length ( ) ; ++ i ) if ( reg_overlap_mentioned_p ( x , args [ i ] ) ) return true ; return false ; } </s>


<s> void function_expander :: prepare_gather_address_operands ( unsigned int argno , bool scaled_p ) { machine_mode mem_mode = memory_vector_mode ( ) ; tree vector_type = base_vector_type ( ) ; units_index units = displacement_units ( ) ; int shift_idx = - 1 ; if ( units == UNITS_none ) { args . quick_insert ( argno , const0_rtx ) ; units = UNITS_bytes ; } else if ( vector_type ) { std :: swap ( args [ argno ] , args [ argno + 1 ] ) ; if ( units == UNITS_elements ) shift_idx = argno ; } else { args [ argno ] = convert_to_pmode ( args [ argno ] ) ; vector_type = displacement_vector_type ( ) ; if ( units == UNITS_elements && ! scaled_p ) shift_idx = argno + 1 ; } tree scalar_displacement_type = TREE_TYPE ( vector_type ) ; if ( shift_idx >= 0 ) { machine_mode arg_mode = GET_MODE ( args [ shift_idx ] ) ; if ( arg_mode == VOIDmode ) arg_mode = DImode ; unsigned int elt_bytes = GET_MODE_UNIT_SIZE ( mem_mode ) ; rtx shift = gen_int_mode ( exact_log2 ( elt_bytes ) , DImode ) ; args [ shift_idx ] = simplify_gen_binary ( ASHIFT , arg_mode , args [ shift_idx ] , shift ) ; units = UNITS_bytes ; } bool uxtw_p = ( TYPE_PRECISION ( scalar_displacement_type ) == 64 || TYPE_UNSIGNED ( scalar_displacement_type ) ) ; unsigned int scale = ( units == UNITS_bytes ? 1 : GET_MODE_UNIT_SIZE ( mem_mode ) ) ; if ( scaled_p ) { args . quick_insert ( argno + 2 , GEN_INT ( uxtw_p ) ) ; args . quick_insert ( argno + 3 , GEN_INT ( scale ) ) ; } else gcc_assert ( uxtw_p && scale == 1 ) ; } </s>

Convert the arguments to a gather/scatter function into the associated md operands . Argument ARGNO is the scalar or vector base and argument ARGNO + 1 is the scalar or vector displacement ( if applicable ) . The md pattern expects : - a scalar base - a vector displacement If SCALED_P is true , it also expects : - a const_int that is 1 if the displacement is zero-extended from 32 bits - a scaling multiplier ( 1 for bytes , 2 for .h indices , etc. ) . If SCALED_P is false , the displacement is implicitly zero-extended and the scaling multiplier is implicitly 1 .


<s> void function_expander :: prepare_prefetch_operands ( ) { unsigned int prfop = INTVAL ( args . last ( ) ) ; args . quick_push ( GEN_INT ( ( prfop & 8 ) != 0 ) ) ; args . quick_push ( GEN_INT ( ( ( prfop >> 1 ) & 3 ) + 1 ) ) ; } </s>

The final argument is an immediate svprfop value . Add two fake arguments to represent the rw and locality operands of a PREFETCH rtx .


<s> bool function_instance :: reads_global_state_p ( ) const { unsigned int flags = call_properties ( ) ; if ( flags & CP_READ_FPCR ) return true ; return flags & ( CP_READ_MEMORY | CP_READ_FFR ) ; } </s>

Return true if calls to the function could read some form of global state .


<s> gimple * gimple_folder :: redirect_call ( const function_instance & instance ) { registered_function * rfn = function_table -> find_with_hash ( instance , instance . hash ( ) ) ; if ( ! rfn ) return NULL ; gimple_call_set_fndecl ( call , rfn -> decl ) ; return call ; } </s>

Fold the call to a call to INSTANCE , with the same arguments .


<s> static void register_builtin_types ( ) { scalar_types [ VECTOR_TYPE_ ## ACLE_NAME ] = SCALAR_TYPE ; for ( unsigned int i = 0 ; i < NUM_VECTOR_TYPES ; ++ i ) { tree eltype = scalar_types [ i ] ; tree vectype ; unsigned int num_zr = 0 , num_pr = 0 ; if ( eltype == boolean_type_node ) { vectype = build_truth_vector_type_for_mode ( BYTES_PER_SVE_VECTOR , VNx16BImode ) ; gcc_assert ( TYPE_MODE ( vectype ) == VNx16BImode && TYPE_MODE ( vectype ) == TYPE_MODE_RAW ( vectype ) && TYPE_ALIGN ( vectype ) == 16 && known_eq ( wi :: to_poly_offset ( TYPE_SIZE ( vectype ) ) , BYTES_PER_SVE_VECTOR ) ) ; num_pr = 1 ; } else { scalar_mode elmode = SCALAR_TYPE_MODE ( eltype ) ; unsigned int elbytes = GET_MODE_SIZE ( elmode ) ; poly_uint64 nunits = exact_div ( BYTES_PER_SVE_VECTOR , elbytes ) ; machine_mode mode = aarch64_sve_data_mode ( elmode , nunits ) . require ( ) ; vectype = build_vector_type_for_mode ( eltype , mode ) ; gcc_assert ( VECTOR_MODE_P ( TYPE_MODE ( vectype ) ) && TYPE_MODE ( vectype ) == mode && TYPE_MODE_RAW ( vectype ) == mode && TYPE_ALIGN ( vectype ) == 128 && known_eq ( wi :: to_poly_offset ( TYPE_SIZE ( vectype ) ) , BITS_PER_SVE_VECTOR ) ) ; num_zr = 1 ; } vectype = build_distinct_type_copy ( vectype ) ; gcc_assert ( vectype == TYPE_MAIN_VARIANT ( vectype ) ) ; SET_TYPE_STRUCTURAL_EQUALITY ( vectype ) ; TYPE_ARTIFICIAL ( vectype ) = 1 ; TYPE_INDIVISIBLE_P ( vectype ) = 1 ; add_sve_type_attribute ( vectype , num_zr , num_pr , vector_types [ i ] . mangled_name ) ; make_type_sizeless ( vectype ) ; abi_vector_types [ i ] = vectype ; lang_hooks . types . register_builtin_type ( vectype , vector_types [ i ] . abi_name ) ; } } </s>

Register the built-in RVV ABI types , such as __rvv_int32m1_t .


<s> void function_builder :: register_function_group ( const function_group_info & group ) { ( * group . shape ) -> build ( * this , group ) ; } </s>


<s> static void register_svpattern ( ) { auto_vec < string_int_pair , 32 > values ; values . quick_push ( string_int_pair ( "SV_" # UPPER , VALUE ) ) ; AARCH64_FOR_SVPATTERN ( PUSH ) acle_svpattern = lang_hooks . types . simulate_enum_decl ( input_location , "svpattern" , values ) ; } </s>


<s> static void register_svprfop ( ) { auto_vec < string_int_pair , 16 > values ; values . quick_push ( string_int_pair ( "SV_" # UPPER , VALUE ) ) ; AARCH64_FOR_SVPRFOP ( PUSH ) acle_svprfop = lang_hooks . types . simulate_enum_decl ( input_location , "svprfop" , values ) ; } </s>


<s> static void register_tuple_type ( unsigned int num_vectors , vector_type_index type ) { tree tuple_type = lang_hooks . types . make_type ( RECORD_TYPE ) ; tree vector_type = acle_vector_types [ 0 ] [ type ] ; tree array_type = build_array_type_nelts ( vector_type , num_vectors ) ; gcc_assert ( VECTOR_MODE_P ( TYPE_MODE ( array_type ) ) && TYPE_MODE_RAW ( array_type ) == TYPE_MODE ( array_type ) && TYPE_ALIGN ( array_type ) == 128 ) ; tree field = build_decl ( input_location , FIELD_DECL , get_identifier ( "__val" ) , array_type ) ; DECL_FIELD_CONTEXT ( field ) = tuple_type ; TYPE_FIELDS ( tuple_type ) = field ; add_sve_type_attribute ( tuple_type , num_vectors , 0 , NULL ) ; make_type_sizeless ( tuple_type ) ; layout_type ( tuple_type ) ; gcc_assert ( VECTOR_MODE_P ( TYPE_MODE ( tuple_type ) ) && TYPE_MODE_RAW ( tuple_type ) == TYPE_MODE ( tuple_type ) && TYPE_ALIGN ( tuple_type ) == 128 ) ; char buffer [ sizeof ( "svbfloat16x4_t" ) ] ; const char * vector_type_name = vector_types [ type ] . acle_name ; snprintf ( buffer , sizeof ( buffer ) , "%.*sx%d_t" , ( int ) strlen ( vector_type_name ) - 2 , vector_type_name , num_vectors ) ; tree decl = build_decl ( input_location , TYPE_DECL , get_identifier ( buffer ) , tuple_type ) ; TYPE_NAME ( tuple_type ) = decl ; TYPE_STUB_DECL ( tuple_type ) = decl ; lang_hooks . decls . pushdecl ( decl ) ; DECL_ORIGINAL_TYPE ( decl ) = NULL_TREE ; acle_vector_types [ num_vectors - 1 ] [ type ] = tuple_type ; } </s>

Register the tuple type that contains NUM_VECTORS vectors of type TYPE .


<s> static void register_vector_type ( vector_type_index type ) { tree vectype = abi_vector_types [ type ] ; tree id = get_identifier ( vector_types [ type ] . acle_name ) ; tree decl = build_decl ( input_location , TYPE_DECL , id , vectype ) ; decl = lang_hooks . decls . pushdecl ( decl ) ; if ( decl && TREE_CODE ( decl ) == TYPE_DECL && TYPE_MAIN_VARIANT ( TREE_TYPE ( decl ) ) == vectype ) vectype = TREE_TYPE ( decl ) ; acle_vector_types [ 0 ] [ type ] = vectype ; } </s>

Register vector type TYPE under its risv_vector.h name .


"ACLE function %qD requires ISA extension %qs"

"you can enable %qs using the command-line"

" option %<-march%>, or by using the %<target%>"

Report an error against LOCATION that the user has tried to use function FNDECL when extension EXTENSION is disabled .


"passing %wd to argument %d of %qE, which expects"

Report that LOCATION has a call to FNDECL in which argument ARGNO has the value ACTUAL , whereas the function requires either VALUE0 or VALUE1 . ARGNO counts from zero .


"argument %d of %qE must be an integer constant"

Report that LOCATION has a call to FNDECL in which argument ARGNO was not an integer constant expression . ARGNO counts from zero .


Report that LOCATION has a call to FNDECL in which argument ARGNO has the value ACTUAL , whereas the function requires a valid value of enum type ENUMTYPE . ARGNO counts from zero .


Report that LOCATION has a call to FNDECL in which argument ARGNO has the value ACTUAL , whereas the function requires one of VALUE0 .. 3 . ARGNO counts from zero .


"%qE has no form that takes %qT arguments"

Report that the function has no form that takes type suffix TYPE . Return error_mark_node .


Report that LOCATION has a call to FNDECL in which argument ARGNO has the value ACTUAL , whereas the function requires a value in the range [ MIN , MAX ] . ARGNO counts from zero .


"passing %qT to argument %d of %qE, which"

Require argument ARGNO to be a ( possibly variable ) scalar , expecting it to have the following properties : - the type class must be the same as for type suffix 0 if EXPECTED_TCLASS is SAME_TYPE_CLASS , otherwise it must be EXPECTED_TCLASS itself . - the element size must be the same as for type suffix 0 if EXPECTED_BITS is SAME_TYPE_SIZE , otherwise it must be EXPECTED_BITS itself . Return true if the argument is valid , otherwise report an appropriate error . Note that we do n't check whether the scalar type actually has the required properties , since that 's subject to implicit promotions and conversions . Instead we just use the expected properties to tune the error message .


<s> bool function_checker :: require_immediate ( unsigned int argno , HOST_WIDE_INT & value_out ) { gcc_assert ( argno < m_nargs ) ; tree arg = m_args [ argno ] ; if ( ! tree_fits_uhwi_p ( arg ) ) { report_non_ice ( location , fndecl , argno ) ; return false ; } value_out = tree_to_uhwi ( arg ) ; return true ; } </s>

Check that argument ARGNO is an integer constant expression and store its value in VALUE_OUT if so . The caller should first check that argument ARGNO exists .


<s> bool function_checker :: require_immediate_either_or ( unsigned int rel_argno , HOST_WIDE_INT value0 , HOST_WIDE_INT value1 ) { unsigned int argno = m_base_arg + rel_argno ; if ( ! argument_exists_p ( argno ) ) return true ; HOST_WIDE_INT actual ; if ( ! require_immediate ( argno , actual ) ) return false ; if ( actual != value0 && actual != value1 ) { report_neither_nor ( location , fndecl , argno , actual , 90 , 270 ) ; return false ; } return true ; } </s>

Check that argument REL_ARGNO is an integer constant expression that has the value VALUE0 or VALUE1 . REL_ARGNO counts from the end of the predication arguments .


<s> bool function_checker :: require_immediate_enum ( unsigned int rel_argno , tree type ) { unsigned int argno = m_base_arg + rel_argno ; if ( ! argument_exists_p ( argno ) ) return true ; HOST_WIDE_INT actual ; if ( ! require_immediate ( argno , actual ) ) return false ; for ( tree entry = TYPE_VALUES ( type ) ; entry ; entry = TREE_CHAIN ( entry ) ) { tree value = TREE_VALUE ( entry ) ; if ( TREE_CODE ( value ) == CONST_DECL ) value = DECL_INITIAL ( value ) ; if ( wi :: to_widest ( value ) == actual ) return true ; } report_not_enum ( location , fndecl , argno , actual , type ) ; return false ; } </s>

Check that argument REL_ARGNO is an integer constant expression that has a valid value for enumeration type TYPE . REL_ARGNO counts from the end of the predication arguments .


<s> bool function_checker :: require_immediate_lane_index ( unsigned int rel_argno , unsigned int group_size ) { unsigned int argno = m_base_arg + rel_argno ; if ( ! argument_exists_p ( argno ) ) return true ; machine_mode mode = TYPE_MODE ( type_argument_type ( m_fntype , argno ) ) ; gcc_assert ( VECTOR_MODE_P ( mode ) ) ; unsigned int nlanes = 128 / ( group_size * GET_MODE_UNIT_BITSIZE ( mode ) ) ; return require_immediate_range ( rel_argno , 0 , nlanes - 1 ) ; } </s>

Check that argument REL_ARGNO is suitable for indexing argument REL_ARGNO - 1 , in groups of GROUP_SIZE elements . REL_ARGNO counts from the end of the predication arguments .


<s> bool function_checker :: require_immediate_one_of ( unsigned int rel_argno , HOST_WIDE_INT value0 , HOST_WIDE_INT value1 , HOST_WIDE_INT value2 , HOST_WIDE_INT value3 ) { unsigned int argno = m_base_arg + rel_argno ; if ( ! argument_exists_p ( argno ) ) return true ; HOST_WIDE_INT actual ; if ( ! require_immediate ( argno , actual ) ) return false ; if ( actual != value0 && actual != value1 && actual != value2 && actual != value3 ) { report_not_one_of ( location , fndecl , argno , actual , value0 , value1 , value2 , value3 ) ; return false ; } return true ; } </s>

Check that argument REL_ARGNO is an integer constant expression that has one of the given values .


<s> bool function_checker :: require_immediate_range ( unsigned int rel_argno , HOST_WIDE_INT min , HOST_WIDE_INT max ) { unsigned int argno = m_base_arg + rel_argno ; if ( ! argument_exists_p ( argno ) ) return true ; gcc_assert ( min >= 0 && min <= max ) ; HOST_WIDE_INT actual ; if ( ! require_immediate ( argno , actual ) ) return false ; if ( ! IN_RANGE ( actual , min , max ) ) { report_out_of_range ( location , fndecl , argno , actual , min , max ) ; return false ; } return true ; } </s>

Check that argument REL_ARGNO is an integer constant expression in the range [ MIN , MAX ] . REL_ARGNO counts from the end of the predication arguments .


<s> bool function_resolver :: require_integer_immediate ( unsigned int argno ) { if ( ! scalar_argument_p ( argno ) ) { report_non_ice ( location , fndecl , argno ) ; return false ; } return true ; } </s>

Require argument ARGNO to be suitable for an integer constant expression . Return true if it is , otherwise report an appropriate error . function_checker checks whether the argument is actually constant and has a suitable range . The reason for distinguishing immediate arguments here is because it provides more consistent error messages than require_scalar_type would .


"call to %qE is ambiguous; argument %d has type"

Argument FIRST_ARGNO is a scalar with type EXPECTED_TYPE , and argument ARGNO should be consistent with it . Return true if it is , otherwise report an appropriate error .


"passing %qT to argument %d of %qE, but"

Require argument ARGNO to match argument FIRST_ARGNO , which was inferred to be a pointer to a scalar element of type TYPE .


Like require_vector_type , but TYPE is inferred from previous arguments rather than being a fixed part of the function signature . This changes the nature of the error messages .


Require argument ARGNO to be some form of pointer , without being specific about its target type . Return true if the argument has the right form , otherwise report an appropriate error .


Require argument ARGNO to be a ( possibly variable ) scalar , using EXPECTED as the name of its expected type . Return true if the argument has the right form , otherwise report an appropriate error .


Require argument ARGNO to be a vector or scalar argument . Return true if it is , otherwise report an appropriate error .


Require argument ARGNO to have vector type TYPE , in cases where this requirement holds for all uses of the function . Return true if the argument has the right form , otherwise report an appropriate error .


<s> tree function_resolver :: resolve ( ) { return shape -> resolve ( * this ) ; } </s>

If the call is erroneous , report an appropriate error and return error_mark_node . Otherwise , if the function is overloaded , return the decl of the non-overloaded function . Return NULL_TREE otherwise , indicating that the call should be processed in the normal way .


"cannot combine a base of type %qT with"

Require arguments ARGNO and ARGNO + 1 to form an ADR-style address , i.e . one with a vector of base addresses and a vector of displacements . The overloaded function 's mode suffix determines the units of the displacement ( bytes for `` _offset '' , elements for `` _index '' ) . Return the associated mode suffix on success , otherwise report an error and return MODE_none .


" which expects a vector or pointer base address"

Require the arguments starting at ARGNO to form a gather-style address . There are three possible uses : - for loading into elements of type TYPE ( when LOAD_P is true ) - for storing from elements of type TYPE ( when LOAD_P is false ) - for prefetching data ( when TYPE is NUM_TYPE_SUFFIXES ) The three possible addresses are : - a vector base with no displacement - a vector base and a scalar displacement - a scalar ( pointer ) base and a vector displacement The overloaded function 's mode suffix determines whether there is a displacement , and if so , what units it uses : - MODE_none : no displacement - MODE_offset : the displacement is measured in bytes - MODE_index : the displacement is measured in elements Return the mode of the non-overloaded function on success , otherwise report an error and return MODE_none .


<s> tree resolve_overloaded_builtin ( location_t location , unsigned int code , vec < tree , va_gc > * arglist ) { if ( code >= vec_safe_length ( registered_functions ) ) return NULL_TREE ; registered_function & rfn = * ( * registered_functions ) [ code ] ; if ( rfn . overloaded_p ) return function_resolver ( location , rfn . instance , rfn . decl , * arglist ) . resolve ( ) ; return NULL_TREE ; } </s>

If we 're implementing manual overloading , check whether the SVE function with subcode CODE is overloaded , and if so attempt to determine the corresponding non-overloaded function . The call occurs at location LOCATION and has the arguments given by ARGLIST . If the call is erroneous , report an appropriate error and return error_mark_node . Otherwise , if the function is overloaded , return the decl of the non-overloaded function . Return NULL_TREE otherwise , indicating that the call should be processed in the normal way .


"when storing %qT, %qE requires a vector"

"%qE requires a vector base and a scalar"

"%qE does not support 32-bit vector type %qT"

"%qE does not support 32-bit sign-extended"

"passing %qT to argument %d of %qE, which when"

" loading %qT expects a vector of %d-bit integers"

" storing %qT expects a vector of %d-bit integers"

"passing %qT to argument %d of %qE, which"

Require argument ARGNO to be a vector displacement in a gather-style address . There are three possible uses : - for loading into elements of type TYPE ( when LOAD_P is true ) - for storing from elements of type TYPE ( when LOAD_P is false ) - for prefetching data ( when TYPE is NUM_TYPE_SUFFIXES ) The overloaded function 's mode suffix determines the units of the displacement ( bytes for `` _offset '' , elements for `` _index '' ) . Return the associated mode on success , otherwise report an error and return MODE_none .


<s> tree function_resolver :: resolve_to ( mode_suffix_index mode , type_suffix_index type0 , type_suffix_index type1 ) { tree res = lookup_form ( mode , type0 , type1 ) ; if ( ! res ) { if ( type1 == NUM_TYPE_SUFFIXES ) return report_no_such_form ( type0 ) ; if ( type0 == type_suffix_ids [ 0 ] ) return report_no_such_form ( type1 ) ; gcc_unreachable ( ) ; } return res ; } </s>

Resolve the function to one with the mode suffix given by MODE and the type suffixes given by TYPE0 and TYPE1 . Return its function decl on success , otherwise report an error and return error_mark_node .


<s> tree function_resolver :: resolve_unary ( type_class_index merge_tclass , unsigned int merge_bits , bool treat_as_merge_p ) { type_suffix_index type ; if ( pred == PRED_m || treat_as_merge_p ) { if ( ! check_num_arguments ( 3 ) ) return error_mark_node ; if ( merge_tclass == SAME_TYPE_CLASS && merge_bits == SAME_SIZE ) { if ( ( type = infer_vector_type ( 0 ) ) == NUM_TYPE_SUFFIXES || ! require_vector_type ( 1 , VECTOR_TYPE_svbool_t ) || ! require_matching_vector_type ( 2 , type ) ) return error_mark_node ; } else { if ( ! require_vector_type ( 1 , VECTOR_TYPE_svbool_t ) || ( type = infer_vector_type ( 2 ) ) == NUM_TYPE_SUFFIXES || ! require_derived_vector_type ( 0 , 2 , type , merge_tclass , merge_bits ) ) return error_mark_node ; } } else { unsigned int i , nargs ; if ( ! check_gp_argument ( 1 , i , nargs ) || ( type = infer_vector_type ( i ) ) == NUM_TYPE_SUFFIXES ) return error_mark_node ; } if ( type_suffix_ids [ 0 ] != NUM_TYPE_SUFFIXES ) return resolve_to ( mode_suffix_id , type_suffix_ids [ 0 ] , type ) ; return resolve_to ( mode_suffix_id , type ) ; } </s>

Resolve a ( possibly predicated ) unary function . If the function uses merge predication or if TREAT_AS_MERGE_P is true , there is an extra vector argument before the governing predicate that specifies the values of inactive elements . This argument has the following properties : - the type class must be the same as for active elements if MERGE_TCLASS is SAME_TYPE_CLASS , otherwise it must be MERGE_TCLASS itself . - the element size must be the same as for active elements if MERGE_BITS is SAME_TYPE_SIZE , otherwise it must be MERGE_BITS itself . Return the function decl of the resolved function on success , otherwise report a suitable error and return error_mark_node .


<s> tree function_resolver :: resolve_uniform ( unsigned int nops , unsigned int nimm ) { unsigned int i , nargs ; type_suffix_index type ; if ( ! check_gp_argument ( nops + nimm , i , nargs ) || ( type = infer_vector_type ( i ) ) == NUM_TYPE_SUFFIXES ) return error_mark_node ; i += 1 ; for ( ; i < nargs - nimm ; ++ i ) if ( ! require_matching_vector_type ( i , type ) ) return error_mark_node ; for ( ; i < nargs ; ++ i ) if ( ! require_integer_immediate ( i ) ) return error_mark_node ; return resolve_to ( mode_suffix_id , type ) ; } </s>

Resolve a ( possibly predicated ) function that takes NOPS like-typed vector arguments followed by NIMM integer immediates . Return the function decl of the resolved function on success , otherwise report a suitable error and return error_mark_node .


<s> tree function_resolver :: resolve_uniform_opt_n ( unsigned int nops ) { unsigned int i , nargs ; type_suffix_index type ; if ( ! check_gp_argument ( nops , i , nargs ) || ( type = infer_vector_type ( i ) ) == NUM_TYPE_SUFFIXES ) return error_mark_node ; unsigned int first_arg = i ++ ; for ( ; i < nargs - 1 ; ++ i ) if ( ! require_matching_vector_type ( i , type ) ) return error_mark_node ; return finish_opt_n_resolution ( i , first_arg , type ) ; } </s>

Resolve a ( possibly predicated ) function that offers a choice between taking : - NOPS like-typed vector arguments or - NOPS - 1 like-typed vector arguments followed by a scalar argument Return the function decl of the resolved function on success , otherwise report a suitable error and return error_mark_node .


<s> void function_expander :: rotate_inputs_left ( unsigned int start , unsigned int end ) { rtx new_last = args [ start ] ; for ( unsigned int i = start ; i < end - 1 ; ++ i ) args [ i ] = args [ i + 1 ] ; args [ end - 1 ] = new_last ; } </s>

Rotate inputs args [ START : END ] one position to the left , so that args [ START ] becomes args [ END - 1 ] .


<s> bool function_resolver :: scalar_argument_p ( unsigned int i ) { tree type = get_argument_type ( i ) ; return ( INTEGRAL_TYPE_P ( type ) || POINTER_TYPE_P ( type ) || SCALAR_FLOAT_TYPE_P ( type ) ) ; } </s>

Return true if argument I is some form of scalar value .


<s> inline tree function_instance :: scalar_type ( unsigned int i ) const { return scalar_types [ type_suffix ( i ) . vector_type ] ; } </s>

Return the scalar type associated with type suffix I .


<s>
static
bool
sizeless_type_p
(
const_tree
type
)
{
if
(
type
==
error_mark_node
)
return
NULL_TREE
;
return
lookup_attribute
(
"SVE sizeless type"
,
TYPE_ATTRIBUTES
(
type
)
)
;
}
</s>

Return true if TYPE is a sizeless type .


<s> bool function_expander :: try_negating_argument ( unsigned int argno , machine_mode mode ) { rtx x = args [ argno ] ; if ( ! VECTOR_MODE_P ( GET_MODE ( x ) ) ) mode = GET_MODE_INNER ( mode ) ; x = simplify_unary_operation ( NEG , mode , x , mode ) ; if ( ! x ) return false ; args [ argno ] = x ; return true ; } </s>

Return true if the negation of argument ARGNO can be folded away , replacing it with the negated value if so . MODE is the associated vector mode , but the argument could be a single element . The main case this handles is constant arguments .


<s> inline tree function_instance :: tuple_type ( unsigned int i ) const { unsigned int num_vectors = vectors_per_tuple ( ) ; return acle_vector_types [ num_vectors - 1 ] [ type_suffix ( i ) . vector_type ] ; } </s>

If the function operates on tuples of vectors , return the tuple type associated with type suffix I , otherwise return the vector type associated with type suffix I .


<s> inline tree tuple_type_field ( tree type ) { for ( tree field = TYPE_FIELDS ( type ) ; field ; field = DECL_CHAIN ( field ) ) if ( TREE_CODE ( field ) == FIELD_DECL ) return field ; gcc_unreachable ( ) ; } </s>

Return the single field in tuple type TYPE .


<s> inline const type_suffix_info & function_instance :: type_suffix ( unsigned int i ) const { return type_suffixes [ type_suffix_ids [ i ] ] ; } </s>

Return information about type suffix I .


<s> rtx function_expander :: use_contiguous_load_insn ( insn_code icode ) { machine_mode mem_mode = memory_vector_mode ( ) ; add_output_operand ( icode ) ; add_mem_operand ( mem_mode , get_contiguous_base ( mem_mode ) ) ; add_input_operand ( icode , args [ 0 ] ) ; if ( GET_MODE_UNIT_BITSIZE ( mem_mode ) < type_suffix ( 0 ) . element_bits ) add_input_operand ( icode , CONSTM1_RTX ( VNx16BImode ) ) ; return generate_insn ( icode ) ; } </s>


<s> rtx function_expander :: use_contiguous_prefetch_insn ( insn_code icode ) { add_input_operand ( icode , args [ 0 ] ) ; add_address_operand ( get_contiguous_base ( VNx16QImode ) ) ; for ( unsigned int i = args . length ( ) - 3 ; i < args . length ( ) ; ++ i ) add_input_operand ( icode , args [ i ] ) ; return generate_insn ( icode ) ; } </s>

Implement the call using instruction ICODE , which prefetches from address operand 1 under the control of predicate operand 0 . Operands 2 , 3 and 4 respectively specify the svprfop value , the PREFETCH rw flag and the PREFETCH locality .


<s> rtx function_expander :: use_contiguous_store_insn ( insn_code icode ) { machine_mode mem_mode = memory_vector_mode ( ) ; add_mem_operand ( mem_mode , get_contiguous_base ( mem_mode ) ) ; add_input_operand ( icode , args . last ( ) ) ; add_input_operand ( icode , args [ 0 ] ) ; return generate_insn ( icode ) ; } </s>


<s> rtx function_expander :: use_exact_insn ( insn_code icode ) { unsigned int nops = insn_data [ icode ] . n_operands ; if ( ! function_returns_void_p ( ) ) { add_output_operand ( icode ) ; nops -= 1 ; } for ( unsigned int i = 0 ; i < nops ; ++ i ) add_input_operand ( icode , args [ i ] ) ; return generate_insn ( icode ) ; } </s>

Implement the call using instruction ICODE , with a 1:1 mapping between arguments and input operands .


<s> rtx function_expander :: use_pred_x_insn ( insn_code icode ) { gcc_assert ( pred == PRED_x ) ; unsigned int nops = args . length ( ) - 1 ; bool has_float_operand_p = FLOAT_MODE_P ( insn_data [ icode ] . operand [ 0 ] . mode ) ; add_output_operand ( icode ) ; add_input_operand ( icode , args [ 0 ] ) ; for ( unsigned int i = 0 ; i < nops ; ++ i ) { add_input_operand ( icode , args [ i + 1 ] ) ; if ( FLOAT_MODE_P ( GET_MODE ( args [ i + 1 ] ) ) ) has_float_operand_p = true ; } if ( has_float_operand_p ) { rtx pred = m_ops [ 1 ] . value ; if ( flag_trapping_math && pred != CONST1_RTX ( GET_MODE ( pred ) ) ) add_integer_operand ( SVE_STRICT_GP ) ; else add_integer_operand ( SVE_RELAXED_GP ) ; } return generate_insn ( icode ) ; } </s>

Implement the call using instruction ICODE , which is a predicated operation that returns arbitrary values for inactive lanes .


<s> rtx function_expander :: use_unpred_insn ( insn_code icode ) { gcc_assert ( pred == PRED_x || pred == PRED_none ) ; unsigned int nops = insn_data [ icode ] . n_operands - 1 ; unsigned int bias = ( pred == PRED_x ? 1 : 0 ) ; unsigned int i = 0 ; add_output_operand ( icode ) ; for ( ; i < nops ; ++ i ) add_input_operand ( icode , args [ i + bias ] ) ; return generate_insn ( icode ) ; } </s>

Implement the call using instruction ICODE , which does not use a governing predicate . We must therefore drop the GP from an _x call .


<s> rtx function_expander :: use_vcond_mask_insn ( insn_code icode , unsigned int merge_argno ) { machine_mode mode = vector_mode ( 0 ) ; unsigned int opno = 0 ; rtx false_arg = get_fallback_value ( mode , 1 , merge_argno , opno ) ; rtx pred_arg = args [ opno ++ ] ; rtx true_arg = args [ opno ++ ] ; add_output_operand ( icode ) ; add_input_operand ( icode , true_arg ) ; add_input_operand ( icode , false_arg ) ; add_input_operand ( icode , pred_arg ) ; return generate_insn ( icode ) ; } </s>

Implement the call using instruction ICODE , which is a select-like operation with the following operands : 0 : output 1 : true value 2 : false value 3 : predicate MERGE_ARGNO is the argument that provides the `` false '' value for _m functions , or DEFAULT_MERGE_ARGNO if we should apply the usual rules .


<s> inline unsigned int function_instance :: vectors_per_tuple ( ) const { return base -> vectors_per_tuple ( ) ; } </s>

If the function operates on tuples of vectors , return the number of vectors in the tuples , otherwise return 1 .


<s> inline machine_mode function_instance :: vector_mode ( unsigned int i ) const { return type_suffix ( i ) . vector_mode ; } </s>

Return the machine_mode of the corresponding vector type .


<s> inline tree function_instance :: vector_type ( unsigned int i ) const { return acle_vector_types [ 0 ] [ type_suffix ( i ) . vector_type ] ; } </s>

Return the vector type associated with type suffix I .


"SVE type %qT does not have a fixed size"

"SVE type %qT does not have a defined alignment"

"variables of type %qT cannot have thread-local"

"member variables cannot have SVE type %qT"

"array elements cannot have SVE type %qT"

"cannot allocate objects with SVE type %qT"

"cannot delete objects with SVE type %qT"

Implement TARGET_VERIFY_TYPE_CONTEXT for RVV types .


<s> static unsigned aarch64_add_stmt_cost ( class vec_info * vinfo , void * data , int count , enum vect_cost_for_stmt kind , struct _stmt_vec_info * stmt_info , tree vectype , int misalign , enum vect_cost_model_location where ) { auto * costs = static_cast < aarch64_vector_costs * > ( data ) ; unsigned retval = 0 ; if ( flag_vect_cost_model ) { int stmt_cost = aarch64_builtin_vectorization_cost ( kind , vectype , misalign ) ; loop_vec_info loop_vinfo = dyn_cast < loop_vec_info > ( vinfo ) ; bb_vec_info bb_vinfo = dyn_cast < bb_vec_info > ( vinfo ) ; if ( ! costs -> analyzed_vinfo && aarch64_use_new_vector_costs_p ( ) ) { if ( loop_vinfo ) aarch64_analyze_loop_vinfo ( loop_vinfo , costs ) ; else aarch64_analyze_bb_vinfo ( bb_vinfo , costs ) ; costs -> analyzed_vinfo = true ; } if ( stmt_info && aarch64_use_new_vector_costs_p ( ) ) { if ( vectype && aarch64_sve_only_stmt_p ( stmt_info , vectype ) ) costs -> saw_sve_only_op = true ; stmt_cost = aarch64_detect_scalar_stmt_subtype ( vinfo , kind , stmt_info , stmt_cost ) ; if ( vectype && costs -> vec_flags ) stmt_cost = aarch64_detect_vector_stmt_subtype ( vinfo , kind , stmt_info , vectype , where , stmt_cost ) ; } if ( stmt_info && vectype && aarch64_sve_mode_p ( TYPE_MODE ( vectype ) ) ) stmt_cost = aarch64_sve_adjust_stmt_cost ( vinfo , kind , stmt_info , vectype , stmt_cost ) ; if ( stmt_info && aarch64_use_new_vector_costs_p ( ) ) { stmt_cost = aarch64_adjust_stmt_cost ( kind , stmt_info , vectype , stmt_cost ) ; auto * issue_info = aarch64_tune_params . vec_costs -> issue_info ; if ( loop_vinfo && issue_info && costs -> vec_flags && where == vect_body && vectype && stmt_cost != 0 ) { aarch64_count_ops ( vinfo , costs , count , kind , stmt_info , vectype , 0 , & costs -> scalar_ops , issue_info -> scalar , vect_nunits_for_cost ( vectype ) ) ; if ( aarch64_sve_mode_p ( vinfo -> vector_mode ) && issue_info -> sve ) { aarch64_count_ops ( vinfo , costs , count , kind , stmt_info , vectype , VEC_ADVSIMD , & costs -> advsimd_ops , issue_info -> advsimd , aarch64_estimated_sve_vq ( ) ) ; aarch64_count_ops ( vinfo , costs , count , kind , stmt_info , vectype , VEC_ANY_SVE , & costs -> sve_ops , issue_info -> sve , 1 ) ; } else aarch64_count_ops ( vinfo , costs , count , kind , stmt_info , vectype , VEC_ADVSIMD , & costs -> advsimd_ops , issue_info -> advsimd , 1 ) ; } if ( where == vect_body && costs -> unrolled_advsimd_niters ) costs -> unrolled_advsimd_stmts += count * costs -> unrolled_advsimd_niters ; } if ( where == vect_body && stmt_info && stmt_in_inner_loop_p ( vinfo , stmt_info ) ) count *= 50 ; retval = ( unsigned ) ( count * stmt_cost ) ; costs -> region [ where ] += retval ; } return retval ; } </s>


<s> static void aarch64_adjust_generic_arch_tuning ( struct tune_params & current_tune ) { if ( TARGET_SVE2 ) current_tune . extra_tuning_flags &= ~ AARCH64_EXTRA_TUNE_CSE_SVE_VL_CONSTANTS ; } </s>

Adjust CURRENT_TUNE ( a generic tuning struct ) with settings that are best for a generic target with the currently-enabled architecture extensions .


<s> static unsigned int aarch64_adjust_stmt_cost ( vect_cost_for_stmt kind , stmt_vec_info stmt_info , tree vectype , unsigned int stmt_cost ) { if ( vectype ) { const simd_vec_cost * simd_costs = aarch64_simd_vec_costs ( vectype ) ; switch ( aarch64_ld234_st234_vectors ( kind , stmt_info ) ) { case 2 : stmt_cost += simd_costs -> ld2_st2_permute_cost ; break ; case 3 : stmt_cost += simd_costs -> ld3_st3_permute_cost ; break ; case 4 : stmt_cost += simd_costs -> ld4_st4_permute_cost ; break ; } if ( kind == vector_stmt || kind == vec_to_scalar ) if ( tree cmp_type = aarch64_embedded_comparison_type ( stmt_info ) ) { if ( FLOAT_TYPE_P ( cmp_type ) ) stmt_cost += simd_costs -> fp_stmt_cost ; else stmt_cost += simd_costs -> int_stmt_cost ; } } if ( kind == scalar_stmt ) if ( tree cmp_type = aarch64_embedded_comparison_type ( stmt_info ) ) { if ( FLOAT_TYPE_P ( cmp_type ) ) stmt_cost += aarch64_tune_params . vec_costs -> scalar_fp_stmt_cost ; else stmt_cost += aarch64_tune_params . vec_costs -> scalar_int_stmt_cost ; } return stmt_cost ; } </s>

STMT_COST is the cost calculated for STMT_INFO , which has cost kind KIND and which when vectorized would operate on vector type VECTYPE . Add the cost of any embedded operations .


<s> static void aarch64_analyze_bb_vinfo ( bb_vec_info bb_vinfo , aarch64_vector_costs * costs ) { costs -> vec_flags = aarch64_classify_vector_mode ( bb_vinfo -> vector_mode ) ; } </s>

Do one-time initialization of COSTS given that we 're costing the block vectorization described by BB_VINFO .


Do one-time initialization of COSTS given that we 're costing the loop vectorization described by LOOP_VINFO .


<s> void aarch64_asm_file_end ( ) { aarch64_sls_emit_shared_blr_thunks ( asm_out_file ) ; TARGET_ASM_FILE_END ( ) ; } </s>


<s> static unsigned int aarch64_autovectorize_vector_modes ( vector_modes * modes , bool ) { static const machine_mode sve_modes [ ] = { VNx16QImode , VNx8QImode , VNx4QImode , VNx2QImode } ; static const machine_mode advsimd_modes [ ] = { V16QImode , V8QImode , V4HImode , V2SImode } ; bool only_asimd_p = aarch64_autovec_preference == 1 ; bool only_sve_p = aarch64_autovec_preference == 2 ; unsigned int sve_i = ( TARGET_SVE && ! only_asimd_p ) ? 0 : ARRAY_SIZE ( sve_modes ) ; unsigned int advsimd_i = 0 ; while ( ! only_sve_p && advsimd_i < ARRAY_SIZE ( advsimd_modes ) ) { if ( sve_i < ARRAY_SIZE ( sve_modes ) && aarch64_cmp_autovec_modes ( sve_modes [ sve_i ] , advsimd_modes [ advsimd_i ] ) ) modes -> safe_push ( sve_modes [ sve_i ++ ] ) ; else modes -> safe_push ( advsimd_modes [ advsimd_i ++ ] ) ; } while ( sve_i < ARRAY_SIZE ( sve_modes ) ) modes -> safe_push ( sve_modes [ sve_i ++ ] ) ; unsigned int flags = 0 ; if ( TARGET_SVE && aarch64_sve_compare_costs ) flags |= VECT_COMPARE_COSTS ; return flags ; } </s>

Return a list of possible vector sizes for the vectorizer to iterate over .


<s> static bool aarch64_base_register_rtx_p ( rtx x , bool strict_p ) { if ( ! strict_p && SUBREG_P ( x ) && contains_reg_of_mode [ GENERAL_REGS ] [ GET_MODE ( SUBREG_REG ( x ) ) ] ) x = SUBREG_REG ( x ) ; return ( REG_P ( x ) && aarch64_regno_ok_for_base_p ( REGNO ( x ) , strict_p ) ) ; } </s>

Return true if X is a valid base register for mode MODE . STRICT_P is true if REG_OK_STRICT is in effect .


<s> static int aarch64_builtin_vectorization_cost ( enum vect_cost_for_stmt type_of_cost , tree vectype , int misalign ATTRIBUTE_UNUSED ) { unsigned elements ; const cpu_vector_cost * costs = aarch64_tune_params . vec_costs ; bool fp = false ; if ( vectype != NULL ) fp = FLOAT_TYPE_P ( vectype ) ; const simd_vec_cost * simd_costs = aarch64_simd_vec_costs ( vectype ) ; switch ( type_of_cost ) { case scalar_stmt : return fp ? costs -> scalar_fp_stmt_cost : costs -> scalar_int_stmt_cost ; case scalar_load : return costs -> scalar_load_cost ; case scalar_store : return costs -> scalar_store_cost ; case vector_stmt : return fp ? simd_costs -> fp_stmt_cost : simd_costs -> int_stmt_cost ; case vector_load : return simd_costs -> align_load_cost ; case vector_store : return simd_costs -> store_cost ; case vec_to_scalar : return simd_costs -> vec_to_scalar_cost ; case scalar_to_vec : return simd_costs -> scalar_to_vec_cost ; case unaligned_load : case vector_gather_load : return simd_costs -> unalign_load_cost ; case unaligned_store : case vector_scatter_store : return simd_costs -> unalign_store_cost ; case cond_branch_taken : return costs -> cond_taken_branch_cost ; case cond_branch_not_taken : return costs -> cond_not_taken_branch_cost ; case vec_perm : return simd_costs -> permute_cost ; case vec_promote_demote : return fp ? simd_costs -> fp_stmt_cost : simd_costs -> int_stmt_cost ; case vec_construct : elements = estimated_poly_value ( TYPE_VECTOR_SUBPARTS ( vectype ) ) ; return elements / 2 + 1 ; default : gcc_unreachable ( ) ; } } </s>


<s> bool aarch64_can_const_movi_rtx_p ( rtx x , machine_mode mode ) { if ( ! TARGET_SIMD ) return false ; machine_mode vmode ; scalar_int_mode imode ; unsigned HOST_WIDE_INT ival ; if ( CONST_DOUBLE_P ( x ) && SCALAR_FLOAT_MODE_P ( mode ) ) { if ( ! aarch64_reinterpret_float_as_int ( x , & ival ) ) return false ; if ( aarch64_float_const_zero_rtx_p ( x ) ) return true ; imode = int_mode_for_mode ( mode ) . require ( ) ; } else if ( CONST_INT_P ( x ) && is_a < scalar_int_mode > ( mode , & imode ) ) ival = INTVAL ( x ) ; else return false ; int width = GET_MODE_BITSIZE ( imode ) == 64 ? 128 : 64 ; vmode = aarch64_simd_container_mode ( imode , width ) ; rtx v_op = aarch64_simd_gen_const_vector_dup ( vmode , ival ) ; return aarch64_simd_valid_immediate ( v_op , NULL ) ; } </s>

Return TRUE if rtx X is immediate constant that fits in a single MOVI immediate operation .


<s> bool aarch64_can_tag_addresses ( ) { return ! TARGET_ILP32 ; } </s>

Implement TARGET_MEMTAG_CAN_TAG_ADDRESSES . Here we tell the rest of the compiler that we automatically ignore the top byte of our pointers , which allows using -fsanitize=hwaddress .


<s> enum aarch64_symbol_type aarch64_classify_symbol ( rtx x , HOST_WIDE_INT offset ) { x = strip_salt ( x ) ; if ( LABEL_REF_P ( x ) ) { switch ( aarch64_cmodel ) { case AARCH64_CMODEL_LARGE : return SYMBOL_FORCE_TO_MEM ; case AARCH64_CMODEL_TINY_PIC : case AARCH64_CMODEL_TINY : return SYMBOL_TINY_ABSOLUTE ; case AARCH64_CMODEL_SMALL_SPIC : case AARCH64_CMODEL_SMALL_PIC : case AARCH64_CMODEL_SMALL : return SYMBOL_SMALL_ABSOLUTE ; default : gcc_unreachable ( ) ; } } if ( SYMBOL_REF_P ( x ) ) { if ( aarch64_tls_symbol_p ( x ) ) return aarch64_classify_tls_symbol ( x ) ; switch ( aarch64_cmodel ) { case AARCH64_CMODEL_TINY : if ( SYMBOL_REF_WEAK ( x ) && ! aarch64_symbol_binds_local_p ( x ) ) return SYMBOL_FORCE_TO_MEM ; if ( ! ( IN_RANGE ( offset , - 0x10000 , 0x10000 ) || offset_within_block_p ( x , offset ) ) ) return SYMBOL_FORCE_TO_MEM ; return SYMBOL_TINY_ABSOLUTE ; case AARCH64_CMODEL_SMALL : if ( SYMBOL_REF_WEAK ( x ) && ! aarch64_symbol_binds_local_p ( x ) ) return SYMBOL_FORCE_TO_MEM ; if ( ! ( IN_RANGE ( offset , - 0x100000 , 0x100000 ) || offset_within_block_p ( x , offset ) ) ) return SYMBOL_FORCE_TO_MEM ; return SYMBOL_SMALL_ABSOLUTE ; case AARCH64_CMODEL_TINY_PIC : if ( ! aarch64_symbol_binds_local_p ( x ) ) return SYMBOL_TINY_GOT ; return SYMBOL_TINY_ABSOLUTE ; case AARCH64_CMODEL_SMALL_SPIC : case AARCH64_CMODEL_SMALL_PIC : if ( ! aarch64_symbol_binds_local_p ( x ) ) return ( aarch64_cmodel == AARCH64_CMODEL_SMALL_SPIC ? SYMBOL_SMALL_GOT_28K : SYMBOL_SMALL_GOT_4G ) ; return SYMBOL_SMALL_ABSOLUTE ; case AARCH64_CMODEL_LARGE : if ( ! aarch64_pcrelative_literal_loads && CONSTANT_POOL_ADDRESS_P ( x ) ) return SYMBOL_SMALL_ABSOLUTE ; else return SYMBOL_FORCE_TO_MEM ; default : gcc_unreachable ( ) ; } } return SYMBOL_FORCE_TO_MEM ; } </s>

Return the method that should be used to access SYMBOL_REF or LABEL_REF X .


<s> static unsigned int aarch64_classify_vector_mode ( machine_mode mode ) { if ( aarch64_advsimd_struct_mode_p ( mode ) ) return VEC_ADVSIMD | VEC_STRUCT ; if ( aarch64_sve_pred_mode_p ( mode ) ) return VEC_SVE_PRED ; switch ( mode ) { case E_VNx2QImode : case E_VNx4QImode : case E_VNx8QImode : case E_VNx2HImode : case E_VNx4HImode : case E_VNx2SImode : case E_VNx2HFmode : case E_VNx4HFmode : case E_VNx2BFmode : case E_VNx4BFmode : case E_VNx2SFmode : return TARGET_SVE ? VEC_SVE_DATA | VEC_PARTIAL : 0 ; case E_VNx16QImode : case E_VNx8HImode : case E_VNx4SImode : case E_VNx2DImode : case E_VNx8BFmode : case E_VNx8HFmode : case E_VNx4SFmode : case E_VNx2DFmode : return TARGET_SVE ? VEC_SVE_DATA : 0 ; case E_VNx32QImode : case E_VNx16HImode : case E_VNx8SImode : case E_VNx4DImode : case E_VNx16BFmode : case E_VNx16HFmode : case E_VNx8SFmode : case E_VNx4DFmode : case E_VNx48QImode : case E_VNx24HImode : case E_VNx12SImode : case E_VNx6DImode : case E_VNx24BFmode : case E_VNx24HFmode : case E_VNx12SFmode : case E_VNx6DFmode : case E_VNx64QImode : case E_VNx32HImode : case E_VNx16SImode : case E_VNx8DImode : case E_VNx32BFmode : case E_VNx32HFmode : case E_VNx16SFmode : case E_VNx8DFmode : return TARGET_SVE ? VEC_SVE_DATA | VEC_STRUCT : 0 ; case E_V8QImode : case E_V4HImode : case E_V2SImode : case E_V4HFmode : case E_V4BFmode : case E_V2SFmode : case E_V1DFmode : case E_V16QImode : case E_V8HImode : case E_V4SImode : case E_V2DImode : case E_V8HFmode : case E_V8BFmode : case E_V4SFmode : case E_V2DFmode : return TARGET_SIMD ? VEC_ADVSIMD : 0 ; default : return 0 ; } } </s>

Return a set of flags describing the vector properties of mode MODE . Ignore modes that are not supported by the current target .


Compare an SVE mode SVE_M and an Advanced SIMD mode ASIMD_M and return whether the SVE mode should be preferred over the Advanced SIMD one in aarch64_autovectorize_vector_modes .


<s> static tree aarch64_comparison_type ( stmt_vec_info stmt_info ) { if ( auto * assign = dyn_cast < gassign * > ( stmt_info -> stmt ) ) if ( TREE_CODE_CLASS ( gimple_assign_rhs_code ( assign ) ) == tcc_comparison ) return TREE_TYPE ( gimple_assign_rhs1 ( assign ) ) ; return aarch64_embedded_comparison_type ( stmt_info ) ; } </s>

If STMT_INFO is a comparison or contains an embedded comparison , return the scalar type of the values being compared . Return null otherwise .


<s>
static
int
aarch64_comp_type_attributes
(
const_tree
type1
,
const_tree
type2
)
{
auto
check_attr
=
[
&
]
(
const
char
*
name
)
{
tree
attr1
=
lookup_attribute
(
name
,
TYPE_ATTRIBUTES
(
type1
)
)
;
tree
attr2
=
lookup_attribute
(
name
,
TYPE_ATTRIBUTES
(
type2
)
)
;
if
(
!
attr1
&&
!
attr2
)
return
true
;
return
attr1
&&
attr2
&&
attribute_value_equal
(
attr1
,
attr2
)
;
}
;
if
(
!
check_attr
(
"aarch64_vector_pcs"
)
)
return
0
;
if
(
!
check_attr
(
"Advanced SIMD type"
)
)
return
0
;
return
1
;
}
</s>


Copy one MODE sized block from SRC to DST , then progress SRC and DST by MODE bytes .


<s> static void aarch64_destroy_cost_data ( void * data ) { delete static_cast < aarch64_vector_costs * > ( data ) ; } </s>


<s> static unsigned int aarch64_detect_scalar_stmt_subtype ( vec_info * vinfo , vect_cost_for_stmt kind , stmt_vec_info stmt_info , unsigned int stmt_cost ) { if ( kind == scalar_stmt && aarch64_extending_load_p ( vinfo , stmt_info ) ) return 0 ; return stmt_cost ; } </s>

STMT_COST is the cost calculated by aarch64_builtin_vectorization_cost for STMT_INFO , which has cost kind KIND . If this is a scalar operation , try to subdivide the target-independent categorization provided by KIND to get a more accurate cost .


<s> static unsigned int aarch64_detect_vector_stmt_subtype ( vec_info * vinfo , vect_cost_for_stmt kind , stmt_vec_info stmt_info , tree vectype , enum vect_cost_model_location where , unsigned int stmt_cost ) { const simd_vec_cost * simd_costs = aarch64_simd_vec_costs ( vectype ) ; const sve_vec_cost * sve_costs = nullptr ; if ( aarch64_sve_mode_p ( TYPE_MODE ( vectype ) ) ) sve_costs = aarch64_tune_params . vec_costs -> sve ; if ( is_a < gphi * > ( stmt_info -> stmt ) ) return 0 ; if ( kind == vec_to_scalar && STMT_VINFO_DATA_REF ( stmt_info ) && DR_IS_WRITE ( STMT_VINFO_DATA_REF ( stmt_info ) ) ) return simd_costs -> store_elt_extra_cost ; if ( kind == scalar_store && sve_costs && STMT_VINFO_MEMORY_ACCESS_TYPE ( stmt_info ) == VMAT_GATHER_SCATTER ) return sve_costs -> scatter_store_elt_cost ; if ( kind == vec_to_scalar && where == vect_body && sve_costs ) { unsigned int latency = aarch64_sve_in_loop_reduction_latency ( vinfo , stmt_info , vectype , sve_costs ) ; if ( latency ) return latency ; } if ( kind == vec_to_scalar && where == vect_epilogue && aarch64_is_reduction ( stmt_info ) ) switch ( GET_MODE_INNER ( TYPE_MODE ( vectype ) ) ) { case E_QImode : return simd_costs -> reduc_i8_cost ; case E_HImode : return simd_costs -> reduc_i16_cost ; case E_SImode : return simd_costs -> reduc_i32_cost ; case E_DImode : return simd_costs -> reduc_i64_cost ; case E_HFmode : case E_BFmode : return simd_costs -> reduc_f16_cost ; case E_SFmode : return simd_costs -> reduc_f32_cost ; case E_DFmode : return simd_costs -> reduc_f64_cost ; default : break ; } return stmt_cost ; } </s>

STMT_COST is the cost calculated by aarch64_builtin_vectorization_cost for the vectorized form of STMT_INFO , which has cost kind KIND and which when vectorized would operate on vector type VECTYPE . Try to subdivide the target-independent categorization provided by KIND to get a more accurate cost . WHERE specifies where the cost associated with KIND occurs .


<s> static tree aarch64_embedded_comparison_type ( stmt_vec_info stmt_info ) { if ( auto * assign = dyn_cast < gassign * > ( stmt_info -> stmt ) ) if ( gimple_assign_rhs_code ( assign ) == COND_EXPR ) { tree cond = gimple_assign_rhs1 ( assign ) ; if ( COMPARISON_CLASS_P ( cond ) ) return TREE_TYPE ( TREE_OPERAND ( cond , 0 ) ) ; } return NULL_TREE ; } </s>

If STMT_INFO is a COND_EXPR that includes an embedded comparison , return the scalar type of the values being compared . Return null otherwise .


<s>
static
void
aarch64_emit_probe_stack_range
(
HOST_WIDE_INT
first
,
poly_int64
poly_size
)
{
HOST_WIDE_INT
size
;
if
(
!
poly_size
.
is_constant
(
&
size
)
)
{
sorry
(
"stack probes for SVE frames"
)
;
return
;
}
rtx
reg1
=
gen_rtx_REG
(
Pmode
,
PROBE_STACK_FIRST_REGNUM
)
;
gcc_assert
(
(
first
%
ARITH_FACTOR
)
==
0
)
;
if
(
size
<=
PROBE_INTERVAL
)
{
const
HOST_WIDE_INT
base
=
ROUND_UP
(
size
,
ARITH_FACTOR
)
;
emit_set_insn
(
reg1
,
plus_constant
(
Pmode
,
stack_pointer_rtx
,
(
first
+
base
)
)
)
;
emit_stack_probe
(
plus_constant
(
Pmode
,
reg1
,
base
size
)
)
;
}
else
if
(
size
<=
4
*
PROBE_INTERVAL
)
{
HOST_WIDE_INT
i
,
rem
;
emit_set_insn
(
reg1
,
plus_constant
(
Pmode
,
stack_pointer_rtx
,
(
first
+
PROBE_INTERVAL
)
)
)
;
emit_stack_probe
(
reg1
)
;
for
(
i
=
2
*
PROBE_INTERVAL
;
i
<
size
;
i
+=
PROBE_INTERVAL
)
{
emit_set_insn
(
reg1
,
plus_constant
(
Pmode
,
reg1
,
PROBE_INTERVAL
)
)
;
emit_stack_probe
(
reg1
)
;
}
rem
=
size
(
i
PROBE_INTERVAL
)
;
if
(
rem
>
256
)
{
const
HOST_WIDE_INT
base
=
ROUND_UP
(
rem
,
ARITH_FACTOR
)
;
emit_set_insn
(
reg1
,
plus_constant
(
Pmode
,
reg1
,
base
)
)
;
emit_stack_probe
(
plus_constant
(
Pmode
,
reg1
,
base
rem
)
)
;
}
else
emit_stack_probe
(
plus_constant
(
Pmode
,
reg1
,
rem
)
)
;
}
else
{
rtx
reg2
=
gen_rtx_REG
(
Pmode
,
PROBE_STACK_SECOND_REGNUM
)
;
HOST_WIDE_INT
rounded_size
=
size
&
PROBE_INTERVAL
;
emit_set_insn
(
reg1
,
plus_constant
(
Pmode
,
stack_pointer_rtx
,
first
)
)
;
HOST_WIDE_INT
adjustment
=
(
first
+
rounded_size
)
;
if
(
!
aarch64_uimm12_shift
(
adjustment
)
)
{
aarch64_internal_mov_immediate
(
reg2
,
GEN_INT
(
adjustment
)
,
true
,
Pmode
)
;
emit_set_insn
(
reg2
,
gen_rtx_PLUS
(
Pmode
,
stack_pointer_rtx
,
reg2
)
)
;
}
else
emit_set_insn
(
reg2
,
plus_constant
(
Pmode
,
stack_pointer_rtx
,
adjustment
)
)
;
emit_insn
(
gen_probe_stack_range
(
reg1
,
reg1
,
reg2
)
)
;
if
(
size
!=
rounded_size
)
{
HOST_WIDE_INT
rem
=
size
rounded_size
;
if
(
rem
>
256
)
{
const
HOST_WIDE_INT
base
=
ROUND_UP
(
rem
,
ARITH_FACTOR
)
;
emit_set_insn
(
reg2
,
plus_constant
(
Pmode
,
reg2
,
base
)
)
;
emit_stack_probe
(
plus_constant
(
Pmode
,
reg2
,
base
rem
)
)
;
}
else
emit_stack_probe
(
plus_constant
(
Pmode
,
reg2
,
rem
)
)
;
}
}
emit_insn
(
gen_blockage
(
)
)
;
}
</s>

Emit code to probe a range of stack addresses from FIRST to FIRST+SIZE , inclusive . These are offsets from the current stack pointer .


<s> static HOST_WIDE_INT aarch64_estimated_poly_value ( poly_int64 val , poly_value_estimate_kind kind = POLY_VALUE_LIKELY ) { enum aarch64_sve_vector_bits_enum width_source = aarch64_tune_params . sve_width ; if ( width_source == SVE_SCALABLE ) switch ( kind ) { case POLY_VALUE_MIN : case POLY_VALUE_LIKELY : return val . coeffs [ 0 ] ; case POLY_VALUE_MAX : return val . coeffs [ 0 ] + val . coeffs [ 1 ] * 15 ; } HOST_WIDE_INT over_128 = width_source - 128 ; return val . coeffs [ 0 ] + val . coeffs [ 1 ] * over_128 / 128 ; } </s>

Implement TARGET_ESTIMATED_POLY_VALUE . Look into the tuning structure for an estimate . KIND specifies the type of requested estimate : min , max or likely . For cores with a known SVE width all three estimates are the same . For generic SVE tuning we want to distinguish the maximum estimate from the minimum and likely ones . The likely estimate is the same as the minimum in that case to give a conservative behavior of auto-vectorizing with SVE when it is a win even for 128-bit SVE . When SVE width information is available VAL.coeffs [ 1 ] is multiplied by the number of VQ chunks over the initial Advanced SIMD 128 bits .


<s> static unsigned int aarch64_estimated_sve_vq ( ) { return estimated_poly_value ( BITS_PER_SVE_VECTOR ) / 128 ; } </s>

Return an estimate for the number of quadwords in an SVE vector . This is equivalent to the number of Advanced SIMD vectors in an SVE vector .


<s> static bool aarch64_evpc_ins ( struct expand_vec_perm_d * d ) { machine_mode mode = d -> vmode ; unsigned HOST_WIDE_INT nelt ; if ( d -> vec_flags != VEC_ADVSIMD ) return false ; nelt = d -> perm . length ( ) . to_constant ( ) ; rtx insv = d -> op0 ; HOST_WIDE_INT idx = - 1 ; for ( unsigned HOST_WIDE_INT i = 0 ; i < nelt ; i ++ ) { HOST_WIDE_INT elt ; if ( ! d -> perm [ i ] . is_constant ( & elt ) ) return false ; if ( elt == ( HOST_WIDE_INT ) i ) continue ; if ( idx != - 1 ) { idx = - 1 ; break ; } idx = i ; } if ( idx == - 1 ) { insv = d -> op1 ; for ( unsigned HOST_WIDE_INT i = 0 ; i < nelt ; i ++ ) { if ( d -> perm [ i ] . to_constant ( ) == ( HOST_WIDE_INT ) ( i + nelt ) ) continue ; if ( idx != - 1 ) return false ; idx = i ; } if ( idx == - 1 ) return false ; } if ( d -> testing_p ) return true ; gcc_assert ( idx != - 1 ) ; unsigned extractindex = d -> perm [ idx ] . to_constant ( ) ; rtx extractv = d -> op0 ; if ( extractindex >= nelt ) { extractv = d -> op1 ; extractindex -= nelt ; } gcc_assert ( extractindex < nelt ) ; emit_move_insn ( d -> target , insv ) ; insn_code icode = code_for_aarch64_simd_vec_copy_lane ( mode ) ; expand_operand ops [ 5 ] ; create_output_operand ( & ops [ 0 ] , d -> target , mode ) ; create_input_operand ( & ops [ 1 ] , d -> target , mode ) ; create_integer_operand ( & ops [ 2 ] , 1 << idx ) ; create_input_operand ( & ops [ 3 ] , extractv , mode ) ; create_integer_operand ( & ops [ 4 ] , extractindex ) ; expand_insn ( icode , 5 , ops ) ; return true ; } </s>

Recognize patterns suitable for the INS instructions .