<s> static bool aarch64_pac_insn_p ( rtx x ) { if ( ! INSN_P ( x ) ) return x ; subrtx_var_iterator :: array_type array ; FOR_EACH_SUBRTX_VAR ( iter , array , PATTERN ( x ) , ALL ) { rtx sub = * iter ; if ( sub && GET_CODE ( sub ) == UNSPEC ) { int unspec_val = XINT ( sub , 1 ) ; switch ( unspec_val ) { case UNSPEC_PACISP : return true ; default : return false ; } iter . skip_subrtxes ( ) ; } } return false ; } </s>

Check if X ( or any sub-rtx of X ) is a PACIASP/PACIBSP instruction .


<s> static unsigned int rest_of_insert_bti ( void ) { timevar_push ( TV_MACH_DEP ) ; rtx bti_insn ; rtx_insn * insn ; basic_block bb ; if ( ! cgraph_node :: get ( cfun -> decl ) -> only_called_directly_p ( ) ) { bb = ENTRY_BLOCK_PTR_FOR_FN ( cfun ) -> next_bb ; insn = BB_HEAD ( bb ) ; if ( ! aarch64_pac_insn_p ( get_first_nonnote_insn ( ) ) ) { bti_insn = gen_bti_c ( ) ; emit_insn_before ( bti_insn , insn ) ; } } bb = 0 ; FOR_EACH_BB_FN ( bb , cfun ) { for ( insn = BB_HEAD ( bb ) ; insn != NEXT_INSN ( BB_END ( bb ) ) ; insn = NEXT_INSN ( insn ) ) { if ( ( LABEL_P ( insn ) && ( LABEL_PRESERVE_P ( insn ) || bb -> flags & BB_NON_LOCAL_GOTO_TARGET ) ) || ( NOTE_P ( insn ) && NOTE_KIND ( insn ) == NOTE_INSN_DELETED_LABEL ) ) { bti_insn = gen_bti_j ( ) ; emit_insn_after ( bti_insn , insn ) ; continue ; } if ( JUMP_P ( insn ) ) { rtx_jump_table_data * table ; if ( tablejump_p ( insn , NULL , & table ) ) { rtvec vec = table -> get_labels ( ) ; int j ; rtx_insn * label ; for ( j = GET_NUM_ELEM ( vec ) - 1 ; j >= 0 ; -- j ) { label = as_a < rtx_insn * > ( XEXP ( RTVEC_ELT ( vec , j ) , 0 ) ) ; bti_insn = gen_bti_j ( ) ; emit_insn_after ( bti_insn , label ) ; } } } if ( CALL_P ( insn ) && ( find_reg_note ( insn , REG_SETJMP , NULL ) ) ) { bti_insn = gen_bti_j ( ) ; emit_insn_after ( bti_insn , insn ) ; continue ; } } } timevar_pop ( TV_MACH_DEP ) ; return 0 ; } </s>

This is implemented as a late RTL pass that runs before branch shortening and does the following .


<s> static bool aarch64_bti_j_insn_p ( rtx_insn * insn ) { if ( ! insn || ! INSN_P ( insn ) ) return false ; rtx pat = PATTERN ( insn ) ; return GET_CODE ( pat ) == UNSPEC_VOLATILE && XINT ( pat , 1 ) == UNSPECV_BTI_J ; } </s>


<s> static bool aarch64_pac_insn_p ( rtx x ) { if ( ! INSN_P ( x ) ) return false ; subrtx_var_iterator :: array_type array ; FOR_EACH_SUBRTX_VAR ( iter , array , PATTERN ( x ) , ALL ) { rtx sub = * iter ; if ( sub && GET_CODE ( sub ) == UNSPEC ) { int unspec_val = XINT ( sub , 1 ) ; switch ( unspec_val ) { case UNSPEC_PACIASP : case UNSPEC_PACIBSP : return true ; default : return false ; } iter . skip_subrtxes ( ) ; } } return false ; } </s>


<s> virtual unsigned int execute ( function * ) { return rest_of_insert_bti ( ) ; } </s>


<s> static unsigned int rest_of_insert_bti ( void ) { timevar_push ( TV_MACH_DEP ) ; rtx bti_insn ; rtx_insn * insn ; basic_block bb ; bb = 0 ; FOR_EACH_BB_FN ( bb , cfun ) { for ( insn = BB_HEAD ( bb ) ; insn != NEXT_INSN ( BB_END ( bb ) ) ; insn = NEXT_INSN ( insn ) ) { if ( LABEL_P ( insn ) && ( LABEL_PRESERVE_P ( insn ) || bb -> flags & BB_NON_LOCAL_GOTO_TARGET ) ) { bti_insn = gen_bti_j ( ) ; emit_insn_after ( bti_insn , insn ) ; continue ; } if ( JUMP_P ( insn ) ) { rtx_jump_table_data * table ; if ( tablejump_p ( insn , NULL , & table ) ) { rtvec vec = table -> get_labels ( ) ; int j ; rtx_insn * label ; for ( j = GET_NUM_ELEM ( vec ) - 1 ; j >= 0 ; -- j ) { label = as_a < rtx_insn * > ( XEXP ( RTVEC_ELT ( vec , j ) , 0 ) ) ; rtx_insn * next = next_nonnote_nondebug_insn ( label ) ; if ( aarch64_bti_j_insn_p ( next ) ) continue ; bti_insn = gen_bti_j ( ) ; emit_insn_after ( bti_insn , label ) ; } } } if ( CALL_P ( insn ) && ( find_reg_note ( insn , REG_SETJMP , NULL ) ) ) { bti_insn = gen_bti_j ( ) ; emit_insn_after ( bti_insn , insn ) ; continue ; } } } if ( ! cgraph_node :: get ( cfun -> decl ) -> only_called_directly_p ( ) ) { bb = ENTRY_BLOCK_PTR_FOR_FN ( cfun ) -> next_bb ; insn = BB_HEAD ( bb ) ; if ( ! aarch64_pac_insn_p ( get_first_nonnote_insn ( ) ) ) { bti_insn = gen_bti_c ( ) ; emit_insn_before ( bti_insn , insn ) ; } } timevar_pop ( TV_MACH_DEP ) ; return 0 ; } </s>


<s> static unsigned int aarch64_call_properties ( aarch64_simd_builtin_datum * d ) { unsigned int flags = d -> flags ; if ( ! ( flags & FLAG_AUTO_FP ) && FLOAT_MODE_P ( d -> mode ) ) flags |= FLAG_FP ; if ( ! flag_trapping_math ) flags &= ~ FLAG_RAISE_FP_EXCEPTIONS ; return flags ; } </s>

Return a set of FLAG_ * flags that describe what the function could do , taking the command-line flags into account .


<s> static bool aarch64_could_trap_p ( aarch64_simd_builtin_datum * d ) { unsigned int flags = aarch64_call_properties ( d ) ; if ( flags & FLAG_RAISE_FP_EXCEPTIONS ) return true ; if ( flags & ( FLAG_READ_MEMORY | FLAG_WRITE_MEMORY ) ) return true ; return false ; } </s>

Return true if calls to the function could raise a signal .


<s> static rtx aarch64_expand_builtin_data_intrinsic ( unsigned int fcode , tree exp , rtx target ) { expand_operand ops [ 2 ] ; machine_mode mode = TYPE_MODE ( TREE_TYPE ( exp ) ) ; create_output_operand ( & ops [ 0 ] , target , mode ) ; create_input_operand ( & ops [ 1 ] , expand_normal ( CALL_EXPR_ARG ( exp , 0 ) ) , mode ) ; enum insn_code icode ; switch ( fcode ) { case AARCH64_REV16 : case AARCH64_REV16L : case AARCH64_REV16LL : icode = code_for_aarch64_rev16 ( mode ) ; break ; case AARCH64_RBIT : case AARCH64_RBITL : case AARCH64_RBITLL : icode = code_for_aarch64_rbit ( mode ) ; break ; default : gcc_unreachable ( ) ; } expand_insn ( icode , 2 , ops ) ; return ops [ 0 ] . value ; } </s>

Function to expand an expression EXP which calls one of the ACLE Data Intrinsic builtins FCODE with the result going to TARGET .


"%Kargument %d must be a constant immediate"

Expand a FCMLA lane expression EXP with code FCODE and result going to TARGET if that is convenient .


<s> tree aarch64_general_fold_builtin ( unsigned int fcode , tree type , unsigned int n_args ATTRIBUTE_UNUSED , tree * args ) { switch ( fcode ) { BUILTIN_VDQF ( UNOP , abs , 2 , ALL ) return fold_build1 ( ABS_EXPR , type , args [ 0 ] ) ; VAR1 ( UNOP , floatv2si , 2 , ALL , v2sf ) VAR1 ( UNOP , floatv4si , 2 , ALL , v4sf ) VAR1 ( UNOP , floatv2di , 2 , ALL , v2df ) return fold_build1 ( FLOAT_EXPR , type , args [ 0 ] ) ; AARCH64_SIMD_VREINTERPRET_BUILTINS return fold_build1 ( VIEW_CONVERT_EXPR , type , args [ 0 ] ) ; case AARCH64_SIMD_BUILTIN_LANE_CHECK : gcc_assert ( n_args == 3 ) ; if ( aarch64_fold_builtin_lane_check ( args [ 0 ] , args [ 1 ] , args [ 2 ] ) ) return void_node ; break ; default : break ; } return NULL_TREE ; } </s>

Try to fold a call to the built-in function with subcode FCODE . The function is passed the N_ARGS arguments in ARGS and it returns a value of type TYPE . Return the new expression on success and NULL_TREE on failure .


<s> void aarch64_general_init_builtins ( void ) { aarch64_init_fpsr_fpcr_builtins ( ) ; aarch64_init_fp16_types ( ) ; aarch64_init_bf16_types ( ) ; { aarch64_simd_switcher simd ; aarch64_init_simd_builtins ( ) ; } aarch64_init_crc32_builtins ( ) ; aarch64_init_builtin_rsqrt ( ) ; aarch64_init_rng_builtins ( ) ; aarch64_init_data_intrinsics ( ) ; tree ftype_jcvt = build_function_type_list ( intSI_type_node , double_type_node , NULL ) ; aarch64_builtin_decls [ AARCH64_JSCVT ] = aarch64_general_add_builtin ( "__builtin_aarch64_jcvtzs" , ftype_jcvt , AARCH64_JSCVT ) ; if ( ! TARGET_ILP32 ) aarch64_init_pauth_hint_builtins ( ) ; if ( TARGET_TME ) aarch64_init_tme_builtins ( ) ; if ( TARGET_MEMTAG ) aarch64_init_memtag_builtins ( ) ; } </s>

Initialize all builtins in the AARCH64_BUILTIN_GENERAL group .


<s> static tree aarch64_get_attributes ( aarch64_simd_builtin_datum * d ) { tree attrs = NULL_TREE ; if ( ! aarch64_modifies_global_state_p ( d ) ) { if ( aarch64_reads_global_state_p ( d ) ) attrs = aarch64_add_attribute ( "pure" , attrs ) ; else attrs = aarch64_add_attribute ( "const" , attrs ) ; } if ( ! flag_non_call_exceptions || ! aarch64_could_trap_p ( d ) ) attrs = aarch64_add_attribute ( "nothrow" , attrs ) ; return aarch64_add_attribute ( "leaf" , attrs ) ; } </s>

Return the appropriate function attributes .


<s> static void aarch64_init_bf16_types ( void ) { lang_hooks . types . register_builtin_type ( bfloat16_type_node , "__bf16" ) ; aarch64_bf16_ptr_type_node = build_pointer_type ( bfloat16_type_node ) ; } </s>

Initialize the backend REAL_TYPE type supporting bfloat types .


<s> void aarch64_init_builtin_rsqrt ( void ) { tree fndecl = NULL ; tree ftype = NULL ; tree V2SF_type_node = build_vector_type ( float_type_node , 2 ) ; tree V2DF_type_node = build_vector_type ( double_type_node , 2 ) ; tree V4SF_type_node = build_vector_type ( float_type_node , 4 ) ; struct builtin_decls_data { tree type_node ; const char * builtin_name ; int function_code ; } ; builtin_decls_data bdda [ ] = { { double_type_node , "__builtin_aarch64_rsqrt_df" , AARCH64_BUILTIN_RSQRT_DF } , { float_type_node , "__builtin_aarch64_rsqrt_sf" , AARCH64_BUILTIN_RSQRT_SF } , { V2DF_type_node , "__builtin_aarch64_rsqrt_v2df" , AARCH64_BUILTIN_RSQRT_V2DF } , { V2SF_type_node , "__builtin_aarch64_rsqrt_v2sf" , AARCH64_BUILTIN_RSQRT_V2SF } , { V4SF_type_node , "__builtin_aarch64_rsqrt_v4sf" , AARCH64_BUILTIN_RSQRT_V4SF } } ; builtin_decls_data * bdd = bdda ; builtin_decls_data * bdd_end = bdd + ( ARRAY_SIZE ( bdda ) ) ; for ( ; bdd < bdd_end ; bdd ++ ) { ftype = build_function_type_list ( bdd -> type_node , bdd -> type_node , NULL_TREE ) ; tree attrs = aarch64_get_attributes ( FLAG_FP , TYPE_MODE ( bdd -> type_node ) ) ; fndecl = aarch64_general_add_builtin ( bdd -> builtin_name , ftype , bdd -> function_code , attrs ) ; aarch64_builtin_decls [ bdd -> function_code ] = fndecl ; } } </s>

Add builtins for reciprocal square root .


<s> void aarch64_init_fcmla_laneq_builtins ( void ) { unsigned int i = 0 ; for ( i = 0 ; i < ARRAY_SIZE ( aarch64_fcmla_lane_builtin_data ) ; ++ i ) { aarch64_fcmla_laneq_builtin_datum * d = & aarch64_fcmla_lane_builtin_data [ i ] ; tree argtype = aarch64_simd_builtin_type ( d -> mode , qualifier_none ) ; machine_mode quadmode = GET_MODE_2XWIDER_MODE ( d -> mode ) . require ( ) ; tree quadtype = aarch64_simd_builtin_type ( quadmode , qualifier_none ) ; tree lanetype = aarch64_simd_builtin_type ( SImode , qualifier_lane_pair_index ) ; tree ftype = build_function_type_list ( argtype , argtype , argtype , quadtype , lanetype , NULL_TREE ) ; tree attrs = aarch64_get_attributes ( FLAG_FP , d -> mode ) ; tree fndecl = aarch64_general_add_builtin ( d -> name , ftype , d -> fcode , attrs ) ; aarch64_builtin_decls [ d -> fcode ] = fndecl ; } } </s>

Due to the architecture not providing lane variant of the lane instructions for fcmla we ca n't use the standard simd builtin expansion code , but we still want the majority of the validation that would normally be done .


<s> static tree aarch64_int_or_fp_type ( machine_mode mode , enum aarch64_type_qualifiers qualifiers ) { ? unsigned_int ## M ## _type_node : int ## M ## _type_node ) ; switch ( mode ) { case E_QImode : return QUAL_TYPE ( QI ) ; case E_HImode : return QUAL_TYPE ( HI ) ; case E_SImode : return QUAL_TYPE ( SI ) ; case E_DImode : return QUAL_TYPE ( DI ) ; case E_TImode : return QUAL_TYPE ( TI ) ; case E_OImode : return aarch64_simd_intOI_type_node ; case E_CImode : return aarch64_simd_intCI_type_node ; case E_XImode : return aarch64_simd_intXI_type_node ; case E_HFmode : return aarch64_fp16_type_node ; case E_SFmode : return float_type_node ; case E_DFmode : return double_type_node ; case E_BFmode : return bfloat16_type_node ; default : gcc_unreachable ( ) ; } } </s>

Helper function for aarch64_simd_builtin_type .


<s> static tree aarch64_lookup_simd_type_in_table ( machine_mode mode , enum aarch64_type_qualifiers qualifiers ) { int i ; int nelts = ARRAY_SIZE ( aarch64_simd_types ) ; int q = qualifiers & ( qualifier_poly | qualifier_unsigned ) ; for ( i = 0 ; i < nelts ; i ++ ) { if ( aarch64_simd_types [ i ] . mode == mode && aarch64_simd_types [ i ] . q == q ) return aarch64_simd_types [ i ] . itype ; if ( aarch64_simd_tuple_types [ i ] [ 0 ] != NULL_TREE ) for ( int j = 0 ; j < 3 ; j ++ ) if ( aarch64_simd_tuple_modes [ i ] [ j ] == mode && aarch64_simd_types [ i ] . q == q ) return aarch64_simd_tuple_types [ i ] [ j ] ; } return NULL_TREE ; } </s>


<s> static bool aarch64_modifies_global_state_p ( aarch64_simd_builtin_datum * d ) { unsigned int flags = aarch64_call_properties ( d ) ; if ( flags & FLAG_RAISE_FP_EXCEPTIONS ) return true ; if ( flags & FLAG_PREFETCH_MEMORY ) return true ; return flags & FLAG_WRITE_MEMORY ; } </s>

Return true if calls to the function could modify some form of global state .


<s> static bool aarch64_reads_global_state_p ( aarch64_simd_builtin_datum * d ) { unsigned int flags = aarch64_call_properties ( d ) ; if ( flags & FLAG_READ_FPCR ) return true ; return flags & FLAG_READ_MEMORY ; } </s>

Return true if calls to the function could read some form of global state .


<s> static void aarch64_record_vector_load_arg ( tree addr ) { tree decl = aarch64_vector_load_decl ( addr ) ; if ( ! decl ) return ; if ( ! cfun -> machine -> vector_load_decls ) cfun -> machine -> vector_load_decls = hash_set < tree > :: create_ggc ( 31 ) ; cfun -> machine -> vector_load_decls -> add ( decl ) ; } </s>

We 've seen a vector load from address ADDR . Record it in vector_load_decls , if appropriate .


<s> static tree aarch64_simd_builtin_type ( machine_mode mode , enum aarch64_type_qualifiers qualifiers ) { tree type = NULL_TREE ; if ( ( qualifiers & qualifier_pointer ) && VECTOR_MODE_P ( mode ) ) mode = GET_MODE_INNER ( mode ) ; if ( ( qualifiers & qualifier_poly ) || VECTOR_MODE_P ( mode ) ) type = aarch64_lookup_simd_type_in_table ( mode , qualifiers ) ; else type = aarch64_int_or_fp_type ( mode , qualifiers ) ; gcc_assert ( type != NULL_TREE ) ; if ( qualifiers & qualifier_const ) type = build_qualified_type ( type , TYPE_QUAL_CONST ) ; if ( qualifiers & qualifier_pointer ) type = build_pointer_type ( type ) ; return type ; } </s>

Return a type for an operand with specified mode and qualifiers .


Enable AARCH64_FL_ * flags EXTRA_FLAGS on top of the base Advanced SIMD set .


<s> void handle_arm_neon_h ( void ) { aarch64_simd_switcher simd ; for ( unsigned int i = 0 ; i < ARM_NEON_H_TYPES_LAST ; i ++ ) for ( unsigned int count = 2 ; count <= 4 ; ++ count ) if ( ! aarch64_scalar_builtin_type_p ( aarch64_simd_types [ i ] . type ) ) register_tuple_type ( count , i ) ; aarch64_init_simd_builtin_functions ( true ) ; aarch64_init_simd_intrinsics ( ) ; } </s>

Implement # pragma GCC aarch64 `` arm_neon.h '' . The types and functions defined here need to be available internally during LTO as well .


<s> tree aarch64_builtin_decl ( unsigned code , bool initialize_p ATTRIBUTE_UNUSED ) { if ( code >= AARCH64_BUILTIN_MAX ) return error_mark_node ; return aarch64_builtin_decls [ code ] ; } </s>


<s> rtx aarch64_expand_builtin ( tree exp , rtx target , rtx subtarget ATTRIBUTE_UNUSED , machine_mode mode ATTRIBUTE_UNUSED , int ignore ATTRIBUTE_UNUSED ) { tree fndecl = TREE_OPERAND ( CALL_EXPR_FN ( exp ) , 0 ) ; int fcode = DECL_FUNCTION_CODE ( fndecl ) ; int icode ; rtx pat , op0 ; tree arg0 ; switch ( fcode ) { case AARCH64_BUILTIN_GET_FPCR : case AARCH64_BUILTIN_SET_FPCR : case AARCH64_BUILTIN_GET_FPSR : case AARCH64_BUILTIN_SET_FPSR : if ( ( fcode == AARCH64_BUILTIN_GET_FPCR ) || ( fcode == AARCH64_BUILTIN_GET_FPSR ) ) { icode = ( fcode == AARCH64_BUILTIN_GET_FPSR ) ? CODE_FOR_get_fpsr : CODE_FOR_get_fpcr ; target = gen_reg_rtx ( SImode ) ; pat = GEN_FCN ( icode ) ( target ) ; } else { target = NULL_RTX ; icode = ( fcode == AARCH64_BUILTIN_SET_FPSR ) ? CODE_FOR_set_fpsr : CODE_FOR_set_fpcr ; arg0 = CALL_EXPR_ARG ( exp , 0 ) ; op0 = expand_normal ( arg0 ) ; pat = GEN_FCN ( icode ) ( op0 ) ; } emit_insn ( pat ) ; return target ; } if ( fcode >= AARCH64_SIMD_BUILTIN_BASE && fcode <= AARCH64_SIMD_BUILTIN_MAX ) return aarch64_simd_expand_builtin ( fcode , exp , target ) ; else if ( fcode >= AARCH64_CRC32_BUILTIN_BASE && fcode <= AARCH64_CRC32_BUILTIN_MAX ) return aarch64_crc32_expand_builtin ( fcode , exp , target ) ; gcc_unreachable ( ) ; } </s>


<s> tree aarch64_fold_builtin ( tree fndecl , int n_args ATTRIBUTE_UNUSED , tree * args , bool ignore ATTRIBUTE_UNUSED ) { int fcode = DECL_FUNCTION_CODE ( fndecl ) ; tree type = TREE_TYPE ( TREE_TYPE ( fndecl ) ) ; switch ( fcode ) { BUILTIN_VDQF ( UNOP , abs , 2 ) return fold_build1 ( ABS_EXPR , type , args [ 0 ] ) ; break ; VAR1 ( UNOP , floatv2si , 2 , v2sf ) VAR1 ( UNOP , floatv4si , 2 , v4sf ) VAR1 ( UNOP , floatv2di , 2 , v2df ) return fold_build1 ( FLOAT_EXPR , type , args [ 0 ] ) ; default : break ; } return NULL_TREE ; } </s>


<s> bool aarch64_gimple_fold_builtin ( gimple_stmt_iterator * gsi ) { bool changed = false ; gimple stmt = gsi_stmt ( * gsi ) ; tree call = gimple_call_fn ( stmt ) ; tree fndecl ; gimple new_stmt = NULL ; if ( call ) { fndecl = gimple_call_fndecl ( stmt ) ; if ( fndecl ) { int fcode = DECL_FUNCTION_CODE ( fndecl ) ; int nargs = gimple_call_num_args ( stmt ) ; tree * args = ( nargs > 0 ? gimple_call_arg_ptr ( stmt , 0 ) : & error_mark_node ) ; switch ( fcode ) { BUILTIN_VALL ( UNOP , reduc_plus_scal_ , 10 ) new_stmt = gimple_build_assign ( gimple_call_lhs ( stmt ) , REDUC_PLUS_EXPR , args [ 0 ] ) ; break ; BUILTIN_VDQIF ( UNOP , reduc_smax_scal_ , 10 ) BUILTIN_VDQ_BHSI ( UNOPU , reduc_umax_scal_ , 10 ) new_stmt = gimple_build_assign ( gimple_call_lhs ( stmt ) , REDUC_MAX_EXPR , args [ 0 ] ) ; break ; BUILTIN_VDQIF ( UNOP , reduc_smin_scal_ , 10 ) BUILTIN_VDQ_BHSI ( UNOPU , reduc_umin_scal_ , 10 ) new_stmt = gimple_build_assign ( gimple_call_lhs ( stmt ) , REDUC_MIN_EXPR , args [ 0 ] ) ; break ; default : break ; } } } if ( new_stmt ) { gsi_replace ( gsi , new_stmt , true ) ; changed = true ; } return changed ; } </s>


<s> static tree aarch64_simd_builtin_type ( enum machine_mode mode , bool unsigned_p , bool poly_p ) { if ( poly_p ) return aarch64_lookup_simd_builtin_type ( mode , qualifier_poly ) ; else if ( unsigned_p ) return aarch64_lookup_simd_builtin_type ( mode , qualifier_unsigned ) ; else return aarch64_lookup_simd_builtin_type ( mode , qualifier_none ) ; } </s>


"%Klane index must be a constant immediate"

"%Ktotal size and element size must be a non-zero constant immediate"

Expand an AArch64 AdvSIMD builtin ( intrinsic ) .


<s> tree aarch64_builtin_rsqrt ( unsigned int fn ) { if ( fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv2df ) return aarch64_builtin_decls [ AARCH64_BUILTIN_RSQRT_V2DF ] ; if ( fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv2sf ) return aarch64_builtin_decls [ AARCH64_BUILTIN_RSQRT_V2SF ] ; if ( fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv4sf ) return aarch64_builtin_decls [ AARCH64_BUILTIN_RSQRT_V4SF ] ; return NULL_TREE ; } </s>

Return builtin for reciprocal square root .


<s> rtx aarch64_expand_builtin ( tree exp , rtx target , rtx subtarget ATTRIBUTE_UNUSED , machine_mode mode ATTRIBUTE_UNUSED , int ignore ATTRIBUTE_UNUSED ) { tree fndecl = TREE_OPERAND ( CALL_EXPR_FN ( exp ) , 0 ) ; int fcode = DECL_FUNCTION_CODE ( fndecl ) ; int icode ; rtx pat , op0 ; tree arg0 ; switch ( fcode ) { case AARCH64_BUILTIN_GET_FPCR : case AARCH64_BUILTIN_SET_FPCR : case AARCH64_BUILTIN_GET_FPSR : case AARCH64_BUILTIN_SET_FPSR : if ( ( fcode == AARCH64_BUILTIN_GET_FPCR ) || ( fcode == AARCH64_BUILTIN_GET_FPSR ) ) { icode = ( fcode == AARCH64_BUILTIN_GET_FPSR ) ? CODE_FOR_get_fpsr : CODE_FOR_get_fpcr ; target = gen_reg_rtx ( SImode ) ; pat = GEN_FCN ( icode ) ( target ) ; } else { target = NULL_RTX ; icode = ( fcode == AARCH64_BUILTIN_SET_FPSR ) ? CODE_FOR_set_fpsr : CODE_FOR_set_fpcr ; arg0 = CALL_EXPR_ARG ( exp , 0 ) ; op0 = force_reg ( SImode , expand_normal ( arg0 ) ) ; pat = GEN_FCN ( icode ) ( op0 ) ; } emit_insn ( pat ) ; return target ; } if ( fcode >= AARCH64_SIMD_BUILTIN_BASE && fcode <= AARCH64_SIMD_BUILTIN_MAX ) return aarch64_simd_expand_builtin ( fcode , exp , target ) ; else if ( fcode >= AARCH64_CRC32_BUILTIN_BASE && fcode <= AARCH64_CRC32_BUILTIN_MAX ) return aarch64_crc32_expand_builtin ( fcode , exp , target ) ; if ( fcode == AARCH64_BUILTIN_RSQRT_DF || fcode == AARCH64_BUILTIN_RSQRT_SF || fcode == AARCH64_BUILTIN_RSQRT_V2DF || fcode == AARCH64_BUILTIN_RSQRT_V2SF || fcode == AARCH64_BUILTIN_RSQRT_V4SF ) return aarch64_expand_builtin_rsqrt ( fcode , exp , target ) ; gcc_unreachable ( ) ; } </s>


<s> void aarch64_init_builtin_rsqrt ( void ) { tree fndecl = NULL ; tree ftype = NULL ; tree V2SF_type_node = build_vector_type ( float_type_node , 2 ) ; tree V2DF_type_node = build_vector_type ( double_type_node , 2 ) ; tree V4SF_type_node = build_vector_type ( float_type_node , 4 ) ; struct builtin_decls_data { tree type_node ; const char * builtin_name ; int function_code ; } ; builtin_decls_data bdda [ ] = { { double_type_node , "__builtin_aarch64_rsqrt_df" , AARCH64_BUILTIN_RSQRT_DF } , { float_type_node , "__builtin_aarch64_rsqrt_sf" , AARCH64_BUILTIN_RSQRT_SF } , { V2DF_type_node , "__builtin_aarch64_rsqrt_v2df" , AARCH64_BUILTIN_RSQRT_V2DF } , { V2SF_type_node , "__builtin_aarch64_rsqrt_v2sf" , AARCH64_BUILTIN_RSQRT_V2SF } , { V4SF_type_node , "__builtin_aarch64_rsqrt_v4sf" , AARCH64_BUILTIN_RSQRT_V4SF } } ; builtin_decls_data * bdd = bdda ; builtin_decls_data * bdd_end = bdd + ( sizeof ( bdda ) / sizeof ( builtin_decls_data ) ) ; for ( ; bdd < bdd_end ; bdd ++ ) { ftype = build_function_type_list ( bdd -> type_node , bdd -> type_node , NULL_TREE ) ; fndecl = add_builtin_function ( bdd -> builtin_name , ftype , bdd -> function_code , BUILT_IN_MD , NULL , NULL_TREE ) ; aarch64_builtin_decls [ bdd -> function_code ] = fndecl ; } } </s>


<s> rtx aarch64_expand_builtin ( tree exp , rtx target , rtx subtarget ATTRIBUTE_UNUSED , machine_mode mode ATTRIBUTE_UNUSED , int ignore ATTRIBUTE_UNUSED ) { tree fndecl = TREE_OPERAND ( CALL_EXPR_FN ( exp ) , 0 ) ; int fcode = DECL_FUNCTION_CODE ( fndecl ) ; int icode ; rtx pat , op0 ; tree arg0 ; switch ( fcode ) { case AARCH64_BUILTIN_GET_FPCR : case AARCH64_BUILTIN_SET_FPCR : case AARCH64_BUILTIN_GET_FPSR : case AARCH64_BUILTIN_SET_FPSR : if ( ( fcode == AARCH64_BUILTIN_GET_FPCR ) || ( fcode == AARCH64_BUILTIN_GET_FPSR ) ) { icode = ( fcode == AARCH64_BUILTIN_GET_FPSR ) ? CODE_FOR_get_fpsr : CODE_FOR_get_fpcr ; target = gen_reg_rtx ( SImode ) ; pat = GEN_FCN ( icode ) ( target ) ; } else { target = NULL_RTX ; icode = ( fcode == AARCH64_BUILTIN_SET_FPSR ) ? CODE_FOR_set_fpsr : CODE_FOR_set_fpcr ; arg0 = CALL_EXPR_ARG ( exp , 0 ) ; op0 = force_reg ( SImode , expand_normal ( arg0 ) ) ; pat = GEN_FCN ( icode ) ( op0 ) ; } emit_insn ( pat ) ; return target ; case AARCH64_PAUTH_BUILTIN_AUTIA1716 : case AARCH64_PAUTH_BUILTIN_PACIA1716 : case AARCH64_PAUTH_BUILTIN_XPACLRI : arg0 = CALL_EXPR_ARG ( exp , 0 ) ; op0 = force_reg ( Pmode , expand_normal ( arg0 ) ) ; if ( ! target ) target = gen_reg_rtx ( Pmode ) ; else target = force_reg ( Pmode , target ) ; emit_move_insn ( target , op0 ) ; if ( fcode == AARCH64_PAUTH_BUILTIN_XPACLRI ) { rtx lr = gen_rtx_REG ( Pmode , R30_REGNUM ) ; icode = CODE_FOR_xpaclri ; emit_move_insn ( lr , op0 ) ; emit_insn ( GEN_FCN ( icode ) ( ) ) ; emit_move_insn ( target , lr ) ; } else { tree arg1 = CALL_EXPR_ARG ( exp , 1 ) ; rtx op1 = force_reg ( Pmode , expand_normal ( arg1 ) ) ; icode = ( fcode == AARCH64_PAUTH_BUILTIN_PACIA1716 ? CODE_FOR_paci1716 : CODE_FOR_auti1716 ) ; rtx x16_reg = gen_rtx_REG ( Pmode , R16_REGNUM ) ; rtx x17_reg = gen_rtx_REG ( Pmode , R17_REGNUM ) ; emit_move_insn ( x17_reg , op0 ) ; emit_move_insn ( x16_reg , op1 ) ; emit_insn ( GEN_FCN ( icode ) ( ) ) ; emit_move_insn ( target , x17_reg ) ; } return target ; } if ( fcode >= AARCH64_SIMD_BUILTIN_BASE && fcode <= AARCH64_SIMD_BUILTIN_MAX ) return aarch64_simd_expand_builtin ( fcode , exp , target ) ; else if ( fcode >= AARCH64_CRC32_BUILTIN_BASE && fcode <= AARCH64_CRC32_BUILTIN_MAX ) return aarch64_crc32_expand_builtin ( fcode , exp , target ) ; if ( fcode == AARCH64_BUILTIN_RSQRT_DF || fcode == AARCH64_BUILTIN_RSQRT_SF || fcode == AARCH64_BUILTIN_RSQRT_V2DF || fcode == AARCH64_BUILTIN_RSQRT_V2SF || fcode == AARCH64_BUILTIN_RSQRT_V4SF ) return aarch64_expand_builtin_rsqrt ( fcode , exp , target ) ; gcc_unreachable ( ) ; } </s>


<s> tree aarch64_fold_builtin ( tree fndecl , int n_args ATTRIBUTE_UNUSED , tree * args , bool ignore ATTRIBUTE_UNUSED ) { int fcode = DECL_FUNCTION_CODE ( fndecl ) ; tree type = TREE_TYPE ( TREE_TYPE ( fndecl ) ) ; switch ( fcode ) { BUILTIN_VDQF ( UNOP , abs , 2 ) return fold_build1 ( ABS_EXPR , type , args [ 0 ] ) ; VAR1 ( UNOP , floatv2si , 2 , v2sf ) VAR1 ( UNOP , floatv4si , 2 , v4sf ) VAR1 ( UNOP , floatv2di , 2 , v2df ) return fold_build1 ( FLOAT_EXPR , type , args [ 0 ] ) ; default : break ; } return NULL_TREE ; } </s>


Pointer authentication builtins that will become NOP on legacy platform . Currently , these builtins are for internal use only ( libgcc EH unwinder ) .


<s> void aarch64_init_fcmla_laneq_builtins ( void ) { unsigned int i = 0 ; for ( i = 0 ; i < ARRAY_SIZE ( aarch64_fcmla_lane_builtin_data ) ; ++ i ) { aarch64_fcmla_laneq_builtin_datum * d = & aarch64_fcmla_lane_builtin_data [ i ] ; tree argtype = aarch64_lookup_simd_builtin_type ( d -> mode , qualifier_none ) ; machine_mode quadmode = GET_MODE_2XWIDER_MODE ( d -> mode ) . require ( ) ; tree quadtype = aarch64_lookup_simd_builtin_type ( quadmode , qualifier_none ) ; tree lanetype = aarch64_simd_builtin_std_type ( SImode , qualifier_lane_pair_index ) ; tree ftype = build_function_type_list ( argtype , argtype , argtype , quadtype , lanetype , NULL_TREE ) ; tree fndecl = add_builtin_function ( d -> name , ftype , d -> fcode , BUILT_IN_MD , NULL , NULL_TREE ) ; aarch64_builtin_decls [ d -> fcode ] = fndecl ; } } </s>


<s> static tree aarch64_add_attribute ( const char * name , tree attrs ) { return tree_cons ( get_identifier ( name ) , NULL_TREE , attrs ) ; } </s>


<s> static unsigned int aarch64_call_properties ( unsigned int flags , machine_mode mode ) { if ( ! ( flags & FLAG_AUTO_FP ) && FLOAT_MODE_P ( mode ) ) flags |= FLAG_FP ; if ( ! flag_trapping_math ) flags &= ~ FLAG_RAISE_FP_EXCEPTIONS ; return flags ; } </s>

Return a set of FLAG_ * flags that describe what the function could do , taking the command-line flags into account .


<s> static bool aarch64_could_trap_p ( unsigned int f , machine_mode mode ) { unsigned int flags = aarch64_call_properties ( f , mode ) ; if ( flags & FLAG_RAISE_FP_EXCEPTIONS ) return true ; if ( flags & ( FLAG_READ_MEMORY | FLAG_WRITE_MEMORY ) ) return true ; return false ; } </s>

Return true if calls to the function could raise a signal .


<s> static rtx aarch64_expand_builtin_ls64 ( int fcode , tree exp , rtx target ) { expand_operand ops [ 3 ] ; switch ( fcode ) { case AARCH64_LS64_BUILTIN_LD64B : { rtx op0 = expand_normal ( CALL_EXPR_ARG ( exp , 0 ) ) ; create_output_operand ( & ops [ 0 ] , target , V8DImode ) ; create_input_operand ( & ops [ 1 ] , op0 , DImode ) ; expand_insn ( CODE_FOR_ld64b , 2 , ops ) ; return ops [ 0 ] . value ; } case AARCH64_LS64_BUILTIN_ST64B : { rtx op0 = expand_normal ( CALL_EXPR_ARG ( exp , 0 ) ) ; rtx op1 = expand_normal ( CALL_EXPR_ARG ( exp , 1 ) ) ; create_output_operand ( & ops [ 0 ] , op0 , DImode ) ; create_input_operand ( & ops [ 1 ] , op1 , V8DImode ) ; expand_insn ( CODE_FOR_st64b , 2 , ops ) ; return const0_rtx ; } case AARCH64_LS64_BUILTIN_ST64BV : { rtx op0 = expand_normal ( CALL_EXPR_ARG ( exp , 0 ) ) ; rtx op1 = expand_normal ( CALL_EXPR_ARG ( exp , 1 ) ) ; create_output_operand ( & ops [ 0 ] , target , DImode ) ; create_input_operand ( & ops [ 1 ] , op0 , DImode ) ; create_input_operand ( & ops [ 2 ] , op1 , V8DImode ) ; expand_insn ( CODE_FOR_st64bv , 3 , ops ) ; return ops [ 0 ] . value ; } case AARCH64_LS64_BUILTIN_ST64BV0 : { rtx op0 = expand_normal ( CALL_EXPR_ARG ( exp , 0 ) ) ; rtx op1 = expand_normal ( CALL_EXPR_ARG ( exp , 1 ) ) ; create_output_operand ( & ops [ 0 ] , target , DImode ) ; create_input_operand ( & ops [ 1 ] , op0 , DImode ) ; create_input_operand ( & ops [ 2 ] , op1 , V8DImode ) ; expand_insn ( CODE_FOR_st64bv0 , 3 , ops ) ; return ops [ 0 ] . value ; } } gcc_unreachable ( ) ; } </s>

Function to expand an expression EXP which calls one of the Load/Store 64 Byte extension ( LS64 ) builtins FCODE with the result going to TARGET .


"Memory Tagging Extension does not support %<-mabi=ilp32%>"

"argument %d must be a constant immediate "

Expand an expression EXP that calls a MEMTAG built-in FCODE with result going to TARGET .


<s> static rtx aarch64_expand_builtin_rsqrt ( int fcode , tree exp , rtx target ) { tree arg0 = CALL_EXPR_ARG ( exp , 0 ) ; rtx op0 = expand_normal ( arg0 ) ; rtx ( * gen ) ( rtx , rtx ) ; switch ( fcode ) { case AARCH64_BUILTIN_RSQRT_DF : gen = gen_rsqrtdf2 ; break ; case AARCH64_BUILTIN_RSQRT_SF : gen = gen_rsqrtsf2 ; break ; case AARCH64_BUILTIN_RSQRT_V2DF : gen = gen_rsqrtv2df2 ; break ; case AARCH64_BUILTIN_RSQRT_V2SF : gen = gen_rsqrtv2sf2 ; break ; case AARCH64_BUILTIN_RSQRT_V4SF : gen = gen_rsqrtv4sf2 ; break ; default : gcc_unreachable ( ) ; } if ( ! target ) target = gen_reg_rtx ( GET_MODE ( op0 ) ) ; emit_insn ( gen ( target , op0 ) ) ; return target ; } </s>

Function to expand reciprocal square root builtins .


"argument must be a 16-bit constant immediate"

Function to expand an expression EXP which calls one of the Transactional Memory Extension ( TME ) builtins FCODE with the result going to TARGET .


"argument %d must be a constant immediate"

Expand a FCMLA lane expression EXP with code FCODE and result going to TARGET if that is convenient .


<s> static rtx aarch64_expand_fpsr_fpcr_getter ( enum insn_code icode , machine_mode mode , rtx target ) { expand_operand op ; create_output_operand ( & op , target , mode ) ; expand_insn ( icode , 1 , & op ) ; return op . value ; } </s>

Expand a fpsr or fpcr getter ( depending on UNSPEC ) using MODE . Return the target .


<s> static void aarch64_expand_fpsr_fpcr_setter ( int unspec , machine_mode mode , tree exp ) { tree arg = CALL_EXPR_ARG ( exp , 0 ) ; rtx op = force_reg ( mode , expand_normal ( arg ) ) ; emit_insn ( gen_aarch64_set ( unspec , mode , op ) ) ; } </s>

Expand an expression EXP as fpsr or fpcr setter ( depending on UNSPEC ) using MODE .


<s> rtx aarch64_expand_rng_builtin ( tree exp , rtx target , int fcode , int ignore ) { rtx pat ; enum insn_code icode ; if ( fcode == AARCH64_BUILTIN_RNG_RNDR ) icode = CODE_FOR_aarch64_rndr ; else if ( fcode == AARCH64_BUILTIN_RNG_RNDRRS ) icode = CODE_FOR_aarch64_rndrrs ; else gcc_unreachable ( ) ; rtx rand = gen_reg_rtx ( DImode ) ; pat = GEN_FCN ( icode ) ( rand ) ; if ( ! pat ) return NULL_RTX ; tree arg0 = CALL_EXPR_ARG ( exp , 0 ) ; rtx res_addr = expand_normal ( arg0 ) ; res_addr = convert_memory_address ( Pmode , res_addr ) ; rtx res_mem = gen_rtx_MEM ( DImode , res_addr ) ; emit_insn ( pat ) ; emit_move_insn ( res_mem , rand ) ; if ( ignore ) return target ; rtx cc_reg = gen_rtx_REG ( CC_Zmode , CC_REGNUM ) ; rtx cmp_rtx = gen_rtx_fmt_ee ( EQ , SImode , cc_reg , const0_rtx ) ; emit_insn ( gen_aarch64_cstoresi ( target , cmp_rtx , cc_reg ) ) ; return target ; } </s>

Expand a random number builtin EXP with code FCODE , putting the result int TARGET . If IGNORE is true the return value is ignored .


<s> static bool aarch64_fold_builtin_lane_check ( tree arg0 , tree arg1 , tree arg2 ) { if ( TREE_CODE ( arg0 ) != INTEGER_CST ) return false ; if ( TREE_CODE ( arg1 ) != INTEGER_CST ) return false ; if ( TREE_CODE ( arg2 ) != INTEGER_CST ) return false ; auto totalsize = wi :: to_widest ( arg0 ) ; auto elementsize = wi :: to_widest ( arg1 ) ; if ( totalsize == 0 || elementsize == 0 ) return false ; auto lane = wi :: to_widest ( arg2 ) ; auto high = wi :: udiv_trunc ( totalsize , elementsize ) ; return wi :: ltu_p ( lane , high ) ; } </s>

Return true if the lane check can be removed as there is no error going to be emitted .


<s> static tree aarch64_general_add_builtin ( const char * name , tree type , unsigned int code , tree attrs = NULL_TREE ) { code = ( code << AARCH64_BUILTIN_SHIFT ) | AARCH64_BUILTIN_GENERAL ; return add_builtin_function ( name , type , code , BUILT_IN_MD , NULL , attrs ) ; } </s>

Wrapper around add_builtin_function . NAME is the name of the built-in function , TYPE is the function type , CODE is the function subcode ( relative to AARCH64_BUILTIN_GENERAL ) , and ATTRS is the function attributes .


<s> tree aarch64_general_builtin_decl ( unsigned code , bool ) { if ( code >= AARCH64_BUILTIN_MAX ) return error_mark_node ; return aarch64_builtin_decls [ code ] ; } </s>

Implement TARGET_BUILTIN_DECL for the AARCH64_BUILTIN_GENERAL group .


<s> tree aarch64_general_builtin_rsqrt ( unsigned int fn ) { if ( fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv2df ) return aarch64_builtin_decls [ AARCH64_BUILTIN_RSQRT_V2DF ] ; if ( fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv2sf ) return aarch64_builtin_decls [ AARCH64_BUILTIN_RSQRT_V2SF ] ; if ( fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv4sf ) return aarch64_builtin_decls [ AARCH64_BUILTIN_RSQRT_V4SF ] ; return NULL_TREE ; } </s>


<s> tree aarch64_general_fold_builtin ( unsigned int fcode , tree type , unsigned int n_args ATTRIBUTE_UNUSED , tree * args ) { switch ( fcode ) { BUILTIN_VDQF ( UNOP , abs , 2 , ALL ) return fold_build1 ( ABS_EXPR , type , args [ 0 ] ) ; VAR1 ( UNOP , floatv2si , 2 , ALL , v2sf ) VAR1 ( UNOP , floatv4si , 2 , ALL , v4sf ) VAR1 ( UNOP , floatv2di , 2 , ALL , v2df ) return fold_build1 ( FLOAT_EXPR , type , args [ 0 ] ) ; case AARCH64_SIMD_BUILTIN_LANE_CHECK : gcc_assert ( n_args == 3 ) ; if ( aarch64_fold_builtin_lane_check ( args [ 0 ] , args [ 1 ] , args [ 2 ] ) ) return void_node ; break ; default : break ; } return NULL_TREE ; } </s>

Try to fold a call to the built-in function with subcode FCODE . The function is passed the N_ARGS arguments in ARGS and it returns a value of type TYPE . Return the new expression on success and NULL_TREE on failure .


<s> void aarch64_general_init_builtins ( void ) { aarch64_init_fpsr_fpcr_builtins ( ) ; aarch64_init_fp16_types ( ) ; aarch64_init_bf16_types ( ) ; { aarch64_simd_switcher simd ; aarch64_init_simd_builtins ( ) ; } aarch64_init_crc32_builtins ( ) ; aarch64_init_builtin_rsqrt ( ) ; aarch64_init_rng_builtins ( ) ; tree ftype_jcvt = build_function_type_list ( intSI_type_node , double_type_node , NULL ) ; aarch64_builtin_decls [ AARCH64_JSCVT ] = aarch64_general_add_builtin ( "__builtin_aarch64_jcvtzs" , ftype_jcvt , AARCH64_JSCVT ) ; if ( ! TARGET_ILP32 ) aarch64_init_pauth_hint_builtins ( ) ; if ( TARGET_TME ) aarch64_init_tme_builtins ( ) ; if ( TARGET_MEMTAG ) aarch64_init_memtag_builtins ( ) ; } </s>

Initialize all builtins in the AARCH64_BUILTIN_GENERAL group .


<s> static tree aarch64_get_attributes ( unsigned int f , machine_mode mode ) { tree attrs = NULL_TREE ; if ( ! aarch64_modifies_global_state_p ( f , mode ) ) { if ( aarch64_reads_global_state_p ( f , mode ) ) attrs = aarch64_add_attribute ( "pure" , attrs ) ; else attrs = aarch64_add_attribute ( "const" , attrs ) ; } if ( ! flag_non_call_exceptions || ! aarch64_could_trap_p ( f , mode ) ) attrs = aarch64_add_attribute ( "nothrow" , attrs ) ; return aarch64_add_attribute ( "leaf" , attrs ) ; } </s>

Return the appropriate function attributes .


<s> static void aarch64_init_bf16_types ( void ) { aarch64_bf16_type_node = make_node ( REAL_TYPE ) ; TYPE_PRECISION ( aarch64_bf16_type_node ) = 16 ; SET_TYPE_MODE ( aarch64_bf16_type_node , BFmode ) ; layout_type ( aarch64_bf16_type_node ) ; lang_hooks . types . register_builtin_type ( aarch64_bf16_type_node , "__bf16" ) ; aarch64_bf16_ptr_type_node = build_pointer_type ( aarch64_bf16_type_node ) ; } </s>

Initialize the backend REAL_TYPE type supporting bfloat types .


<s> void aarch64_init_builtin_rsqrt ( void ) { tree fndecl = NULL ; tree ftype = NULL ; tree V2SF_type_node = build_vector_type ( float_type_node , 2 ) ; tree V2DF_type_node = build_vector_type ( double_type_node , 2 ) ; tree V4SF_type_node = build_vector_type ( float_type_node , 4 ) ; struct builtin_decls_data { tree type_node ; const char * builtin_name ; int function_code ; } ; builtin_decls_data bdda [ ] = { { double_type_node , "__builtin_aarch64_rsqrt_df" , AARCH64_BUILTIN_RSQRT_DF } , { float_type_node , "__builtin_aarch64_rsqrt_sf" , AARCH64_BUILTIN_RSQRT_SF } , { V2DF_type_node , "__builtin_aarch64_rsqrt_v2df" , AARCH64_BUILTIN_RSQRT_V2DF } , { V2SF_type_node , "__builtin_aarch64_rsqrt_v2sf" , AARCH64_BUILTIN_RSQRT_V2SF } , { V4SF_type_node , "__builtin_aarch64_rsqrt_v4sf" , AARCH64_BUILTIN_RSQRT_V4SF } } ; builtin_decls_data * bdd = bdda ; builtin_decls_data * bdd_end = bdd + ( sizeof ( bdda ) / sizeof ( builtin_decls_data ) ) ; for ( ; bdd < bdd_end ; bdd ++ ) { ftype = build_function_type_list ( bdd -> type_node , bdd -> type_node , NULL_TREE ) ; tree attrs = aarch64_get_attributes ( FLAG_FP , TYPE_MODE ( bdd -> type_node ) ) ; fndecl = aarch64_general_add_builtin ( bdd -> builtin_name , ftype , bdd -> function_code , attrs ) ; aarch64_builtin_decls [ bdd -> function_code ] = fndecl ; } } </s>

Add builtins for reciprocal square root .


<s> void aarch64_init_fcmla_laneq_builtins ( void ) { unsigned int i = 0 ; for ( i = 0 ; i < ARRAY_SIZE ( aarch64_fcmla_lane_builtin_data ) ; ++ i ) { aarch64_fcmla_laneq_builtin_datum * d = & aarch64_fcmla_lane_builtin_data [ i ] ; tree argtype = aarch64_lookup_simd_builtin_type ( d -> mode , qualifier_none ) ; machine_mode quadmode = GET_MODE_2XWIDER_MODE ( d -> mode ) . require ( ) ; tree quadtype = aarch64_lookup_simd_builtin_type ( quadmode , qualifier_none ) ; tree lanetype = aarch64_simd_builtin_std_type ( SImode , qualifier_lane_pair_index ) ; tree ftype = build_function_type_list ( argtype , argtype , argtype , quadtype , lanetype , NULL_TREE ) ; tree attrs = aarch64_get_attributes ( FLAG_FP , d -> mode ) ; tree fndecl = aarch64_general_add_builtin ( d -> name , ftype , d -> fcode , attrs ) ; aarch64_builtin_decls [ d -> fcode ] = fndecl ; } } </s>

Due to the architecture not providing lane variant of the lane instructions for fcmla we ca n't use the standard simd builtin expansion code , but we still want the majority of the validation that would normally be done .


<s> static void aarch64_init_fp16_types ( void ) { aarch64_fp16_type_node = make_node ( REAL_TYPE ) ; TYPE_PRECISION ( aarch64_fp16_type_node ) = 16 ; layout_type ( aarch64_fp16_type_node ) ; ( * lang_hooks . types . register_builtin_type ) ( aarch64_fp16_type_node , "__fp16" ) ; aarch64_fp16_ptr_type_node = build_pointer_type ( aarch64_fp16_type_node ) ; } </s>

Initialize the backend types that support the user-visible __fp16 type , also initialize a pointer to that type , to be used when forming HFAs .


Initialize fpsr fpcr getters and setters .


Pointer authentication builtins that will become NOP on legacy platform . Currently , these builtins are for internal use only ( libgcc EH unwinder ) .


<s> static void aarch64_init_rng_builtins ( void ) { tree unsigned_ptr_type = build_pointer_type ( unsigned_intDI_type_node ) ; tree ftype = build_function_type_list ( integer_type_node , unsigned_ptr_type , NULL ) ; aarch64_builtin_decls [ AARCH64_BUILTIN_RNG_RNDR ] = aarch64_general_add_builtin ( "__builtin_aarch64_rndr" , ftype , AARCH64_BUILTIN_RNG_RNDR ) ; aarch64_builtin_decls [ AARCH64_BUILTIN_RNG_RNDRRS ] = aarch64_general_add_builtin ( "__builtin_aarch64_rndrrs" , ftype , AARCH64_BUILTIN_RNG_RNDRRS ) ; } </s>

Add builtins for Random Number instructions .


Initialize the transactional memory extension ( TME ) builtins .


<s> static bool aarch64_modifies_global_state_p ( unsigned int f , machine_mode mode ) { unsigned int flags = aarch64_call_properties ( f , mode ) ; if ( flags & FLAG_RAISE_FP_EXCEPTIONS ) return true ; if ( flags & FLAG_PREFETCH_MEMORY ) return true ; return flags & FLAG_WRITE_MEMORY ; } </s>

Return true if calls to the function could modify some form of global state .


<s> static bool aarch64_reads_global_state_p ( unsigned int f , machine_mode mode ) { unsigned int flags = aarch64_call_properties ( f , mode ) ; if ( flags & FLAG_READ_FPCR ) return true ; return flags & FLAG_READ_MEMORY ; } </s>

Return true if calls to the function could read some form of global state .


Called at aarch64_resolve_overloaded_builtin in aarch64-c.cc .


<s> static tree aarch64_simd_builtin_type ( machine_mode mode , bool unsigned_p , bool poly_p ) { if ( poly_p ) return aarch64_lookup_simd_builtin_type ( mode , qualifier_poly ) ; else if ( unsigned_p ) return aarch64_lookup_simd_builtin_type ( mode , qualifier_unsigned ) ; else return aarch64_lookup_simd_builtin_type ( mode , qualifier_none ) ; } </s>

Return a type for an operand with specified mode and qualifiers .


"lane index must be a constant immediate"

"total size and element size must be a nonzero "

Expand an AArch64 AdvSIMD builtin ( intrinsic ) .


Enable AARCH64_FL_ * flags EXTRA_FLAGS on top of the base Advanced SIMD set .


<s> void handle_arm_acle_h ( void ) { if ( TARGET_LS64 ) aarch64_init_ls64_builtins ( ) ; } </s>

Implement # pragma GCC aarch64 `` arm_acle.h '' .


<s> void handle_arm_neon_h ( void ) { aarch64_simd_switcher simd ; for ( unsigned int i = 0 ; i < ARM_NEON_H_TYPES_LAST ; i ++ ) for ( unsigned int count = 2 ; count <= 4 ; ++ count ) if ( ! aarch64_scalar_builtin_type_p ( aarch64_simd_types [ i ] . type ) ) register_tuple_type ( count , i ) ; aarch64_init_simd_builtin_functions ( true ) ; } </s>

Implement # pragma GCC aarch64 `` arm_neon.h '' . The types and functions defined here need to be available internally during LTO as well .


<s> static void register_tuple_type ( unsigned int num_vectors , unsigned int type_index ) { aarch64_simd_type_info * type = & aarch64_simd_types [ type_index ] ; const char * vector_type_name = type -> name ; char tuple_type_name [ sizeof ( "bfloat16x4x2_t" ) ] ; snprintf ( tuple_type_name , sizeof ( tuple_type_name ) , "%.*sx%d_t" , ( int ) strlen ( vector_type_name ) - 4 , vector_type_name + 2 , num_vectors ) ; tuple_type_name [ 0 ] = TOLOWER ( tuple_type_name [ 0 ] ) ; tree vector_type = type -> itype ; tree array_type = build_array_type_nelts ( vector_type , num_vectors ) ; if ( type -> mode == DImode ) { if ( num_vectors == 2 ) SET_TYPE_MODE ( array_type , V2x1DImode ) ; else if ( num_vectors == 3 ) SET_TYPE_MODE ( array_type , V3x1DImode ) ; else if ( num_vectors == 4 ) SET_TYPE_MODE ( array_type , V4x1DImode ) ; } unsigned int alignment = known_eq ( GET_MODE_SIZE ( type -> mode ) , 16 ) ? 128 : 64 ; machine_mode tuple_mode = TYPE_MODE_RAW ( array_type ) ; gcc_assert ( VECTOR_MODE_P ( tuple_mode ) && TYPE_MODE ( array_type ) == tuple_mode && TYPE_ALIGN ( array_type ) == alignment ) ; tree field = build_decl ( input_location , FIELD_DECL , get_identifier ( "val" ) , array_type ) ; tree t = lang_hooks . types . simulate_record_decl ( input_location , tuple_type_name , make_array_slice ( & field , 1 ) ) ; gcc_assert ( TYPE_MODE_RAW ( t ) == TYPE_MODE ( t ) && ( flag_pack_struct || maximum_field_alignment || ( TYPE_MODE_RAW ( t ) == tuple_mode && TYPE_ALIGN ( t ) == alignment ) ) ) ; aarch64_simd_tuple_modes [ type_index ] [ num_vectors - 2 ] = tuple_mode ; aarch64_simd_tuple_types [ type_index ] [ num_vectors - 2 ] = t ; } </s>

Register the tuple type that contains NUM_VECTORS vectors of type TYPE .


Define the macros that we always expect to have on AArch64 .


<s> static bool aarch64_pragma_target_parse ( tree args , tree pop_target ) { if ( args ) { if ( ! aarch64_process_target_attr ( args ) ) return false ; aarch64_override_options_internal ( & global_options ) ; } else { pop_target = pop_target ? pop_target : target_option_default_node ; cl_target_option_restore ( & global_options , & global_options_set , TREE_TARGET_OPTION ( pop_target ) ) ; } target_option_current_node = build_target_option_node ( & global_options , & global_options_set ) ; aarch64_reset_previous_fndecl ( ) ; cpp_options * cpp_opts = cpp_get_options ( parse_in ) ; unsigned char saved_warn_unused_macros = cpp_opts -> warn_unused_macros ; cpp_opts -> warn_unused_macros = 0 ; cpp_force_token_locations ( parse_in , BUILTINS_LOCATION ) ; aarch64_update_cpp_builtins ( parse_in ) ; cpp_stop_forcing_token_locations ( parse_in ) ; cpp_opts -> warn_unused_macros = saved_warn_unused_macros ; if ( pop_target ) aarch64_save_restore_target_globals ( pop_target ) ; return true ; } </s>

Hook to validate the current # pragma GCC target and set the state , and update the macros based on what was changed . If ARGS is NULL , then POP_TARGET is used to reset the options .


<s> static bool aarch64_pragma_target_parse ( tree args , tree pop_target ) { if ( args ) { if ( ! aarch64_process_target_attr ( args , "pragma" ) ) return false ; aarch64_override_options_internal ( & global_options ) ; } else { pop_target = pop_target ? pop_target : target_option_default_node ; cl_target_option_restore ( & global_options , TREE_TARGET_OPTION ( pop_target ) ) ; } target_option_current_node = build_target_option_node ( & global_options ) ; aarch64_reset_previous_fndecl ( ) ; cpp_options * cpp_opts = cpp_get_options ( parse_in ) ; unsigned char saved_warn_unused_macros = cpp_opts -> warn_unused_macros ; cpp_opts -> warn_unused_macros = 0 ; aarch64_update_cpp_builtins ( parse_in ) ; cpp_opts -> warn_unused_macros = saved_warn_unused_macros ; if ( pop_target ) aarch64_save_restore_target_globals ( pop_target ) ; if ( TARGET_SIMD ) { tree saved_current_target_pragma = current_target_pragma ; current_target_pragma = NULL ; aarch64_init_simd_builtins ( ) ; current_target_pragma = saved_current_target_pragma ; } return true ; } </s>


<s> void aarch64_register_pragmas ( void ) { targetm . target_option . pragma_parse = aarch64_pragma_target_parse ; } </s>


<s> static void aarch64_update_cpp_builtins ( cpp_reader * pfile ) { aarch64_def_or_undef ( flag_unsafe_math_optimizations , "__ARM_FP_FAST" , pfile ) ; builtin_define_with_int_value ( "__ARM_ARCH" , aarch64_architecture_version ) ; builtin_define_with_int_value ( "__ARM_SIZEOF_MINIMAL_ENUM" , flag_short_enums ? 1 : 4 ) ; aarch64_def_or_undef ( TARGET_BIG_END , "__AARCH64EB__" , pfile ) ; aarch64_def_or_undef ( TARGET_BIG_END , "__ARM_BIG_ENDIAN" , pfile ) ; aarch64_def_or_undef ( ! TARGET_BIG_END , "__AARCH64EL__" , pfile ) ; aarch64_def_or_undef ( TARGET_FLOAT , "__ARM_FEATURE_FMA" , pfile ) ; if ( TARGET_FLOAT || TARGET_SIMD ) { builtin_define_with_int_value ( "__ARM_FP" , 0x0E ) ; builtin_define ( "__ARM_FP16_FORMAT_IEEE" ) ; builtin_define ( "__ARM_FP16_ARGS" ) ; } else cpp_undef ( pfile , "__ARM_FP" ) ; aarch64_def_or_undef ( TARGET_SIMD , "__ARM_FEATURE_NUMERIC_MAXMIN" , pfile ) ; aarch64_def_or_undef ( TARGET_SIMD , "__ARM_NEON" , pfile ) ; aarch64_def_or_undef ( TARGET_CRC32 , "__ARM_FEATURE_CRC32" , pfile ) ; cpp_undef ( pfile , "__AARCH64_CMODEL_TINY__" ) ; cpp_undef ( pfile , "__AARCH64_CMODEL_SMALL__" ) ; cpp_undef ( pfile , "__AARCH64_CMODEL_LARGE__" ) ; switch ( aarch64_cmodel ) { case AARCH64_CMODEL_TINY : case AARCH64_CMODEL_TINY_PIC : builtin_define ( "__AARCH64_CMODEL_TINY__" ) ; break ; case AARCH64_CMODEL_SMALL : case AARCH64_CMODEL_SMALL_PIC : builtin_define ( "__AARCH64_CMODEL_SMALL__" ) ; break ; case AARCH64_CMODEL_LARGE : builtin_define ( "__AARCH64_CMODEL_LARGE__" ) ; break ; default : break ; } aarch64_def_or_undef ( TARGET_ILP32 , "_ILP32" , pfile ) ; aarch64_def_or_undef ( TARGET_ILP32 , "__ILP32__" , pfile ) ; aarch64_def_or_undef ( TARGET_CRYPTO , "__ARM_FEATURE_CRYPTO" , pfile ) ; aarch64_def_or_undef ( TARGET_SIMD_RDMA , "__ARM_FEATURE_QRDMX" , pfile ) ; } </s>

Undefine/redefine macros that depend on the current backend state and may need to change when a target pragma modifies the backend state .


<s> static void aarch64_update_cpp_builtins ( cpp_reader * pfile ) { aarch64_def_or_undef ( flag_unsafe_math_optimizations , "__ARM_FP_FAST" , pfile ) ; builtin_define_with_int_value ( "__ARM_ARCH" , aarch64_architecture_version ) ; builtin_define_with_int_value ( "__ARM_SIZEOF_MINIMAL_ENUM" , flag_short_enums ? 1 : 4 ) ; aarch64_def_or_undef ( TARGET_BIG_END , "__AARCH64EB__" , pfile ) ; aarch64_def_or_undef ( TARGET_BIG_END , "__ARM_BIG_ENDIAN" , pfile ) ; aarch64_def_or_undef ( ! TARGET_BIG_END , "__AARCH64EL__" , pfile ) ; aarch64_def_or_undef ( TARGET_FLOAT , "__ARM_FEATURE_FMA" , pfile ) ; if ( TARGET_FLOAT || TARGET_SIMD ) { builtin_define_with_int_value ( "__ARM_FP" , 0x0E ) ; builtin_define ( "__ARM_FP16_FORMAT_IEEE" ) ; builtin_define ( "__ARM_FP16_ARGS" ) ; } else cpp_undef ( pfile , "__ARM_FP" ) ; aarch64_def_or_undef ( TARGET_FP_F16INST , "__ARM_FEATURE_FP16_SCALAR_ARITHMETIC" , pfile ) ; aarch64_def_or_undef ( TARGET_SIMD_F16INST , "__ARM_FEATURE_FP16_VECTOR_ARITHMETIC" , pfile ) ; aarch64_def_or_undef ( TARGET_SIMD , "__ARM_FEATURE_NUMERIC_MAXMIN" , pfile ) ; aarch64_def_or_undef ( TARGET_SIMD , "__ARM_NEON" , pfile ) ; aarch64_def_or_undef ( TARGET_CRC32 , "__ARM_FEATURE_CRC32" , pfile ) ; cpp_undef ( pfile , "__AARCH64_CMODEL_TINY__" ) ; cpp_undef ( pfile , "__AARCH64_CMODEL_SMALL__" ) ; cpp_undef ( pfile , "__AARCH64_CMODEL_LARGE__" ) ; switch ( aarch64_cmodel ) { case AARCH64_CMODEL_TINY : case AARCH64_CMODEL_TINY_PIC : builtin_define ( "__AARCH64_CMODEL_TINY__" ) ; break ; case AARCH64_CMODEL_SMALL : case AARCH64_CMODEL_SMALL_PIC : builtin_define ( "__AARCH64_CMODEL_SMALL__" ) ; break ; case AARCH64_CMODEL_LARGE : builtin_define ( "__AARCH64_CMODEL_LARGE__" ) ; break ; default : break ; } aarch64_def_or_undef ( TARGET_ILP32 , "_ILP32" , pfile ) ; aarch64_def_or_undef ( TARGET_ILP32 , "__ILP32__" , pfile ) ; aarch64_def_or_undef ( TARGET_CRYPTO , "__ARM_FEATURE_CRYPTO" , pfile ) ; aarch64_def_or_undef ( TARGET_SIMD_RDMA , "__ARM_FEATURE_QRDMX" , pfile ) ; cpp_undef ( pfile , "__FLT_EVAL_METHOD__" ) ; builtin_define_with_int_value ( "__FLT_EVAL_METHOD__" , c_flt_eval_method ( true ) ) ; cpp_undef ( pfile , "__FLT_EVAL_METHOD_C99__" ) ; builtin_define_with_int_value ( "__FLT_EVAL_METHOD_C99__" , c_flt_eval_method ( false ) ) ; } </s>


<s> static void aarch64_update_cpp_builtins ( cpp_reader * pfile ) { aarch64_def_or_undef ( flag_unsafe_math_optimizations , "__ARM_FP_FAST" , pfile ) ; builtin_define_with_int_value ( "__ARM_ARCH" , aarch64_architecture_version ) ; builtin_define_with_int_value ( "__ARM_SIZEOF_MINIMAL_ENUM" , flag_short_enums ? 1 : 4 ) ; aarch64_def_or_undef ( TARGET_BIG_END , "__AARCH64EB__" , pfile ) ; aarch64_def_or_undef ( TARGET_BIG_END , "__ARM_BIG_ENDIAN" , pfile ) ; aarch64_def_or_undef ( ! TARGET_BIG_END , "__AARCH64EL__" , pfile ) ; aarch64_def_or_undef ( TARGET_FLOAT , "__ARM_FEATURE_FMA" , pfile ) ; if ( TARGET_FLOAT || TARGET_SIMD ) { builtin_define_with_int_value ( "__ARM_FP" , 0x0E ) ; builtin_define ( "__ARM_FP16_FORMAT_IEEE" ) ; builtin_define ( "__ARM_FP16_ARGS" ) ; } else cpp_undef ( pfile , "__ARM_FP" ) ; aarch64_def_or_undef ( TARGET_FP_F16INST , "__ARM_FEATURE_FP16_SCALAR_ARITHMETIC" , pfile ) ; aarch64_def_or_undef ( TARGET_SIMD_F16INST , "__ARM_FEATURE_FP16_VECTOR_ARITHMETIC" , pfile ) ; aarch64_def_or_undef ( TARGET_SIMD , "__ARM_FEATURE_NUMERIC_MAXMIN" , pfile ) ; aarch64_def_or_undef ( TARGET_SIMD , "__ARM_NEON" , pfile ) ; aarch64_def_or_undef ( TARGET_CRC32 , "__ARM_FEATURE_CRC32" , pfile ) ; aarch64_def_or_undef ( TARGET_DOTPROD , "__ARM_FEATURE_DOTPROD" , pfile ) ; cpp_undef ( pfile , "__AARCH64_CMODEL_TINY__" ) ; cpp_undef ( pfile , "__AARCH64_CMODEL_SMALL__" ) ; cpp_undef ( pfile , "__AARCH64_CMODEL_LARGE__" ) ; switch ( aarch64_cmodel ) { case AARCH64_CMODEL_TINY : case AARCH64_CMODEL_TINY_PIC : builtin_define ( "__AARCH64_CMODEL_TINY__" ) ; break ; case AARCH64_CMODEL_SMALL : case AARCH64_CMODEL_SMALL_PIC : builtin_define ( "__AARCH64_CMODEL_SMALL__" ) ; break ; case AARCH64_CMODEL_LARGE : builtin_define ( "__AARCH64_CMODEL_LARGE__" ) ; break ; default : break ; } aarch64_def_or_undef ( TARGET_ILP32 , "_ILP32" , pfile ) ; aarch64_def_or_undef ( TARGET_ILP32 , "__ILP32__" , pfile ) ; aarch64_def_or_undef ( TARGET_CRYPTO , "__ARM_FEATURE_CRYPTO" , pfile ) ; aarch64_def_or_undef ( TARGET_SIMD_RDMA , "__ARM_FEATURE_QRDMX" , pfile ) ; aarch64_def_or_undef ( TARGET_SVE , "__ARM_FEATURE_SVE" , pfile ) ; cpp_undef ( pfile , "__ARM_FEATURE_SVE_BITS" ) ; if ( TARGET_SVE ) { int bits ; if ( ! BITS_PER_SVE_VECTOR . is_constant ( & bits ) ) bits = 0 ; builtin_define_with_int_value ( "__ARM_FEATURE_SVE_BITS" , bits ) ; } aarch64_def_or_undef ( TARGET_AES , "__ARM_FEATURE_AES" , pfile ) ; aarch64_def_or_undef ( TARGET_SHA2 , "__ARM_FEATURE_SHA2" , pfile ) ; aarch64_def_or_undef ( TARGET_SHA3 , "__ARM_FEATURE_SHA3" , pfile ) ; aarch64_def_or_undef ( TARGET_SHA3 , "__ARM_FEATURE_SHA512" , pfile ) ; aarch64_def_or_undef ( TARGET_SM4 , "__ARM_FEATURE_SM3" , pfile ) ; aarch64_def_or_undef ( TARGET_SM4 , "__ARM_FEATURE_SM4" , pfile ) ; aarch64_def_or_undef ( TARGET_F16FML , "__ARM_FEATURE_FP16_FML" , pfile ) ; cpp_undef ( pfile , "__FLT_EVAL_METHOD__" ) ; builtin_define_with_int_value ( "__FLT_EVAL_METHOD__" , c_flt_eval_method ( true ) ) ; cpp_undef ( pfile , "__FLT_EVAL_METHOD_C99__" ) ; builtin_define_with_int_value ( "__FLT_EVAL_METHOD_C99__" , c_flt_eval_method ( false ) ) ; } </s>


<s> static void aarch64_update_cpp_builtins ( cpp_reader * pfile ) { aarch64_def_or_undef ( flag_unsafe_math_optimizations , "__ARM_FP_FAST" , pfile ) ; builtin_define_with_int_value ( "__ARM_ARCH" , aarch64_architecture_version ) ; builtin_define_with_int_value ( "__ARM_SIZEOF_MINIMAL_ENUM" , flag_short_enums ? 1 : 4 ) ; aarch64_def_or_undef ( TARGET_BIG_END , "__AARCH64EB__" , pfile ) ; aarch64_def_or_undef ( TARGET_BIG_END , "__ARM_BIG_ENDIAN" , pfile ) ; aarch64_def_or_undef ( ! TARGET_BIG_END , "__AARCH64EL__" , pfile ) ; aarch64_def_or_undef ( TARGET_FLOAT , "__ARM_FEATURE_FMA" , pfile ) ; if ( TARGET_FLOAT || TARGET_SIMD ) { builtin_define_with_int_value ( "__ARM_FP" , 0x0E ) ; builtin_define ( "__ARM_FP16_FORMAT_IEEE" ) ; builtin_define ( "__ARM_FP16_ARGS" ) ; } else cpp_undef ( pfile , "__ARM_FP" ) ; aarch64_def_or_undef ( TARGET_FP_F16INST , "__ARM_FEATURE_FP16_SCALAR_ARITHMETIC" , pfile ) ; aarch64_def_or_undef ( TARGET_SIMD_F16INST , "__ARM_FEATURE_FP16_VECTOR_ARITHMETIC" , pfile ) ; aarch64_def_or_undef ( TARGET_SIMD , "__ARM_FEATURE_NUMERIC_MAXMIN" , pfile ) ; aarch64_def_or_undef ( TARGET_SIMD , "__ARM_NEON" , pfile ) ; aarch64_def_or_undef ( TARGET_CRC32 , "__ARM_FEATURE_CRC32" , pfile ) ; aarch64_def_or_undef ( TARGET_DOTPROD , "__ARM_FEATURE_DOTPROD" , pfile ) ; aarch64_def_or_undef ( TARGET_COMPLEX , "__ARM_FEATURE_COMPLEX" , pfile ) ; cpp_undef ( pfile , "__AARCH64_CMODEL_TINY__" ) ; cpp_undef ( pfile , "__AARCH64_CMODEL_SMALL__" ) ; cpp_undef ( pfile , "__AARCH64_CMODEL_LARGE__" ) ; switch ( aarch64_cmodel ) { case AARCH64_CMODEL_TINY : case AARCH64_CMODEL_TINY_PIC : builtin_define ( "__AARCH64_CMODEL_TINY__" ) ; break ; case AARCH64_CMODEL_SMALL : case AARCH64_CMODEL_SMALL_PIC : builtin_define ( "__AARCH64_CMODEL_SMALL__" ) ; break ; case AARCH64_CMODEL_LARGE : builtin_define ( "__AARCH64_CMODEL_LARGE__" ) ; break ; default : break ; } aarch64_def_or_undef ( TARGET_ILP32 , "_ILP32" , pfile ) ; aarch64_def_or_undef ( TARGET_ILP32 , "__ILP32__" , pfile ) ; aarch64_def_or_undef ( TARGET_CRYPTO , "__ARM_FEATURE_CRYPTO" , pfile ) ; aarch64_def_or_undef ( TARGET_SIMD_RDMA , "__ARM_FEATURE_QRDMX" , pfile ) ; aarch64_def_or_undef ( TARGET_SVE , "__ARM_FEATURE_SVE" , pfile ) ; cpp_undef ( pfile , "__ARM_FEATURE_SVE_BITS" ) ; if ( TARGET_SVE ) { int bits ; if ( ! BITS_PER_SVE_VECTOR . is_constant ( & bits ) ) bits = 0 ; builtin_define_with_int_value ( "__ARM_FEATURE_SVE_BITS" , bits ) ; } aarch64_def_or_undef ( TARGET_LSE , "__ARM_FEATURE_ATOMICS" , pfile ) ; aarch64_def_or_undef ( TARGET_AES , "__ARM_FEATURE_AES" , pfile ) ; aarch64_def_or_undef ( TARGET_SHA2 , "__ARM_FEATURE_SHA2" , pfile ) ; aarch64_def_or_undef ( TARGET_SHA3 , "__ARM_FEATURE_SHA3" , pfile ) ; aarch64_def_or_undef ( TARGET_SHA3 , "__ARM_FEATURE_SHA512" , pfile ) ; aarch64_def_or_undef ( TARGET_SM4 , "__ARM_FEATURE_SM3" , pfile ) ; aarch64_def_or_undef ( TARGET_SM4 , "__ARM_FEATURE_SM4" , pfile ) ; aarch64_def_or_undef ( TARGET_F16FML , "__ARM_FEATURE_FP16_FML" , pfile ) ; cpp_undef ( pfile , "__FLT_EVAL_METHOD__" ) ; builtin_define_with_int_value ( "__FLT_EVAL_METHOD__" , c_flt_eval_method ( true ) ) ; cpp_undef ( pfile , "__FLT_EVAL_METHOD_C99__" ) ; builtin_define_with_int_value ( "__FLT_EVAL_METHOD_C99__" , c_flt_eval_method ( false ) ) ; } </s>


<s> unsigned int pass_cc_fusion :: execute ( function * ) { cc_fusion ( ) . execute ( ) ; return 0 ; } </s>


<s> static bool aarch64_check_builtin_call ( location_t loc , vec < location_t > arg_loc , tree fndecl , tree orig_fndecl , unsigned int nargs , tree * args ) { unsigned int code = DECL_MD_FUNCTION_CODE ( fndecl ) ; unsigned int subcode = code >> AARCH64_BUILTIN_SHIFT ; switch ( code & AARCH64_BUILTIN_CLASS ) { case AARCH64_BUILTIN_GENERAL : return true ; case AARCH64_BUILTIN_SVE : return aarch64_sve :: check_builtin_call ( loc , arg_loc , subcode , orig_fndecl , nargs , args ) ; } gcc_unreachable ( ) ; } </s>


<s> void aarch64_cpu_cpp_builtins ( cpp_reader * pfile ) { aarch64_define_unconditional_macros ( pfile ) ; aarch64_update_cpp_builtins ( pfile ) ; } </s>


"%<#pragma GCC aarch64%> requires a string parameter"

"unknown %<#pragma GCC aarch64%> option %qs"


<s> static bool aarch64_pragma_target_parse ( tree args , tree pop_target ) { if ( args ) { if ( ! aarch64_process_target_attr ( args ) ) return false ; aarch64_override_options_internal ( & global_options ) ; } else { pop_target = pop_target ? pop_target : target_option_default_node ; cl_target_option_restore ( & global_options , & global_options_set , TREE_TARGET_OPTION ( pop_target ) ) ; } target_option_current_node = build_target_option_node ( & global_options , & global_options_set ) ; aarch64_reset_previous_fndecl ( ) ; cpp_options * cpp_opts = cpp_get_options ( parse_in ) ; unsigned char saved_warn_unused_macros = cpp_opts -> warn_unused_macros ; cpp_opts -> warn_unused_macros = 0 ; cpp_force_token_locations ( parse_in , BUILTINS_LOCATION ) ; aarch64_update_cpp_builtins ( parse_in ) ; cpp_stop_forcing_token_locations ( parse_in ) ; cpp_opts -> warn_unused_macros = saved_warn_unused_macros ; if ( pop_target ) aarch64_save_restore_target_globals ( pop_target ) ; if ( TARGET_SIMD ) { tree saved_current_target_pragma = current_target_pragma ; current_target_pragma = NULL ; aarch64_init_simd_builtins ( ) ; current_target_pragma = saved_current_target_pragma ; } return true ; } </s>


<s> void aarch64_register_pragmas ( void ) { targetm . target_option . pragma_parse = aarch64_pragma_target_parse ; targetm . resolve_overloaded_builtin = aarch64_resolve_overloaded_builtin ; targetm . check_builtin_call = aarch64_check_builtin_call ; c_register_pragma ( "GCC" , "aarch64" , aarch64_pragma_aarch64 ) ; } </s>


<s> static tree aarch64_d_handle_target_float_abi ( void ) { const char * abi = "hard" ; return build_string_literal ( strlen ( abi ) + 1 , abi ) ; } </s>

Handle a call to ` __traits ( getTargetInfo , `` floatAbi '' ) ' .


<s> void aarch64_d_register_target_info ( void ) { const struct d_target_info_spec handlers [ ] = { { "floatAbi" , aarch64_d_handle_target_float_abi } , { NULL , NULL } , } ; d_add_target_info_handlers ( handlers ) ; } </s>


<s> void aarch64_d_target_versions ( void ) { d_add_builtin_version ( "AArch64" ) ; d_add_builtin_version ( "D_HardFloat" ) ; } </s>

Implement TARGET_D_CPU_VERSIONS for AArch64 targets .


<s> aarch64_feature_flags get_flags_off ( aarch64_feature_flags mask ) { return ( 0 | ( feature_deps :: IDENT ( ) . enable & mask ? AARCH64_FL_ ## IDENT : 0 ) ) ; } </s>

Return the set of all features that would need to be disabled if the features in MASK are disabled . Note that the size of the expression varies linearly with the number of features , which means that invoking this function once per feature is quadratic in the number of features . However , collecting the same information at compiler start-up is likely to be quadratic too , so we 're better off paying the cost once per compiler build rather than once per compiler run .


<s> static rtx_insn * aarch64_speculation_clobber_sp ( ) { rtx sp = gen_rtx_REG ( DImode , SP_REGNUM ) ; rtx tracker = gen_rtx_REG ( DImode , SPECULATION_TRACKER_REGNUM ) ; rtx scratch = gen_rtx_REG ( DImode , SPECULATION_SCRATCH_REGNUM ) ; start_sequence ( ) ; emit_insn ( gen_rtx_SET ( scratch , sp ) ) ; emit_insn ( gen_anddi3 ( scratch , scratch , tracker ) ) ; emit_insn ( gen_rtx_SET ( sp , scratch ) ) ; rtx_insn * seq = get_insns ( ) ; end_sequence ( ) ; return seq ; } </s>

Generate a code sequence to clobber SP if speculating incorreclty .


<s> static rtx_insn * aarch64_speculation_establish_tracker ( ) { rtx sp = gen_rtx_REG ( DImode , SP_REGNUM ) ; rtx tracker = gen_rtx_REG ( DImode , SPECULATION_TRACKER_REGNUM ) ; start_sequence ( ) ; rtx cc = aarch64_gen_compare_reg ( EQ , sp , const0_rtx ) ; emit_insn ( gen_cstoredi_neg ( tracker , gen_rtx_NE ( CCmode , cc , const0_rtx ) , cc ) ) ; rtx_insn * seq = get_insns ( ) ; end_sequence ( ) ; return seq ; } </s>

Generate a code sequence to establish the tracker variable from the contents of SP .


<s> virtual unsigned int execute ( function * ) { return aarch64_do_track_speculation ( ) ; } </s>


<s> rtl_opt_pass * make_pass_track_speculation ( gcc :: context * ctxt ) { return new pass_track_speculation ( ctxt ) ; } </s>


<s> rtx expand ( function_expander & e ) const OVERRIDE { if ( m_su ) e . rotate_inputs_left ( 1 , 2 ) ; e . rotate_inputs_left ( 0 , 3 ) ; machine_mode mode = e . vector_mode ( 0 ) ; insn_code icode = code_for_dot_prod ( UNSPEC_USDOT , mode ) ; return e . use_exact_insn ( icode ) ; } </s>


<s> unsigned int call_properties ( const function_instance & ) const override { return CP_WRITE_MEMORY ; } </s>

Default implementation of function_base : :call_properties , with conservatively correct behavior for floating-point instructions .


<s> rtx expand ( function_expander & e ) const override { return e . use_exact_insn ( CODE_FOR_aarch64_wrffr ) ; } </s>


<s> gimple * fold ( gimple_folder & f ) const override { poly_uint64 nelts = TYPE_VECTOR_SUBPARTS ( TREE_TYPE ( f . lhs ) ) ; poly_uint64 base = m_base * exact_div ( nelts , 2 ) ; vec_perm_builder builder ( nelts , 2 , 3 ) ; for ( unsigned int i = 0 ; i < 3 ; ++ i ) { builder . quick_push ( base + i ) ; builder . quick_push ( base + i + nelts ) ; } return fold_permute ( f , builder ) ; } </s>

Try to fold the call . Return the new statement on success and null on failure .


<s> tree memory_scalar_type ( const function_instance & fi ) const override { return fi . scalar_type ( 0 ) ; } </s>

If the function addresses memory , return the type of a single scalar memory element .


<s> machine_mode memory_vector_mode ( const function_instance & ) const override { return m_mode ; } </s>

If the function addresses memory , return a vector mode whose GET_MODE_NUNITS is the number of elements addressed and whose GET_MODE_INNER is the mode of a single scalar memory element .


<s> unsigned int call_properties ( const function_instance & ) const OVERRIDE { return CP_WRITE_MEMORY ; } </s>


<s> rtx expand ( function_expander & e ) const OVERRIDE { return e . use_exact_insn ( CODE_FOR_aarch64_wrffr ) ; } </s>


<s> static rtx expand_mad ( function_expander & e , unsigned int merge_argno = DEFAULT_MERGE_ARGNO ) { if ( e . pred == PRED_x ) { insn_code icode ; if ( e . type_suffix ( 0 ) . integer_p ) icode = code_for_aarch64_pred_fma ( e . vector_mode ( 0 ) ) ; else icode = code_for_aarch64_pred ( UNSPEC_COND_FMLA , e . vector_mode ( 0 ) ) ; return e . use_pred_x_insn ( icode ) ; } insn_code icode = e . direct_optab_handler ( cond_fma_optab ) ; return e . use_cond_insn ( icode , merge_argno ) ; } </s>

Expand a call to svmad , or svmla after reordering its operands . Make _m forms merge with argument MERGE_ARGNO .


<s> static rtx expand_mla_mls_lane ( function_expander & e , int unspec ) { e . rotate_inputs_left ( 0 , 4 ) ; insn_code icode = code_for_aarch64_lane ( unspec , e . vector_mode ( 0 ) ) ; return e . use_exact_insn ( icode ) ; } </s>

Expand a call to svmla_lane or svmls_lane using floating-point unspec UNSPEC .


<s> static rtx expand_msb ( function_expander & e , unsigned int merge_argno = DEFAULT_MERGE_ARGNO ) { if ( e . pred == PRED_x ) { insn_code icode ; if ( e . type_suffix ( 0 ) . integer_p ) icode = code_for_aarch64_pred_fnma ( e . vector_mode ( 0 ) ) ; else icode = code_for_aarch64_pred ( UNSPEC_COND_FMLS , e . vector_mode ( 0 ) ) ; return e . use_pred_x_insn ( icode ) ; } insn_code icode = e . direct_optab_handler ( cond_fnma_optab ) ; return e . use_cond_insn ( icode , merge_argno ) ; } </s>

Expand a call to svmsb , or svmls after reordering its operands . Make _m forms merge with argument MERGE_ARGNO .


<s> gimple * fold ( gimple_folder & f ) const OVERRIDE { poly_uint64 nelts = TYPE_VECTOR_SUBPARTS ( TREE_TYPE ( f . lhs ) ) ; poly_uint64 base = m_base * exact_div ( nelts , 2 ) ; vec_perm_builder builder ( nelts , 2 , 3 ) ; for ( unsigned int i = 0 ; i < 3 ; ++ i ) { builder . quick_push ( base + i ) ; builder . quick_push ( base + i + nelts ) ; } return fold_permute ( f , builder ) ; } </s>


<s> tree memory_scalar_type ( const function_instance & fi ) const OVERRIDE { return fi . scalar_type ( 0 ) ; } </s>


<s> machine_mode memory_vector_mode ( const function_instance & ) const OVERRIDE { return m_mode ; } </s>


<s> static int unspec_cmla ( int rot ) { switch ( rot ) { case 0 : return UNSPEC_CMLA ; case 90 : return UNSPEC_CMLA90 ; case 180 : return UNSPEC_CMLA180 ; case 270 : return UNSPEC_CMLA270 ; default : gcc_unreachable ( ) ; } } </s>

Return the UNSPEC_CMLA * unspec for rotation amount ROT .


<s> static int unspec_cond_fcmla ( int rot ) { switch ( rot ) { case 0 : return UNSPEC_COND_FCMLA ; case 90 : return UNSPEC_COND_FCMLA90 ; case 180 : return UNSPEC_COND_FCMLA180 ; case 270 : return UNSPEC_COND_FCMLA270 ; default : gcc_unreachable ( ) ; } } </s>

Return the UNSPEC_COND_FCMLA * unspec for rotation amount ROT .


<s> static int unspec_fcmla ( int rot ) { switch ( rot ) { case 0 : return UNSPEC_FCMLA ; case 90 : return UNSPEC_FCMLA90 ; case 180 : return UNSPEC_FCMLA180 ; case 270 : return UNSPEC_FCMLA270 ; default : gcc_unreachable ( ) ; } } </s>

Return the UNSPEC_FCMLA * unspec for rotation amount ROT .


<s> unsigned int call_properties ( const function_instance & ) const override { return 0 ; } </s>


<s> rtx expand ( function_expander & e ) const override { int unspec = ( e . type_suffix ( 1 ) . unsigned_p ? m_unspec_for_uint : m_unspec_for_sint ) ; machine_mode pred_mode = e . vector_mode ( 0 ) ; scalar_mode reg_mode = GET_MODE_INNER ( e . vector_mode ( 1 ) ) ; return e . use_exact_insn ( code_for_while ( unspec , reg_mode , pred_mode ) ) ; } </s>


<s> tree memory_scalar_type ( const function_instance & fi ) const override { type_class_index tclass = fi . type_suffix ( 0 ) . tclass ; unsigned int element_bits = GET_MODE_BITSIZE ( m_to_mode ) ; type_suffix_index suffix = find_type_suffix ( tclass , element_bits ) ; return scalar_types [ type_suffixes [ suffix ] . vector_type ] ; } </s>


<s> machine_mode memory_vector_mode ( const function_instance & fi ) const override { poly_uint64 nunits = GET_MODE_NUNITS ( fi . vector_mode ( 0 ) ) ; return aarch64_sve_data_mode ( m_to_mode , nunits ) . require ( ) ; } </s>


<s> unsigned int vectors_per_tuple ( ) const override { return m_vectors_per_tuple ; } </s>

If the function operates on tuples of vectors , return the number of vectors in the tuples , otherwise return 1 .


<s> unsigned int call_properties ( const function_instance & ) const OVERRIDE { return 0 ; } </s>


<s> rtx expand ( function_expander & e ) const OVERRIDE { int unspec = ( e . type_suffix ( 1 ) . unsigned_p ? m_unspec_for_uint : m_unspec_for_sint ) ; machine_mode pred_mode = e . vector_mode ( 0 ) ; scalar_mode reg_mode = GET_MODE_INNER ( e . vector_mode ( 1 ) ) ; return e . use_exact_insn ( code_for_while ( unspec , reg_mode , pred_mode ) ) ; } </s>


<s> tree memory_scalar_type ( const function_instance & fi ) const OVERRIDE { type_class_index tclass = fi . type_suffix ( 0 ) . tclass ; unsigned int element_bits = GET_MODE_BITSIZE ( m_to_mode ) ; type_suffix_index suffix = find_type_suffix ( tclass , element_bits ) ; return scalar_types [ type_suffixes [ suffix ] . vector_type ] ; } </s>


<s> machine_mode memory_vector_mode ( const function_instance & fi ) const OVERRIDE { poly_uint64 nunits = GET_MODE_NUNITS ( fi . vector_mode ( 0 ) ) ; return aarch64_sve_data_mode ( m_to_mode , nunits ) . require ( ) ; } </s>


<s> unsigned int vectors_per_tuple ( ) const OVERRIDE { return m_vectors_per_tuple ; } </s>


<s> static tree parse_type ( const function_instance & instance , const char * & format ) { int ch = * format ++ ; if ( ch == '_' ) return void_type_node ; if ( ch == 'a' ) { ch = * format ++ ; if ( ch == 'l' ) return build_const_pointer ( instance . memory_scalar_type ( ) ) ; if ( ch == 'p' ) return const_ptr_type_node ; if ( ch == 's' ) return build_pointer_type ( instance . memory_scalar_type ( ) ) ; gcc_unreachable ( ) ; } if ( ch == 'b' ) return instance . base_vector_type ( ) ; if ( ch == 'd' ) return instance . displacement_vector_type ( ) ; if ( ch == 'e' ) { if ( startswith ( format , "pattern" ) ) { format += 7 ; return acle_svpattern ; } if ( startswith ( format , "prfop" ) ) { format += 5 ; return acle_svprfop ; } gcc_unreachable ( ) ; } if ( ch == 's' ) { type_suffix_index suffix = parse_element_type ( instance , format ) ; return scalar_types [ type_suffixes [ suffix ] . vector_type ] ; } if ( ch == 't' ) { type_suffix_index suffix = parse_element_type ( instance , format ) ; vector_type_index vector_type = type_suffixes [ suffix ] . vector_type ; unsigned int num_vectors = instance . vectors_per_tuple ( ) ; return acle_vector_types [ num_vectors - 1 ] [ vector_type ] ; } if ( ch == 'v' ) { type_suffix_index suffix = parse_element_type ( instance , format ) ; return acle_vector_types [ 0 ] [ type_suffixes [ suffix ] . vector_type ] ; } gcc_unreachable ( ) ; } </s>

Read and return a type from FORMAT for function INSTANCE . Advance FORMAT beyond the type string . The format is : _ - void al - array pointer for loads ap - array pointer for prefetches as - array pointer for stores b - base vector type ( from a _ < m0 > base suffix ) d - displacement vector type ( from a _ < m1 > index or _ < m1 > offset suffix ) e < name > - an enum with the given name s < elt > - a scalar type with the given element suffix t < elt > - a vector or tuple type with given element suffix [ * 1 ] v < elt > - a vector with the given element suffix where < elt > has the format described above parse_element_type [ * 1 ] the vectors_per_tuple function indicates whether the type should be a tuple , and if so , how many vectors it should contain .


<s> void build ( function_builder & b , const function_group_info & group ) const override { b . add_overloaded_functions ( group , MODE_none ) ; build_all ( b , "v0,vu0" , group , MODE_none ) ; } </s>

Build new chain starting from insn INSN_UID recursively adding all dependent uses and definitions . Return true if OK , false if the chain discovery was aborted .


<s> bool check ( function_checker & c ) const override { return c . require_immediate_range ( 2 , 0 , 7 ) ; } </s>

Perform semantic checks on the call . Return true if the call is valid , otherwise report a suitable error .


<s> tree resolve ( function_resolver & r ) const override { unsigned int i , nargs ; type_suffix_index type ; if ( ! r . check_gp_argument ( 1 , i , nargs ) || ( type = r . infer_vector_type ( i ) ) == NUM_TYPE_SUFFIXES ) return error_mark_node ; if ( type == TYPE_SUFFIX_b ) return r . resolve_to ( r . mode_suffix_id , type ) ; if ( type_suffixes [ type ] . integer_p && type_suffixes [ type ] . element_bits < 64 ) { type_suffix_index wide_suffix = find_type_suffix ( type_suffixes [ type ] . tclass , type_suffixes [ type ] . element_bits * 2 ) ; if ( tree res = r . lookup_form ( r . mode_suffix_id , wide_suffix ) ) return res ; } return r . report_no_such_form ( type ) ; } </s>

If the call is erroneous , report an appropriate error and return error_mark_node . Otherwise , if the function is overloaded , return the decl of the non-overloaded function . Return NULL_TREE otherwise , indicating that the call should be processed in the normal way .


<s> static void apply_predication ( const function_instance & instance , tree return_type , vec < tree > & argument_types ) { if ( instance . pred != PRED_none ) { argument_types . quick_insert ( 0 , get_svbool_t ( ) ) ; if ( ( argument_types . length ( ) == 2 && instance . pred == PRED_m ) || instance . shape == shapes :: unary_convert_narrowt ) argument_types . quick_insert ( 0 , return_type ) ; } } </s>


<s> void build ( function_builder & b , const function_group_info & group ) const OVERRIDE { b . add_overloaded_functions ( group , MODE_none ) ; build_all ( b , "v0,vu0" , group , MODE_none ) ; } </s>


<s> static void build_32_64 ( function_builder & b , const char * signature , const function_group_info & group , mode_suffix_index mode32 , mode_suffix_index mode64 , bool force_direct_overloads = false ) { for ( unsigned int pi = 0 ; group . preds [ pi ] != NUM_PREDS ; ++ pi ) if ( group . types [ 0 ] [ 0 ] == NUM_TYPE_SUFFIXES ) { gcc_assert ( mode32 != MODE_none && mode64 != MODE_none ) ; build_one ( b , signature , group , mode32 , 0 , pi , force_direct_overloads ) ; build_one ( b , signature , group , mode64 , 0 , pi , force_direct_overloads ) ; } else for ( unsigned int ti = 0 ; group . types [ ti ] [ 0 ] != NUM_TYPE_SUFFIXES ; ++ ti ) { unsigned int bits = type_suffixes [ group . types [ ti ] [ 0 ] ] . element_bits ; gcc_assert ( bits == 32 || bits == 64 ) ; mode_suffix_index mode = bits == 32 ? mode32 : mode64 ; if ( mode != MODE_none ) build_one ( b , signature , group , mode , ti , pi , force_direct_overloads ) ; } } </s>

GROUP describes some sort of gather or scatter operation . There are two cases : - If the function has any type suffixes ( as for loads and stores ) , the first function type suffix specifies either a 32-bit or a 64-bit type , which in turn selects either MODE32 or MODE64 as the addressing mode . Add a function instance for every type and predicate combination in GROUP for which the associated addressing mode is not MODE_none . - If the function has no type suffixes ( as for prefetches ) , add one MODE32 form and one MODE64 form for each predication type . The other arguments are as for build_all .


<s> static void build_all ( function_builder & b , const char * signature , const function_group_info & group , mode_suffix_index mode_suffix_id , bool force_direct_overloads = false ) { for ( unsigned int pi = 0 ; group . preds [ pi ] != NUM_PREDS ; ++ pi ) for ( unsigned int ti = 0 ; ti == 0 || group . types [ ti ] [ 0 ] != NUM_TYPE_SUFFIXES ; ++ ti ) build_one ( b , signature , group , mode_suffix_id , ti , pi , force_direct_overloads ) ; } </s>

Add a function instance for every operand & & predicate & & args combination in GROUP . Take the function base name from GROUP & & operand suffix from operand_suffixes & & mode suffix from type_suffixes & & predication suffix from predication_suffixes . Use apply_predication to add in the predicate .


<s> static tree build_const_pointer ( tree t ) { return build_pointer_type ( build_qualified_type ( t , TYPE_QUAL_CONST ) ) ; } </s>

Return a representation of `` const T * '' .


<s> static void build_one ( function_builder & b , const char * signature , const function_group_info & group , mode_suffix_index mode_suffix_id , unsigned int ti , unsigned int pi , bool force_direct_overloads ) { auto_vec < tree , 16 > argument_types ; function_instance instance ( group . base_name , * group . base , * group . shape , mode_suffix_id , group . types [ ti ] , group . preds [ pi ] ) ; tree return_type = parse_signature ( instance , signature , argument_types ) ; apply_predication ( instance , return_type , argument_types ) ; b . add_unique_function ( instance , return_type , argument_types , group . required_extensions , force_direct_overloads ) ; } </s>

Add one function instance for GROUP , using operand suffix at index OI , mode suffix at index PAIR & & bi and predication suffix at index pred_idx .


<s> static void build_sv_index ( function_builder & b , const char * signature , const function_group_info & group ) { build_32_64 ( b , signature , group , MODE_s32index , MODE_s64index ) ; build_32_64 ( b , signature , group , MODE_u32index , MODE_u64index ) ; } </s>

For every type and predicate combination in GROUP , add one function that takes a scalar ( pointer ) base and a signed vector array index , and another that instead takes an unsigned vector array index . The vector array index has the same element size as the first function type suffix . SIGNATURE is as for build_all .


<s> static void build_sv_index64 ( function_builder & b , const char * signature , const function_group_info & group ) { build_32_64 ( b , signature , group , MODE_none , MODE_s64index ) ; build_32_64 ( b , signature , group , MODE_none , MODE_u64index ) ; } </s>

Like build_sv_index , but only handle 64-bit types .


<s> static void build_sv_offset ( function_builder & b , const char * signature , const function_group_info & group ) { build_32_64 ( b , signature , group , MODE_s32offset , MODE_s64offset ) ; build_32_64 ( b , signature , group , MODE_u32offset , MODE_u64offset ) ; } </s>

Like build_sv_index , but taking vector byte offsets instead of vector array indices .


<s> static void build_sv_uint_offset ( function_builder & b , const char * signature , const function_group_info & group ) { build_32_64 ( b , signature , group , MODE_none , MODE_s64offset ) ; build_32_64 ( b , signature , group , MODE_u32offset , MODE_u64offset ) ; } </s>

Like build_sv_offset , but exclude offsets that must be interpreted as signed ( i.e . s32offset ) .


<s> static void build_vs_index ( function_builder & b , const char * signature , const function_group_info & group , bool force_direct_overloads = false ) { build_32_64 ( b , signature , group , MODE_u32base_index , MODE_u64base_index , force_direct_overloads ) ; } </s>

Like build_v_base , but for functions that also take a scalar array index .


<s> static void build_vs_offset ( function_builder & b , const char * signature , const function_group_info & group , bool force_direct_overloads = false ) { build_32_64 ( b , signature , group , MODE_u32base_offset , MODE_u64base_offset , force_direct_overloads ) ; } </s>

Like build_v_base , but for functions that also take a scalar byte offset .


<s> static void build_v_base ( function_builder & b , const char * signature , const function_group_info & group , bool force_direct_overloads = false ) { build_32_64 ( b , signature , group , MODE_u32base , MODE_u64base , force_direct_overloads ) ; } </s>

For every type and predicate combination in GROUP , add a function that takes a vector base address and no displacement . The vector base has the same element size as the first type suffix . The other arguments are as for build_all .


<s> bool check ( function_checker & c ) const OVERRIDE { return c . require_immediate_range ( 2 , 0 , 7 ) ; } </s>


<s> static type_suffix_index long_type_suffix ( function_resolver & r , type_suffix_index type ) { unsigned int element_bits = type_suffixes [ type ] . element_bits ; if ( type_suffixes [ type ] . integer_p && element_bits < 64 ) return find_type_suffix ( type_suffixes [ type ] . tclass , element_bits * 2 ) ; r . report_no_such_form ( type ) ; return NUM_TYPE_SUFFIXES ; } </s>

TYPE is the largest type suffix associated with the arguments of R , but the result is twice as wide . Return the associated type suffix if it exists , otherwise report an appropriate error and return NUM_TYPE_SUFFIXES .


<s> static unsigned int parse_count ( const function_instance & instance , const char * & format ) { if ( format [ 0 ] == '*' && format [ 1 ] == 'q' ) { format += 2 ; return instance . elements_per_vq ( 0 ) ; } if ( format [ 0 ] == '*' && format [ 1 ] == 't' ) { format += 2 ; return instance . vectors_per_tuple ( ) ; } return 1 ; } </s>

Read and move past any argument count at FORMAT for the function signature of INSTANCE . The counts are : * q : one argument per element in a 128-bit quadword ( as for svdupq ) * t : one argument per vector in a tuple ( as for svcreate ) Otherwise the count is 1 .


<s> static type_suffix_index parse_element_type ( const function_instance & instance , const char * & format ) { int ch = * format ++ ; if ( ch == 'f' || ch == 's' || ch == 'u' ) { type_class_index tclass = ( ch == 'f' ? TYPE_float : ch == 's' ? TYPE_signed : TYPE_unsigned ) ; char * end ; unsigned int bits = strtol ( format , & end , 10 ) ; format = end ; if ( bits == 0 || bits == 1 ) bits = instance . type_suffix ( bits ) . element_bits ; return find_type_suffix ( tclass , bits ) ; } if ( ch == 'w' ) { type_suffix_index suffix = parse_element_type ( instance , format ) ; if ( type_suffixes [ suffix ] . integer_p ) return find_type_suffix ( type_suffixes [ suffix ] . tclass , 64 ) ; return suffix ; } if ( ch == 'p' ) return TYPE_SUFFIX_b ; if ( ch == 'B' ) return TYPE_SUFFIX_bf16 ; if ( ch == 'q' ) { type_suffix_index suffix = parse_element_type ( instance , format ) ; return find_type_suffix ( type_suffixes [ suffix ] . tclass , type_suffixes [ suffix ] . element_bits / 4 ) ; } if ( ch == 'h' ) { type_suffix_index suffix = parse_element_type ( instance , format ) ; if ( suffix == TYPE_SUFFIX_b ) return suffix ; return find_type_suffix ( type_suffixes [ suffix ] . tclass , type_suffixes [ suffix ] . element_bits / 2 ) ; } if ( ch == '0' || ch == '1' ) return instance . type_suffix_ids [ ch - '0' ] ; gcc_unreachable ( ) ; } </s>

Parse and move past an element type in FORMAT and return it as a type suffix . The format is : [ 01 ] - the element type in type suffix 0 or 1 of INSTANCE f < bits > - a floating-point type with the given number of bits f [ 01 ] - a floating-point type with the same width as type suffix 0 or 1 B - bfloat16_t h < elt > - a half-sized version of < elt > p - a predicate ( represented as TYPE_SUFFIX_b ) q < elt > - a quarter-sized version of < elt > s < bits > - a signed type with the given number of bits s [ 01 ] - a signed type with the same width as type suffix 0 or 1 u < bits > - an unsigned type with the given number of bits u [ 01 ] - an unsigned type with the same width as type suffix 0 or 1 w < elt > - a 64-bit version of < elt > if < elt > is integral , otherwise < elt > where < elt > is another element type .


<s> static tree parse_signature ( const function_instance & instance , const char * format , vec < tree > & argument_types ) { tree return_type = parse_type ( instance , format ) ; while ( format [ 0 ] == ',' ) { format += 1 ; tree argument_type = parse_type ( instance , format ) ; unsigned int count = parse_count ( instance , format ) ; for ( unsigned int i = 0 ; i < count ; ++ i ) argument_types . quick_push ( argument_type ) ; } gcc_assert ( format [ 0 ] == 0 ) ; return return_type ; } </s>

Read a type signature for INSTANCE from FORMAT . Add the argument types to ARGUMENT_TYPES and return the return type . The format is a comma-separated list of types ( as for parse_type ) , with the first type being the return type and the rest being the argument types . Each argument type can be followed by an optional count ( as for parse_count ) .


<s> static tree parse_type ( const function_instance & instance , const char * & format ) { int ch = * format ++ ; if ( ch == '_' ) return void_type_node ; if ( ch == 'a' ) { ch = * format ++ ; if ( ch == 'l' ) return build_const_pointer ( instance . memory_scalar_type ( ) ) ; if ( ch == 'p' ) return const_ptr_type_node ; if ( ch == 's' ) return build_pointer_type ( instance . memory_scalar_type ( ) ) ; gcc_unreachable ( ) ; } if ( ch == 'b' ) return instance . base_vector_type ( ) ; if ( ch == 'd' ) return instance . displacement_vector_type ( ) ; if ( ch == 'e' ) { if ( strncmp ( format , "pattern" , 7 ) == 0 ) { format += 7 ; return acle_svpattern ; } if ( strncmp ( format , "prfop" , 5 ) == 0 ) { format += 5 ; return acle_svprfop ; } gcc_unreachable ( ) ; } if ( ch == 's' ) { type_suffix_index suffix = parse_element_type ( instance , format ) ; return scalar_types [ type_suffixes [ suffix ] . vector_type ] ; } if ( ch == 't' ) { type_suffix_index suffix = parse_element_type ( instance , format ) ; vector_type_index vector_type = type_suffixes [ suffix ] . vector_type ; unsigned int num_vectors = instance . vectors_per_tuple ( ) ; return acle_vector_types [ num_vectors - 1 ] [ vector_type ] ; } if ( ch == 'v' ) { type_suffix_index suffix = parse_element_type ( instance , format ) ; return acle_vector_types [ 0 ] [ type_suffixes [ suffix ] . vector_type ] ; } gcc_unreachable ( ) ; } </s>

Read and return a type from FORMAT for function INSTANCE . Advance FORMAT beyond the type string . The format is : _ - void al - array pointer for loads ap - array pointer for prefetches as - array pointer for stores b - base vector type ( from a _ < m0 > base suffix ) d - displacement vector type ( from a _ < m1 > index or _ < m1 > offset suffix ) e < name > - an enum with the given name s < elt > - a scalar type with the given element suffix t < elt > - a vector or tuple type with given element suffix [ * 1 ] v < elt > - a vector with the given element suffix where < elt > has the format described above parse_element_type [ * 1 ] the vectors_per_tuple function indicates whether the type should be a tuple , and if so , how many vectors it should contain .


<s> tree resolve ( function_resolver & r ) const OVERRIDE { unsigned int i , nargs ; type_suffix_index type ; if ( ! r . check_gp_argument ( 1 , i , nargs ) || ( type = r . infer_vector_type ( i ) ) == NUM_TYPE_SUFFIXES ) return error_mark_node ; if ( type == TYPE_SUFFIX_b ) return r . resolve_to ( r . mode_suffix_id , type ) ; if ( type_suffixes [ type ] . integer_p && type_suffixes [ type ] . element_bits < 64 ) { type_suffix_index wide_suffix = find_type_suffix ( type_suffixes [ type ] . tclass , type_suffixes [ type ] . element_bits * 2 ) ; if ( tree res = r . lookup_form ( r . mode_suffix_id , wide_suffix ) ) return res ; } return r . report_no_such_form ( type ) ; } </s>

If the call is erroneous , report an appropriate error and return error_mark_node . Otherwise , if the function is overloaded , return the decl of the non-overloaded function . Return NULL_TREE otherwise , indicating that the call should be processed in the normal way .


<s> rtx expand ( function_expander & e ) const override { for ( unsigned int i = 0 ; i < 2 ; ++ i ) e . args [ i ] = e . convert_to_pmode ( e . args [ i ] ) ; return e . use_exact_insn ( code_for_while ( m_unspec , Pmode , e . gp_mode ( 0 ) ) ) ; } </s>


<s> gimple * fold ( gimple_folder & f ) const override { if ( tree amount = uniform_integer_cst_p ( gimple_call_arg ( f . call , 2 ) ) ) { if ( wi :: to_widest ( amount ) >= 0 ) { function_instance instance ( "svlsl" , functions :: svlsl , shapes :: binary_uint_opt_n , MODE_n , f . type_suffix_ids , f . pred ) ; gcall * call = as_a < gcall * > ( f . redirect_call ( instance ) ) ; gimple_call_set_arg ( call , 2 , amount ) ; return call ; } int element_bits = f . type_suffix ( 0 ) . element_bits ; if ( wi :: to_widest ( amount ) >= - element_bits ) { amount = wide_int_to_tree ( TREE_TYPE ( amount ) , - wi :: to_wide ( amount ) ) ; function_instance instance ( "svrshr" , functions :: svrshr , shapes :: shift_right_imm , MODE_n , f . type_suffix_ids , f . pred ) ; gcall * call = as_a < gcall * > ( f . redirect_call ( instance ) ) ; gimple_call_set_arg ( call , 2 , amount ) ; return call ; } } return NULL ; } </s>

Try to fold the call . Return the new statement on success and null on failure .


<s> rtx expand ( function_expander & e ) const OVERRIDE { for ( unsigned int i = 0 ; i < 2 ; ++ i ) e . args [ i ] = e . convert_to_pmode ( e . args [ i ] ) ; return e . use_exact_insn ( code_for_while ( m_unspec , Pmode , e . gp_mode ( 0 ) ) ) ; } </s>


<s> gimple * fold ( gimple_folder & f ) const OVERRIDE { if ( tree amount = uniform_integer_cst_p ( gimple_call_arg ( f . call , 2 ) ) ) { if ( wi :: to_widest ( amount ) >= 0 ) { function_instance instance ( "svlsl" , functions :: svlsl , shapes :: binary_uint_opt_n , MODE_n , f . type_suffix_ids , f . pred ) ; gcall * call = as_a < gcall * > ( f . redirect_call ( instance ) ) ; gimple_call_set_arg ( call , 2 , amount ) ; return call ; } int element_bits = f . type_suffix ( 0 ) . element_bits ; if ( wi :: to_widest ( amount ) >= - element_bits ) { amount = wide_int_to_tree ( TREE_TYPE ( amount ) , - wi :: to_wide ( amount ) ) ; function_instance instance ( "svrshr" , functions :: svrshr , shapes :: shift_right_imm , MODE_n , f . type_suffix_ids , f . pred ) ; gcall * call = as_a < gcall * > ( f . redirect_call ( instance ) ) ; gimple_call_set_arg ( call , 2 , amount ) ; return call ; } } return NULL ; } </s>


<s> static int unspec_cdot ( int rot ) { switch ( rot ) { case 0 : return UNSPEC_CDOT ; case 90 : return UNSPEC_CDOT90 ; case 180 : return UNSPEC_CDOT180 ; case 270 : return UNSPEC_CDOT270 ; default : gcc_unreachable ( ) ; } } </s>

Return the UNSPEC_CDOT * unspec for rotation amount ROT .


<s> static int unspec_sqrdcmlah ( int rot ) { switch ( rot ) { case 0 : return UNSPEC_SQRDCMLAH ; case 90 : return UNSPEC_SQRDCMLAH90 ; case 180 : return UNSPEC_SQRDCMLAH180 ; case 270 : return UNSPEC_SQRDCMLAH270 ; default : gcc_unreachable ( ) ; } } </s>

Return the UNSPEC_SQRDCMLAH * unspec for rotation amount ROT .


<s> void function_expander :: add_address_operand ( rtx x ) { m_ops . safe_grow ( m_ops . length ( ) + 1 , true ) ; create_address_operand ( & m_ops . last ( ) , x ) ; } </s>

Add an address operand with value X . The static operand data says what mode and form the address must have .


<s> void function_expander :: add_fixed_operand ( rtx x ) { m_ops . safe_grow ( m_ops . length ( ) + 1 , true ) ; create_fixed_operand ( & m_ops . last ( ) , x ) ; } </s>

Add an operand that must be X . The only way of legitimizing an invalid X is to reload the address of a MEM .


<s> registered_function & function_builder :: add_function ( const function_instance & instance , const char * name , tree fntype , tree attrs , uint64_t required_extensions , bool overloaded_p , bool placeholder_p ) { unsigned int code = vec_safe_length ( registered_functions ) ; code = ( code << AARCH64_BUILTIN_SHIFT ) | AARCH64_BUILTIN_SVE ; tree decl = placeholder_p ? integer_zero_node : simulate_builtin_function_decl ( input_location , name , fntype , code , NULL , attrs ) ; registered_function & rfn = * ggc_alloc < registered_function > ( ) ; rfn . instance = instance ; rfn . decl = decl ; rfn . required_extensions = required_extensions ; rfn . overloaded_p = overloaded_p ; vec_safe_push ( registered_functions , & rfn ) ; return rfn ; } </s>

Add a function called NAME with type FNTYPE and attributes ATTRS . INSTANCE describes what the function does .


<s> void function_expander :: add_input_operand ( insn_code icode , rtx x ) { unsigned int opno = m_ops . length ( ) ; const insn_operand_data & operand = insn_data [ icode ] . operand [ opno ] ; machine_mode mode = operand . mode ; if ( mode == VOIDmode ) { gcc_assert ( operand . predicate == aarch64_any_register_operand ) ; mode = GET_MODE ( x ) ; } else if ( ! VECTOR_MODE_P ( GET_MODE ( x ) ) && VECTOR_MODE_P ( mode ) ) x = expand_vector_broadcast ( mode , x ) ; else if ( GET_MODE_CLASS ( mode ) == MODE_VECTOR_BOOL ) { gcc_assert ( GET_MODE ( x ) == VNx16BImode ) ; x = gen_lowpart ( mode , x ) ; } m_ops . safe_grow ( m_ops . length ( ) + 1 , true ) ; create_input_operand ( & m_ops . last ( ) , x , mode ) ; } </s>

Create op and add it into M_OPS and increase OPNO .


<s> void function_expander :: add_integer_operand ( HOST_WIDE_INT x ) { m_ops . safe_grow ( m_ops . length ( ) + 1 , true ) ; create_integer_operand ( & m_ops . last ( ) , x ) ; } </s>


<s> void function_expander :: add_output_operand ( insn_code icode ) { unsigned int opno = m_ops . length ( ) ; machine_mode mode = insn_data [ icode ] . operand [ opno ] . mode ; m_ops . safe_grow ( opno + 1 , true ) ; create_output_operand ( & m_ops . last ( ) , possible_target , mode ) ; } </s>

Create output and add it into M_OPS and increase OPNO .


<s> void function_builder :: add_overloaded_function ( const function_instance & instance , uint64_t required_extensions ) { char * name = get_name ( instance , true ) ; if ( registered_function * * map_value = m_overload_names . get ( name ) ) { gcc_assert ( ( * map_value ) -> instance == instance && ( ( * map_value ) -> required_extensions & ~ required_extensions ) == 0 ) ; obstack_free ( & m_string_obstack , name ) ; } else { registered_function & rfn = add_function ( instance , name , m_overload_type , NULL_TREE , required_extensions , true , m_direct_overloads ) ; m_overload_names . put ( name , & rfn ) ; } } </s>

Add one function decl for INSTANCE , to be used with manual overload resolution . REQUIRED_EXTENSIONS are the set of architecture extensions that the function requires . For simplicity , deal with duplicate attempts to add the same function , including cases in which the new function requires more features than the original one did . In that case we 'll check whether the required features are available as part of resolving the function to the relevant unique function .


<s> void function_builder :: add_overloaded_functions ( const function_group_info & group , mode_suffix_index mode ) { unsigned int explicit_type0 = ( * group . shape ) -> explicit_type_suffix_p ( 0 ) ; unsigned int explicit_type1 = ( * group . shape ) -> explicit_type_suffix_p ( 1 ) ; for ( unsigned int pi = 0 ; group . preds [ pi ] != NUM_PREDS ; ++ pi ) { if ( ! explicit_type0 && ! explicit_type1 ) { function_instance instance ( group . base_name , * group . base , * group . shape , mode , types_none [ 0 ] , group . preds [ pi ] ) ; add_overloaded_function ( instance , group . required_extensions ) ; } else for ( unsigned int ti = 0 ; group . types [ ti ] [ 0 ] != NUM_TYPE_SUFFIXES ; ++ ti ) { type_suffix_pair types = { explicit_type0 ? group . types [ ti ] [ 0 ] : NUM_TYPE_SUFFIXES , explicit_type1 ? group . types [ ti ] [ 1 ] : NUM_TYPE_SUFFIXES } ; function_instance instance ( group . base_name , * group . base , * group . shape , mode , types , group . preds [ pi ] ) ; add_overloaded_function ( instance , group . required_extensions ) ; } } } </s>

If we are using manual overload resolution , add one function decl for each overloaded function in GROUP . Take the function base name from GROUP and the mode from MODE .


<s>
static
void
add_sve_type_attribute
(
tree
type
,
unsigned
int
num_zr
,
unsigned
int
num_pr
,
const
char
*
mangled_name
,
const
char
*
acle_name
)
{
tree
mangled_name_tree
=
(
mangled_name
get_identifier
(
mangled_name
)
:
NULL_TREE
)
;
tree
value
=
tree_cons
(
NULL_TREE
,
get_identifier
(
acle_name
)
,
NULL_TREE
)
;
value
=
tree_cons
(
NULL_TREE
,
mangled_name_tree
,
value
)
;
value
=
tree_cons
(
NULL_TREE
,
size_int
(
num_pr
)
,
value
)
;
value
=
tree_cons
(
NULL_TREE
,
size_int
(
num_zr
)
,
value
)
;
TYPE_ATTRIBUTES
(
type
)
=
tree_cons
(
get_identifier
(
"SVE type"
)
,
value
,
TYPE_ATTRIBUTES
(
type
)
)
;
}
</s>

Record that TYPE is an ABI-defined SVE type that contains NUM_ZR SVE vectors and NUM_PR SVE predicates . MANGLED_NAME , if nonnull , is the ABI-defined mangling of the type . ACLE_NAME is the < arm_sve.h > name of the type .


<s> void function_builder :: add_unique_function ( const function_instance & instance , tree return_type , vec < tree > & argument_types , uint64_t required_extensions , bool force_direct_overloads ) { char * name = get_name ( instance , false ) ; tree fntype = build_function_type_array ( return_type , argument_types . length ( ) , argument_types . address ( ) ) ; tree attrs = get_attributes ( instance ) ; registered_function & rfn = add_function ( instance , name , fntype , attrs , required_extensions , false , false ) ; hashval_t hash = instance . hash ( ) ; registered_function * * rfn_slot = function_table -> find_slot_with_hash ( instance , hash , INSERT ) ; gcc_assert ( ! * rfn_slot ) ; * rfn_slot = & rfn ; char * overload_name = get_name ( instance , true ) ; if ( strcmp ( name , overload_name ) != 0 ) { tree attrs = get_attributes ( instance ) ; bool placeholder_p = ! ( m_direct_overloads || force_direct_overloads ) ; add_function ( instance , overload_name , fntype , attrs , required_extensions , false , placeholder_p ) ; } obstack_free ( & m_string_obstack , name ) ; } </s>

Add a built-in function for INSTANCE , with the argument types given by ARGUMENT_TYPES and the return type given by RETURN_TYPE . NAME is the `` full '' name for C function . OVERLOAD_NAME is the `` short '' name for C++ overloaded function . OVERLOAD_NAME can be nullptr because some instance does n't have C++ overloaded function .


<s>
static
tree
get_arm_sve_vector_bits_attributes
(
tree
old_attrs
,
tree
new_sve_type_args
)
{
tree
new_attrs
=
NULL_TREE
;
tree
*
ptr
=
&
new_attrs
;
for
(
tree
attr
=
old_attrs
;
attr
;
attr
=
TREE_CHAIN
(
attr
)
)
{
tree
name
=
get_attribute_name
(
attr
)
;
if
(
is_attribute_p
(
"SVE sizeless type"
,
name
)
)
continue
;
tree
args
=
TREE_VALUE
(
attr
)
;
if
(
is_attribute_p
(
"SVE type"
,
name
)
)
args
=
new_sve_type_args
;
*
ptr
=
tree_cons
(
TREE_PURPOSE
(
attr
)
,
args
,
NULL_TREE
)
;
ptr
=
&
TREE_CHAIN
(
*
ptr
)
;
}
return
new_attrs
;
}
</s>

ATTRS is the attribute list for a sizeless SVE type . Return the attributes of the associated fixed-length SVE type , taking the `` SVE type '' attributes from NEW_SVE_TYPE_ARGS .


"%qs applied to type %qT, which already has a size"

"%qs requires an integer constant expression"

An attribute callback for the `` arm_sve_vector_bits '' attribute .


<s> tree gimple_folder :: load_store_cookie ( tree type ) { return build_int_cst ( build_pointer_type ( type ) , TYPE_ALIGN ( type ) ) ; } </s>

Return the alignment and TBAA argument to an internal load or store function like IFN_MASK_LOAD or IFN_MASK_STORE , given that it accesses memory elements of type TYPE .