<s> void aarch64_split_combinev16qi ( rtx operands [ 3 ] ) { unsigned int dest = REGNO ( operands [ 0 ] ) ; unsigned int src1 = REGNO ( operands [ 1 ] ) ; unsigned int src2 = REGNO ( operands [ 2 ] ) ; machine_mode halfmode = GET_MODE ( operands [ 1 ] ) ; unsigned int halfregs = REG_NREGS ( operands [ 1 ] ) ; rtx destlo , desthi ; gcc_assert ( halfmode == V16QImode ) ; if ( src1 == dest && src2 == dest + halfregs ) { emit_note ( NOTE_INSN_DELETED ) ; return ; } destlo = gen_rtx_REG_offset ( operands [ 0 ] , halfmode , dest , 0 ) ; desthi = gen_rtx_REG_offset ( operands [ 0 ] , halfmode , dest + halfregs , GET_MODE_SIZE ( halfmode ) ) ; if ( reg_overlap_mentioned_p ( operands [ 2 ] , destlo ) && reg_overlap_mentioned_p ( operands [ 1 ] , desthi ) ) { emit_insn ( gen_xorv16qi3 ( operands [ 1 ] , operands [ 1 ] , operands [ 2 ] ) ) ; emit_insn ( gen_xorv16qi3 ( operands [ 2 ] , operands [ 1 ] , operands [ 2 ] ) ) ; emit_insn ( gen_xorv16qi3 ( operands [ 1 ] , operands [ 1 ] , operands [ 2 ] ) ) ; } else if ( ! reg_overlap_mentioned_p ( operands [ 2 ] , destlo ) ) { if ( src1 != dest ) emit_move_insn ( destlo , operands [ 1 ] ) ; if ( src2 != dest + halfregs ) emit_move_insn ( desthi , operands [ 2 ] ) ; } else { if ( src2 != dest + halfregs ) emit_move_insn ( desthi , operands [ 2 ] ) ; if ( src1 != dest ) emit_move_insn ( destlo , operands [ 1 ] ) ; } } </s>

Split operands into moves from op [ 1 ] + op [ 2 ] into op [ 0 ] .


<s> void aarch64_split_compare_and_swap ( rtx operands [ ] ) { gcc_assert ( epilogue_completed ) ; rtx rval , mem , oldval , newval , scratch , x , model_rtx ; machine_mode mode ; bool is_weak ; rtx_code_label * label1 , * label2 ; enum memmodel model ; rval = operands [ 0 ] ; mem = operands [ 1 ] ; oldval = operands [ 2 ] ; newval = operands [ 3 ] ; is_weak = ( operands [ 4 ] != const0_rtx ) ; model_rtx = operands [ 5 ] ; scratch = operands [ 7 ] ; mode = GET_MODE ( mem ) ; model = memmodel_from_int ( INTVAL ( model_rtx ) ) ; bool strong_zero_p = ( ! is_weak && ! aarch64_track_speculation && oldval == const0_rtx && mode != TImode ) ; label1 = NULL ; if ( ! is_weak ) { label1 = gen_label_rtx ( ) ; emit_label ( label1 ) ; } label2 = gen_label_rtx ( ) ; if ( is_mm_sync ( model ) ) aarch64_emit_load_exclusive ( mode , rval , mem , GEN_INT ( MEMMODEL_RELAXED ) ) ; else aarch64_emit_load_exclusive ( mode , rval , mem , model_rtx ) ; if ( strong_zero_p ) x = gen_rtx_NE ( VOIDmode , rval , const0_rtx ) ; else { rtx cc_reg = aarch64_gen_compare_reg_maybe_ze ( NE , rval , oldval , mode ) ; x = gen_rtx_NE ( VOIDmode , cc_reg , const0_rtx ) ; } x = gen_rtx_IF_THEN_ELSE ( VOIDmode , x , gen_rtx_LABEL_REF ( Pmode , label2 ) , pc_rtx ) ; aarch64_emit_unlikely_jump ( gen_rtx_SET ( pc_rtx , x ) ) ; aarch64_emit_store_exclusive ( mode , scratch , mem , newval , model_rtx ) ; if ( ! is_weak ) { if ( aarch64_track_speculation ) { rtx cc_reg = aarch64_gen_compare_reg ( NE , scratch , const0_rtx ) ; x = gen_rtx_NE ( GET_MODE ( cc_reg ) , cc_reg , const0_rtx ) ; } else x = gen_rtx_NE ( VOIDmode , scratch , const0_rtx ) ; x = gen_rtx_IF_THEN_ELSE ( VOIDmode , x , gen_rtx_LABEL_REF ( Pmode , label1 ) , pc_rtx ) ; aarch64_emit_unlikely_jump ( gen_rtx_SET ( pc_rtx , x ) ) ; } else aarch64_gen_compare_reg ( NE , scratch , const0_rtx ) ; emit_label ( label2 ) ; if ( strong_zero_p ) aarch64_gen_compare_reg ( NE , rval , const0_rtx ) ; if ( is_mm_sync ( model ) ) aarch64_emit_post_barrier ( model ) ; } </s>


<s> bool aarch64_split_dimode_const_store ( rtx dst , rtx src ) { rtx lo = gen_lowpart ( SImode , src ) ; rtx hi = gen_highpart_mode ( SImode , DImode , src ) ; bool size_p = optimize_function_for_size_p ( cfun ) ; if ( ! rtx_equal_p ( lo , hi ) ) return false ; unsigned int orig_cost = aarch64_internal_mov_immediate ( NULL_RTX , src , false , DImode ) ; unsigned int lo_cost = aarch64_internal_mov_immediate ( NULL_RTX , lo , false , SImode ) ; if ( size_p && orig_cost <= lo_cost ) return false ; if ( ! size_p && ( orig_cost <= lo_cost + 1 ) ) return false ; rtx mem_lo = adjust_address ( dst , SImode , 0 ) ; if ( ! aarch64_mem_pair_operand ( mem_lo , SImode ) ) return false ; rtx tmp_reg = gen_reg_rtx ( SImode ) ; aarch64_expand_mov_immediate ( tmp_reg , lo ) ; rtx mem_hi = aarch64_move_pointer ( mem_lo , GET_MODE_SIZE ( SImode ) ) ; emit_move_insn ( mem_lo , tmp_reg ) ; emit_move_insn ( mem_hi , tmp_reg ) ; return true ; } </s>

Split a DImode store of a CONST_INT SRC to MEM DST as two SImode stores . Handle the case when the constant has identical bottom and top halves . This is beneficial when the two stores can be merged into an STP and we avoid synthesising potentially expensive immediates twice . Return true if such a split is possible .


<s> void aarch64_split_simd_move ( rtx dst , rtx src ) { machine_mode src_mode = GET_MODE ( src ) ; machine_mode dst_mode = GET_MODE ( dst ) ; gcc_assert ( VECTOR_MODE_P ( dst_mode ) ) ; if ( REG_P ( dst ) && REG_P ( src ) ) { gcc_assert ( VECTOR_MODE_P ( src_mode ) ) ; emit_insn ( gen_aarch64_split_simd_mov ( src_mode , dst , src ) ) ; } } </s>


<s> void aarch64_split_sve_subreg_move ( rtx dest , rtx ptrue , rtx src ) { machine_mode mode_with_wider_elts = aarch64_sve_int_mode ( GET_MODE ( dest ) ) ; machine_mode mode_with_narrower_elts = aarch64_sve_int_mode ( GET_MODE ( src ) ) ; if ( GET_MODE_UNIT_SIZE ( mode_with_wider_elts ) < GET_MODE_UNIT_SIZE ( mode_with_narrower_elts ) ) std :: swap ( mode_with_wider_elts , mode_with_narrower_elts ) ; unsigned int unspec = aarch64_sve_rev_unspec ( mode_with_narrower_elts ) ; machine_mode pred_mode = aarch64_sve_pred_mode ( mode_with_wider_elts ) ; ptrue = gen_lowpart ( pred_mode , ptrue ) ; dest = aarch64_replace_reg_mode ( dest , mode_with_wider_elts ) ; src = aarch64_replace_reg_mode ( src , mode_with_wider_elts ) ; emit_insn ( gen_aarch64_pred ( unspec , mode_with_wider_elts , dest , ptrue , src ) ) ; } </s>

Split a * aarch64_sve_mov < mode > _subreg_be pattern with the given operands .


On AArch64 we have an ABI defined safe buffer . This constant is used to determining the probe offset for alloca .


<s> static tree aarch64_stack_protect_guard ( void ) { if ( aarch64_stack_protector_guard == SSP_GLOBAL ) return default_stack_protect_guard ( ) ; return NULL_TREE ; } </s>

Implement TARGET_STACK_PROTECT_GUARD . In case of a global variable based guard use the default else return a null tree .


Implements TARGET_ASM_FILE_START . Output the assembly header .


<s> bool aarch64_stepped_int_parallel_p ( rtx op , int step ) { if ( GET_CODE ( op ) != PARALLEL || ! CONST_INT_P ( XVECEXP ( op , 0 , 0 ) ) ) return false ; unsigned HOST_WIDE_INT base = UINTVAL ( XVECEXP ( op , 0 , 0 ) ) ; for ( int i = 1 ; i < XVECLEN ( op , 0 ) ; ++ i ) if ( ! CONST_INT_P ( XVECEXP ( op , 0 , i ) ) || UINTVAL ( XVECEXP ( op , 0 , i ) ) != base + i * step ) return false ; return true ; } </s>

Return true if OP is a PARALLEL of CONST_INTs that form a linear series with step STEP .


<s> static rtx aarch64_strip_extend ( rtx x , bool strip_shift ) { scalar_int_mode mode ; rtx op = x ; if ( ! is_a < scalar_int_mode > ( GET_MODE ( op ) , & mode ) ) return op ; if ( ( GET_CODE ( op ) == ZERO_EXTRACT || GET_CODE ( op ) == SIGN_EXTRACT ) && XEXP ( op , 2 ) == const0_rtx && GET_CODE ( XEXP ( op , 0 ) ) == MULT && aarch64_is_extend_from_extract ( mode , XEXP ( XEXP ( op , 0 ) , 1 ) , XEXP ( op , 1 ) ) ) return XEXP ( XEXP ( op , 0 ) , 0 ) ; if ( GET_CODE ( op ) == AND && GET_CODE ( XEXP ( op , 0 ) ) == MULT && CONST_INT_P ( XEXP ( XEXP ( op , 0 ) , 1 ) ) && CONST_INT_P ( XEXP ( op , 1 ) ) && aarch64_uxt_size ( exact_log2 ( INTVAL ( XEXP ( XEXP ( op , 0 ) , 1 ) ) ) , INTVAL ( XEXP ( op , 1 ) ) ) != 0 ) return XEXP ( XEXP ( op , 0 ) , 0 ) ; if ( strip_shift && GET_CODE ( op ) == ASHIFT && CONST_INT_P ( XEXP ( op , 1 ) ) && ( ( unsigned HOST_WIDE_INT ) INTVAL ( XEXP ( op , 1 ) ) ) <= 4 ) op = XEXP ( op , 0 ) ; if ( GET_CODE ( op ) == ZERO_EXTEND || GET_CODE ( op ) == SIGN_EXTEND ) op = XEXP ( op , 0 ) ; if ( op != x ) return op ; return x ; } </s>

Helper function for rtx cost calculation . Strip an extend expression from X . Returns the inner operand if successful , or the original expression on failure . We deal with a number of possible canonicalization variations here .


<s> static rtx aarch64_strip_shift ( rtx x ) { rtx op = x ; if ( ( GET_CODE ( op ) == ASHIFT || GET_CODE ( op ) == ASHIFTRT || GET_CODE ( op ) == LSHIFTRT || GET_CODE ( op ) == ROTATERT || GET_CODE ( op ) == ROTATE ) && CONST_INT_P ( XEXP ( op , 1 ) ) ) return XEXP ( op , 0 ) ; if ( GET_CODE ( op ) == MULT && CONST_INT_P ( XEXP ( op , 1 ) ) && ( ( unsigned ) exact_log2 ( INTVAL ( XEXP ( op , 1 ) ) ) ) < 64 ) return XEXP ( op , 0 ) ; return x ; } </s>

Helper function for rtx cost calculation . Strip a shift expression from X . Returns the inner operand if successful , or the original expression on failure .


<s> static rtx aarch64_struct_value_rtx ( tree fndecl ATTRIBUTE_UNUSED , int incoming ATTRIBUTE_UNUSED ) { return gen_rtx_REG ( Pmode , AARCH64_STRUCT_VALUE_REGNUM ) ; } </s>


<s> void aarch64_subvti_scratch_regs ( rtx op1 , rtx op2 , rtx * low_dest , rtx * low_in1 , rtx * low_in2 , rtx * high_dest , rtx * high_in1 , rtx * high_in2 ) { * low_dest = gen_reg_rtx ( DImode ) ; * low_in1 = simplify_gen_subreg ( DImode , op1 , TImode , subreg_lowpart_offset ( DImode , TImode ) ) ; * low_in2 = simplify_gen_subreg ( DImode , op2 , TImode , subreg_lowpart_offset ( DImode , TImode ) ) ; * high_dest = gen_reg_rtx ( DImode ) ; * high_in1 = simplify_gen_subreg ( DImode , op1 , TImode , subreg_highpart_offset ( DImode , TImode ) ) ; * high_in2 = simplify_gen_subreg ( DImode , op2 , TImode , subreg_highpart_offset ( DImode , TImode ) ) ; } </s>

Generate DImode scratch registers for 128-bit ( TImode ) subtraction . This function differs from 'arch64_addti_scratch_regs ' in that OP1 can be an immediate constant ( zero ) . We must call subreg_highpart_offset with DImode and TImode arguments , otherwise VOIDmode will be used for the const_int which generates an internal error from subreg_size_highpart_offset which does not expect a size of zero . OP1 represents the TImode destination operand 1 OP2 represents the TImode destination operand 2 LOW_DEST represents the low half ( DImode ) of TImode operand 0 LOW_IN1 represents the low half ( DImode ) of TImode operand 1 LOW_IN2 represents the low half ( DImode ) of TImode operand 2 HIGH_DEST represents the high half ( DImode ) of TImode operand 0 HIGH_IN1 represents the high half ( DImode ) of TImode operand 1 HIGH_IN2 represents the high half ( DImode ) of TImode operand 2 .


<s> static inline void aarch64_sub_sp ( rtx temp1 , rtx temp2 , poly_int64 delta , bool frame_related_p , bool emit_move_imm = true ) { aarch64_add_offset ( Pmode , stack_pointer_rtx , stack_pointer_rtx , - delta , temp1 , temp2 , frame_related_p , emit_move_imm ) ; } </s>

Subtract DELTA from the stack pointer , marking the instructions frame-related if FRAME_RELATED_P . TEMP1 is available as a temporary if nonnull .


<s> static const predefined_function_abi & aarch64_sve_abi ( void ) { predefined_function_abi & sve_abi = function_abis [ ARM_PCS_SVE ] ; if ( ! sve_abi . initialized_p ( ) ) { HARD_REG_SET full_reg_clobbers = default_function_abi . full_reg_clobbers ( ) ; for ( int regno = V8_REGNUM ; regno <= V23_REGNUM ; ++ regno ) CLEAR_HARD_REG_BIT ( full_reg_clobbers , regno ) ; for ( int regno = P4_REGNUM ; regno <= P15_REGNUM ; ++ regno ) CLEAR_HARD_REG_BIT ( full_reg_clobbers , regno ) ; sve_abi . initialize ( ARM_PCS_SVE , full_reg_clobbers ) ; } return sve_abi ; } </s>


<s> bool aarch64_sve_addvl_addpl_immediate_p ( rtx x ) { poly_int64 value ; return ( poly_int_rtx_p ( x , & value ) && aarch64_sve_addvl_addpl_immediate_p ( value ) ) ; } </s>


<s> static unsigned int aarch64_sve_adjust_stmt_cost ( vect_cost_for_stmt kind , stmt_vec_info stmt_info , tree vectype , unsigned int stmt_cost ) { if ( kind == vector_stmt && aarch64_extending_load_p ( stmt_info ) ) stmt_cost = 0 ; if ( kind == vector_stmt && aarch64_integer_truncation_p ( stmt_info ) ) stmt_cost = 0 ; if ( STMT_VINFO_GROUPED_ACCESS ( stmt_info ) ) { stmt_vec_info first = DR_GROUP_FIRST_ELEMENT ( stmt_info ) ; unsigned int count = DR_GROUP_SIZE ( first ) - DR_GROUP_GAP ( first ) ; unsigned int elt_bits = GET_MODE_UNIT_BITSIZE ( TYPE_MODE ( vectype ) ) ; if ( multiple_p ( count * elt_bits , 256 ) && aarch64_advsimd_ldp_stp_p ( kind , stmt_info ) ) stmt_cost *= 2 ; } return stmt_cost ; } </s>

STMT_COST is the cost calculated by aarch64_builtin_vectorization_cost for STMT_INFO , which has cost kind KIND and which when vectorized would operate on vector type VECTYPE . Adjust the cost as necessary for SVE targets .


<s> bool aarch64_sve_arith_immediate_p ( machine_mode mode , rtx x , bool negate_p ) { rtx elt = unwrap_const_vec_duplicate ( x ) ; if ( ! CONST_INT_P ( elt ) ) return false ; HOST_WIDE_INT val = INTVAL ( elt ) ; if ( negate_p ) val = - val ; val &= GET_MODE_MASK ( GET_MODE_INNER ( mode ) ) ; if ( val & 0xff ) return IN_RANGE ( val , 0 , 0xff ) ; return IN_RANGE ( val , 0 , 0xff00 ) ; } </s>

Return true if X is a valid immediate for the SVE ADD and SUB instructions . Negate X first if NEGATE_P is true .


<s> bool aarch64_sve_bitmask_immediate_p ( rtx x ) { rtx elt ; return ( const_vec_duplicate_p ( x , & elt ) && CONST_INT_P ( elt ) && aarch64_bitmask_imm ( INTVAL ( elt ) , GET_MODE_INNER ( GET_MODE ( x ) ) ) ) ; } </s>

Return true if X is a valid immediate operand for an SVE logical instruction such as AND .


<s> bool aarch64_sve_cmp_immediate_p ( rtx x , bool signed_p ) { x = unwrap_const_vec_duplicate ( x ) ; return ( CONST_INT_P ( x ) && ( signed_p ? IN_RANGE ( INTVAL ( x ) , - 16 , 15 ) : IN_RANGE ( INTVAL ( x ) , 0 , 127 ) ) ) ; } </s>

Return true if X is a valid immediate operand for an SVE CMP instruction . SIGNED_P says whether the operand is signed rather than unsigned .


<s> bool aarch64_sve_cnt_immediate_p ( rtx x ) { poly_int64 value ; return poly_int_rtx_p ( x , & value ) && aarch64_sve_cnt_immediate_p ( value ) ; } </s>


<s> static unsigned int aarch64_sve_container_bits ( machine_mode mode ) { unsigned int vec_flags = aarch64_classify_vector_mode ( mode ) ; poly_uint64 vector_bits = ( vec_flags & ( VEC_PARTIAL | VEC_SVE_PRED ) ? BITS_PER_SVE_VECTOR : GET_MODE_BITSIZE ( mode ) ) ; return vector_element_size ( vector_bits , GET_MODE_NUNITS ( mode ) ) ; } </s>

MODE is some form of SVE vector mode . For data modes , return the number of vector register bits that each element of MODE occupies , such as 64 for both VNx2DImode and VNx2SImode ( where each 32-bit value is stored in a 64-bit container ) . For predicate modes , return the number of data bits controlled by each significant predicate bit .


<s> static scalar_int_mode aarch64_sve_container_int_mode ( machine_mode mode ) { return int_mode_for_size ( aarch64_sve_container_bits ( mode ) , 0 ) . require ( ) ; } </s>

Return an integer element mode that contains exactly aarch64_sve_container_bits ( MODE ) bits . This is wider than aarch64_sve_element_int_mode if MODE is a partial vector , otherwise it 's the same .


<s> opt_machine_mode aarch64_sve_data_mode ( scalar_mode inner_mode , poly_uint64 nunits ) { enum mode_class mclass = ( is_a < scalar_float_mode > ( inner_mode ) ? MODE_VECTOR_FLOAT : MODE_VECTOR_INT ) ; machine_mode mode ; FOR_EACH_MODE_IN_CLASS ( mode , mclass ) if ( inner_mode == GET_MODE_INNER ( mode ) && known_eq ( nunits , GET_MODE_NUNITS ( mode ) ) && aarch64_sve_data_mode_p ( mode ) ) return mode ; return opt_machine_mode ( ) ; } </s>

Return the SVE vector mode that has NUNITS elements of mode INNER_MODE .


<s> static bool aarch64_sve_data_mode_p ( machine_mode mode ) { return aarch64_classify_vector_mode ( mode ) & VEC_SVE_DATA ; } </s>

Return true if MODE is an SVE data vector mode ; either a single vector or a structure of vectors .


<s> bool aarch64_sve_dup_immediate_p ( rtx x ) { x = aarch64_bit_representation ( unwrap_const_vec_duplicate ( x ) ) ; if ( ! CONST_INT_P ( x ) ) return false ; HOST_WIDE_INT val = INTVAL ( x ) ; if ( val & 0xff ) return IN_RANGE ( val , - 0x80 , 0x7f ) ; return IN_RANGE ( val , - 0x8000 , 0x7f00 ) ; } </s>

Return true if X is a valid immediate for the SVE DUP and CPY instructions .


<s> static scalar_int_mode aarch64_sve_element_int_mode ( machine_mode mode ) { poly_uint64 vector_bits = ( GET_MODE_CLASS ( mode ) == MODE_VECTOR_BOOL ? BITS_PER_SVE_VECTOR : GET_MODE_BITSIZE ( mode ) ) ; unsigned int elt_bits = vector_element_size ( vector_bits , GET_MODE_NUNITS ( mode ) ) ; return int_mode_for_size ( elt_bits , 0 ) . require ( ) ; } </s>

Return the integer element mode associated with SVE mode MODE .


<s> static rtx aarch64_sve_emit_int_cmp ( rtx target , machine_mode pred_mode , rtx_code cmp , machine_mode data_mode , rtx op1 , rtx op2 ) { insn_code icode = code_for_aarch64_pred_cmp ( cmp , data_mode ) ; expand_operand ops [ 5 ] ; create_output_operand ( & ops [ 0 ] , target , pred_mode ) ; create_input_operand ( & ops [ 1 ] , CONSTM1_RTX ( pred_mode ) , pred_mode ) ; create_integer_operand ( & ops [ 2 ] , SVE_KNOWN_PTRUE ) ; create_input_operand ( & ops [ 3 ] , op1 , data_mode ) ; create_input_operand ( & ops [ 4 ] , op2 , data_mode ) ; expand_insn ( icode , 5 , ops ) ; return ops [ 0 ] . value ; } </s>

Emit a comparison CMP between OP0 and OP1 , both of which have mode DATA_MODE , and return the result in a predicate of mode PRED_MODE . Use TARGET as the target register if nonnull and convenient .


Initialize register TARGET from the elements in PARALLEL rtx VALS .


Subroutine of aarch64_sve_expand_vector_init . Works as follows : ( a ) Initialize TARGET by broadcasting element NELTS_REQD - 1 of BUILDER . ( b ) Skip trailing elements from BUILDER , which are the same as element NELTS_REQD - 1 . ( c ) Insert earlier elements in reverse order in TARGET using insr .


<s> bool aarch64_sve_float_arith_immediate_p ( rtx x , bool negate_p ) { rtx elt ; REAL_VALUE_TYPE r ; if ( ! const_vec_duplicate_p ( x , & elt ) || GET_CODE ( elt ) != CONST_DOUBLE ) return false ; r = * CONST_DOUBLE_REAL_VALUE ( elt ) ; if ( negate_p ) r = real_value_negate ( & r ) ; if ( real_equal ( & r , & dconst1 ) ) return true ; if ( real_equal ( & r , & dconsthalf ) ) return true ; return false ; } </s>

Return true if X is a valid immediate operand for an SVE FADD or FSUB instruction . Negate X first if NEGATE_P is true .


<s> bool aarch64_sve_float_mul_immediate_p ( rtx x ) { rtx elt ; return ( const_vec_duplicate_p ( x , & elt ) && GET_CODE ( elt ) == CONST_DOUBLE && ( real_equal ( CONST_DOUBLE_REAL_VALUE ( elt ) , & dconsthalf ) || real_equal ( CONST_DOUBLE_REAL_VALUE ( elt ) , & dconst2 ) ) ) ; } </s>

Return true if X is a valid immediate operand for an SVE FMUL instruction .


<s> bool aarch64_sve_index_immediate_p ( rtx base_or_step ) { return ( CONST_INT_P ( base_or_step ) && IN_RANGE ( INTVAL ( base_or_step ) , - 16 , 15 ) ) ; } </s>

Return true if BASE_OR_STEP is a valid immediate operand for an SVE INDEX instruction .


<s> machine_mode aarch64_sve_int_mode ( machine_mode mode ) { scalar_int_mode int_mode = aarch64_sve_element_int_mode ( mode ) ; return aarch64_sve_data_mode ( int_mode , GET_MODE_NUNITS ( mode ) ) . require ( ) ; } </s>

Return the integer vector mode associated with SVE mode MODE . Unlike related_int_vector_mode , this can handle the case in which MODE is a predicate ( and thus has a different total size ) .


<s> bool aarch64_sve_ld1ro_operand_p ( rtx op , scalar_mode elem_mode ) { return aarch64_sve_ld1rq_ld1ro_operand_p ( op , OImode , elem_mode ) ; } </s>

Return true if OP is a valid MEM operand for an SVE LD1RO instruction for accessing a vector where the element size is specified by ` elem_mode ` .


<s> bool aarch64_sve_ld1rq_ld1ro_operand_p ( rtx op , machine_mode mode , scalar_mode elem_mode ) { struct aarch64_address_info addr ; if ( ! MEM_P ( op ) || ! aarch64_classify_address ( & addr , XEXP ( op , 0 ) , elem_mode , false ) ) return false ; if ( addr . type == ADDRESS_REG_IMM ) return offset_4bit_signed_scaled_p ( mode , addr . const_offset ) ; if ( addr . type == ADDRESS_REG_REG ) return ( 1U << addr . shift ) == GET_MODE_SIZE ( elem_mode ) ; return false ; } </s>

Return true if OP is a valid MEM operand for an SVE LD1R { Q , O } instruction where the size of the read data is specified by ` mode ` and the size of the vector elements are specified by ` elem_mode ` .


<s> bool aarch64_sve_ld1rq_operand_p ( rtx op ) { return aarch64_sve_ld1rq_ld1ro_operand_p ( op , TImode , GET_MODE_INNER ( GET_MODE ( op ) ) ) ; } </s>

Return true if OP is a valid MEM operand for an SVE LD1RQ instruction .


<s> bool aarch64_sve_ld1r_operand_p ( rtx op ) { struct aarch64_address_info addr ; scalar_mode mode ; return ( MEM_P ( op ) && is_a < scalar_mode > ( GET_MODE ( op ) , & mode ) && aarch64_classify_address ( & addr , XEXP ( op , 0 ) , mode , false ) && addr . type == ADDRESS_REG_IMM && offset_6bit_unsigned_scaled_p ( mode , addr . const_offset ) ) ; } </s>

Return true if OP is a valid MEM operand for an SVE LD1R instruction .


<s> bool aarch64_sve_ldff1_operand_p ( rtx op ) { if ( ! MEM_P ( op ) ) return false ; struct aarch64_address_info addr ; if ( ! aarch64_classify_address ( & addr , XEXP ( op , 0 ) , GET_MODE ( op ) , false ) ) return false ; if ( addr . type == ADDRESS_REG_IMM ) return known_eq ( addr . const_offset , 0 ) ; return addr . type == ADDRESS_REG_REG ; } </s>

Return true if OP is a valid MEM operand for an SVE LDFF1 instruction .


<s> bool aarch64_sve_ldnf1_operand_p ( rtx op ) { struct aarch64_address_info addr ; return ( MEM_P ( op ) && aarch64_classify_address ( & addr , XEXP ( op , 0 ) , GET_MODE ( op ) , false ) && addr . type == ADDRESS_REG_IMM ) ; } </s>

Return true if OP is a valid MEM operand for an SVE LDNF1 instruction .


<s> bool aarch64_sve_ldr_operand_p ( rtx op ) { struct aarch64_address_info addr ; return ( MEM_P ( op ) && aarch64_classify_address ( & addr , XEXP ( op , 0 ) , GET_MODE ( op ) , false , ADDR_QUERY_ANY ) && addr . type == ADDRESS_REG_IMM ) ; } </s>

Return true if OP is a valid MEM operand for an SVE LDR instruction . The conditions for STR are the same .


<s> bool aarch64_sve_mode_p ( machine_mode mode ) { return aarch64_classify_vector_mode ( mode ) & VEC_ANY_SVE ; } </s>

Return true if MODE is any form of SVE mode , including predicates , vectors and structures .


<s> static rtx aarch64_sve_move_pred_via_while ( rtx target , machine_mode mode , unsigned int vl ) { rtx limit = force_reg ( DImode , gen_int_mode ( vl , DImode ) ) ; target = aarch64_target_reg ( target , mode ) ; emit_insn ( gen_while ( UNSPEC_WHILELO , DImode , mode , target , const0_rtx , limit ) ) ; return target ; } </s>

Use WHILE to set a predicate register of mode MODE in which the first VL bits are set and the rest are clear . Use TARGET for the register if it 's nonnull and convenient .


<s> bool aarch64_sve_pred_dominates_p ( rtx * pred1 , rtx pred2 ) { machine_mode mode = GET_MODE ( pred2 ) ; gcc_assert ( GET_MODE_CLASS ( mode ) == MODE_VECTOR_BOOL && mode == GET_MODE ( pred1 [ 0 ] ) && aarch64_sve_gp_strictness ( pred1 [ 1 ] , SImode ) ) ; return ( pred1 [ 0 ] == CONSTM1_RTX ( mode ) || INTVAL ( pred1 [ 1 ] ) == SVE_RELAXED_GP || rtx_equal_p ( pred1 [ 0 ] , pred2 ) ) ; } </s>

Return true if predicate PRED1 [ 0 ] is true whenever predicate PRED2 is true , or alternatively if we know that the operation predicated by PRED1 [ 0 ] is safe to perform whenever PRED2 is true . PRED1 [ 1 ] is a aarch64_sve_gp_strictness operand that describes the operation predicated by PRED1 [ 0 ] .


<s> machine_mode aarch64_sve_pred_mode ( machine_mode mode ) { unsigned int bits = aarch64_sve_container_bits ( mode ) ; return aarch64_sve_pred_mode ( bits / BITS_PER_UNIT ) . require ( ) ; } </s>

Return the SVE predicate mode that should be used to control SVE mode MODE .


<s> static bool aarch64_sve_pred_mode_p ( machine_mode mode ) { return ( TARGET_SVE && ( mode == VNx16BImode || mode == VNx8BImode || mode == VNx4BImode || mode == VNx2BImode ) ) ; } </s>

Return true if MODE is an SVE predicate mode .


<s> static bool aarch64_sve_pred_valid_immediate ( rtx x , simd_immediate_info * info ) { if ( aarch64_sve_ptrue_svpattern_p ( x , info ) ) return true ; if ( x == CONST0_RTX ( GET_MODE ( x ) ) ) { if ( info ) * info = simd_immediate_info ( DImode , 0 ) ; return true ; } rtx_vector_builder builder ; if ( ! aarch64_get_sve_pred_bits ( builder , x ) ) return false ; unsigned int elt_size = aarch64_widest_sve_pred_elt_size ( builder ) ; if ( int vl = aarch64_partial_ptrue_length ( builder , elt_size ) ) { machine_mode mode = aarch64_sve_pred_mode ( elt_size ) . require ( ) ; aarch64_svpattern pattern = aarch64_svpattern_for_vl ( mode , vl ) ; if ( pattern != AARCH64_NUM_SVPATTERNS ) { if ( info ) { scalar_int_mode int_mode = aarch64_sve_element_int_mode ( mode ) ; * info = simd_immediate_info ( int_mode , pattern ) ; } return true ; } } return false ; } </s>

Return true if X is a valid SVE predicate . If INFO is nonnull , use it to describe valid immediates .


<s> bool aarch64_sve_prefetch_operand_p ( rtx op , machine_mode mode ) { struct aarch64_address_info addr ; if ( ! aarch64_classify_address ( & addr , op , mode , false ) ) return false ; if ( addr . type == ADDRESS_REG_IMM ) return known_eq ( addr . const_offset , 0 ) ; return addr . type == ADDRESS_REG_REG ; } </s>

Return true if OP is a valid address for an SVE PRF [ BHWD ] instruction , addressing memory of mode MODE .


<s> bool aarch64_sve_ptrue_svpattern_p ( rtx x , struct simd_immediate_info * info ) { if ( GET_CODE ( x ) != CONST ) return false ; x = XEXP ( x , 0 ) ; if ( GET_CODE ( x ) != UNSPEC || XINT ( x , 1 ) != UNSPEC_PTRUE ) return false ; if ( info ) { aarch64_svpattern pattern = ( aarch64_svpattern ) INTVAL ( XVECEXP ( x , 0 , 0 ) ) ; machine_mode pred_mode = GET_MODE ( XVECEXP ( x , 0 , 1 ) ) ; scalar_int_mode int_mode = aarch64_sve_element_int_mode ( pred_mode ) ; * info = simd_immediate_info ( int_mode , pattern ) ; } return true ; } </s>

Return true if X is an UNSPEC_PTRUE constant of the form : ( const ( unspec [ PATTERN ZERO ] UNSPEC_PTRUE ) ) where PATTERN is the svpattern as a CONST_INT and where ZERO is a zero constant of the required PTRUE mode ( which can have fewer elements than X 's mode , if zero bits are significant ) . If so , and if INFO is nonnull , describe the immediate in INFO .


<s> static unsigned int aarch64_sve_rev_unspec ( machine_mode mode ) { switch ( GET_MODE_UNIT_SIZE ( mode ) ) { case 1 : return UNSPEC_REVB ; case 2 : return UNSPEC_REVH ; case 4 : return UNSPEC_REVW ; } gcc_unreachable ( ) ; } </s>

Return the SVE REV [ BHW ] unspec for reversing quantites of mode MODE stored in wider integer containers .


<s> bool aarch64_sve_same_pred_for_ptest_p ( rtx * pred1 , rtx * pred2 ) { machine_mode mode = GET_MODE ( pred1 [ 0 ] ) ; gcc_assert ( GET_MODE_CLASS ( mode ) == MODE_VECTOR_BOOL && mode == GET_MODE ( pred2 [ 0 ] ) && aarch64_sve_ptrue_flag ( pred1 [ 1 ] , SImode ) && aarch64_sve_ptrue_flag ( pred2 [ 1 ] , SImode ) ) ; bool ptrue1_p = ( pred1 [ 0 ] == CONSTM1_RTX ( mode ) || INTVAL ( pred1 [ 1 ] ) == SVE_KNOWN_PTRUE ) ; bool ptrue2_p = ( pred2 [ 0 ] == CONSTM1_RTX ( mode ) || INTVAL ( pred2 [ 1 ] ) == SVE_KNOWN_PTRUE ) ; return ( ptrue1_p && ptrue2_p ) || rtx_equal_p ( pred1 [ 0 ] , pred2 [ 0 ] ) ; } </s>

PRED1 [ 0 ] is a PTEST predicate and PRED1 [ 1 ] is an aarch64_sve_ptrue_flag for it . PRED2 [ 0 ] is the predicate for the instruction whose result is tested by the PTEST and PRED2 [ 1 ] is again an aarch64_sve_ptrue_flag for it . Return true if we can prove that the two predicates are equivalent for PTEST purposes ; that is , if we can replace PRED2 [ 0 ] with PRED1 [ 0 ] without changing behavior .


<s> bool aarch64_sve_scalar_inc_dec_immediate_p ( rtx x ) { poly_int64 value ; return ( poly_int_rtx_p ( x , & value ) && ( aarch64_sve_cnt_immediate_p ( value ) || aarch64_sve_cnt_immediate_p ( - value ) ) ) ; } </s>

Return true if we can add X using a single SVE INC or DEC instruction .


<s> bool aarch64_sve_sqadd_sqsub_immediate_p ( machine_mode mode , rtx x , bool negate_p ) { if ( ! aarch64_sve_arith_immediate_p ( mode , x , negate_p ) ) return false ; rtx elt = unwrap_const_vec_duplicate ( x ) ; return negate_p == ( INTVAL ( elt ) < 0 ) ; } </s>

Return true if X is a valid immediate for the SVE SQADD and SQSUB instructions when applied to mode MODE . Negate X first if NEGATE_P is true .


<s> bool aarch64_sve_struct_memory_operand_p ( rtx op ) { if ( ! MEM_P ( op ) ) return false ; machine_mode mode = GET_MODE ( op ) ; struct aarch64_address_info addr ; if ( ! aarch64_classify_address ( & addr , XEXP ( op , 0 ) , SVE_BYTE_MODE , false , ADDR_QUERY_ANY ) || addr . type != ADDRESS_REG_IMM ) return false ; poly_int64 first = addr . const_offset ; poly_int64 last = first + GET_MODE_SIZE ( mode ) - BYTES_PER_SVE_VECTOR ; return ( offset_4bit_signed_scaled_p ( SVE_BYTE_MODE , first ) && offset_4bit_signed_scaled_p ( SVE_BYTE_MODE , last ) ) ; } </s>

Return true if OP is a valid MEM operand for an SVE_STRUCT mode . We need to be able to access the individual pieces , so the range is different from LD [ 234 ] and ST [ 234 ] .


<s> static bool aarch64_sve_valid_immediate ( unsigned HOST_WIDE_INT val64 , simd_immediate_info * info ) { scalar_int_mode mode = DImode ; unsigned int val32 = val64 & 0xffffffff ; if ( val32 == ( val64 >> 32 ) ) { mode = SImode ; unsigned int val16 = val32 & 0xffff ; if ( val16 == ( val32 >> 16 ) ) { mode = HImode ; unsigned int val8 = val16 & 0xff ; if ( val8 == ( val16 >> 8 ) ) mode = QImode ; } } HOST_WIDE_INT val = trunc_int_for_mode ( val64 , mode ) ; if ( IN_RANGE ( val , - 0x80 , 0x7f ) ) { if ( info ) * info = simd_immediate_info ( mode , val ) ; return true ; } if ( ( val & 0xff ) == 0 && IN_RANGE ( val , - 0x8000 , 0x7f00 ) ) { if ( info ) * info = simd_immediate_info ( mode , val ) ; return true ; } if ( aarch64_bitmask_imm ( val64 , mode ) ) { if ( info ) * info = simd_immediate_info ( mode , val ) ; return true ; } return false ; } </s>

Return true if replicating VAL64 gives a valid immediate for an SVE MOV instruction . If INFO is nonnull , use it to describe valid immediates .


<s> bool aarch64_sve_vector_inc_dec_immediate_p ( rtx x ) { return aarch64_sve_vector_inc_dec_immediate_p ( x , NULL , NULL ) ; } </s>

Return true if X is a valid immediate for an SVE vector INC or DEC instruction .


<s> aarch64_svpattern aarch64_svpattern_for_vl ( machine_mode pred_mode , int vl ) { if ( vl < 0 ) return AARCH64_SV_ALL ; if ( maybe_gt ( vl , GET_MODE_NUNITS ( pred_mode ) ) ) return AARCH64_NUM_SVPATTERNS ; if ( vl >= 1 && vl <= 8 ) return aarch64_svpattern ( AARCH64_SV_VL1 + ( vl - 1 ) ) ; if ( vl >= 16 && vl <= 256 && pow2p_hwi ( vl ) ) return aarch64_svpattern ( AARCH64_SV_VL16 + ( exact_log2 ( vl ) - 4 ) ) ; int max_vl ; if ( GET_MODE_NUNITS ( pred_mode ) . is_constant ( & max_vl ) ) { if ( vl == ( max_vl / 3 ) * 3 ) return AARCH64_SV_MUL3 ; if ( vl == ( max_vl & - 4 ) ) return AARCH64_SV_MUL4 ; if ( vl == ( 1 << floor_log2 ( max_vl ) ) ) return AARCH64_SV_POW2 ; if ( vl == max_vl ) return AARCH64_SV_ALL ; } return AARCH64_NUM_SVPATTERNS ; } </s>

See if there is an svpattern that encodes an SVE predicate of mode PRED_MODE in which the first VL bits are set and the rest are clear . Return the pattern if so , otherwise return AARCH64_NUM_SVPATTERNS . A VL of -1 indicates an all-true vector .


<s> void aarch64_swap_ldrstr_operands ( rtx * operands , bool load ) { rtx mem_1 , mem_2 , base_1 , base_2 , offset_1 , offset_2 ; HOST_WIDE_INT offval_1 , offval_2 ; if ( load ) { mem_1 = operands [ 1 ] ; mem_2 = operands [ 3 ] ; } else { mem_1 = operands [ 0 ] ; mem_2 = operands [ 2 ] ; } extract_base_offset_in_addr ( mem_1 , & base_1 , & offset_1 ) ; extract_base_offset_in_addr ( mem_2 , & base_2 , & offset_2 ) ; offval_1 = INTVAL ( offset_1 ) ; offval_2 = INTVAL ( offset_2 ) ; if ( offval_1 > offval_2 ) { std :: swap ( operands [ 0 ] , operands [ 2 ] ) ; std :: swap ( operands [ 1 ] , operands [ 3 ] ) ; } } </s>

Given OPERANDS of consecutive load/store that can be merged , swap them if they are not in ascending order .


<s> static bool aarch64_symbol_binds_local_p ( const_rtx x ) { return ( SYMBOL_REF_DECL ( x ) ? targetm . binds_local_p ( SYMBOL_REF_DECL ( x ) ) : SYMBOL_REF_LOCAL_P ( x ) ) ; } </s>

Return true if SYMBOL_REF X binds locally .


<s> static bool aarch64_takes_arguments_in_sve_regs_p ( const_tree fntype ) { CUMULATIVE_ARGS args_so_far_v ; aarch64_init_cumulative_args ( & args_so_far_v , NULL_TREE , NULL_RTX , NULL_TREE , 0 , true ) ; cumulative_args_t args_so_far = pack_cumulative_args ( & args_so_far_v ) ; for ( tree chain = TYPE_ARG_TYPES ( fntype ) ; chain && chain != void_list_node ; chain = TREE_CHAIN ( chain ) ) { tree arg_type = TREE_VALUE ( chain ) ; if ( arg_type == error_mark_node ) return false ; function_arg_info arg ( arg_type , true ) ; apply_pass_by_reference_rules ( & args_so_far_v , arg ) ; pure_scalable_type_info pst_info ; if ( pst_info . analyze_registers ( arg . type ) ) { unsigned int end_zr = args_so_far_v . aapcs_nvrn + pst_info . num_zr ( ) ; unsigned int end_pr = args_so_far_v . aapcs_nprn + pst_info . num_pr ( ) ; gcc_assert ( end_zr <= NUM_FP_ARG_REGS && end_pr <= NUM_PR_ARG_REGS ) ; return true ; } targetm . calls . function_arg_advance ( args_so_far , arg ) ; } return false ; } </s>

Return true if a function with type FNTYPE takes arguments in SVE vector or predicate registers .


<s> static rtx aarch64_target_reg ( rtx target , machine_mode mode ) { if ( target && REG_P ( target ) && GET_MODE ( target ) == mode ) return target ; if ( ! can_create_pseudo_p ( ) ) { gcc_assert ( target ) ; return gen_lowpart ( mode , target ) ; } return gen_reg_rtx ( mode ) ; } </s>

Return TARGET if it is nonnull and a register of mode MODE . Otherwise , return a fresh register of mode MODE if we can , or TARGET reinterpreted as MODE if we ca n't .


<s> static void aarch64_test_loading_full_dump ( ) { rtl_dump_test t ( SELFTEST_LOCATION , locate_file ( "aarch64/times-two.rtl" ) ) ; ASSERT_STREQ ( "times_two" , IDENTIFIER_POINTER ( DECL_NAME ( cfun -> decl ) ) ) ; rtx_insn * insn_1 = get_insn_by_uid ( 1 ) ; ASSERT_EQ ( NOTE , GET_CODE ( insn_1 ) ) ; rtx_insn * insn_15 = get_insn_by_uid ( 15 ) ; ASSERT_EQ ( INSN , GET_CODE ( insn_15 ) ) ; ASSERT_EQ ( USE , GET_CODE ( PATTERN ( insn_15 ) ) ) ; ASSERT_EQ ( REG , GET_CODE ( crtl -> return_rtx ) ) ; ASSERT_EQ ( 0 , REGNO ( crtl -> return_rtx ) ) ; ASSERT_EQ ( SImode , GET_MODE ( crtl -> return_rtx ) ) ; } </s>

Selftest for the RTL loader . Verify that the RTL loader copes with a dump from print_rtx_function . This is essentially just a test that class function_reader can handle a real dump , but it also verifies that lookup_reg_by_dump_name correctly handles hard regs . The presence of hard reg names in the dump means that the test is target-specific , hence it is in this file .


<s> unsigned int aarch64_tlsdesc_abi_id ( ) { predefined_function_abi & tlsdesc_abi = function_abis [ ARM_PCS_TLSDESC ] ; if ( ! tlsdesc_abi . initialized_p ( ) ) { HARD_REG_SET full_reg_clobbers ; CLEAR_HARD_REG_SET ( full_reg_clobbers ) ; SET_HARD_REG_BIT ( full_reg_clobbers , R0_REGNUM ) ; SET_HARD_REG_BIT ( full_reg_clobbers , CC_REGNUM ) ; for ( int regno = P0_REGNUM ; regno <= P15_REGNUM ; ++ regno ) SET_HARD_REG_BIT ( full_reg_clobbers , regno ) ; tlsdesc_abi . initialize ( ARM_PCS_TLSDESC , full_reg_clobbers ) ; } return tlsdesc_abi . id ( ) ; } </s>

Return the ID of the TLDESC ABI , initializing the descriptor if has n't been already .


<s> static bool aarch64_tls_symbol_p ( rtx x ) { if ( ! TARGET_HAVE_TLS ) return false ; if ( GET_CODE ( x ) != SYMBOL_REF ) return false ; return SYMBOL_REF_TLS_MODEL ( x ) != 0 ; } </s>

Return true if SYMBOL_REF X is thread local


<s> static bool aarch64_tribools_ok_for_inlining_p ( int caller , int callee , int dont_care , int def ) { if ( callee == dont_care ) return true ; if ( caller == dont_care ) return true ; return ( callee == caller || callee == def ) ; } </s>

Helper for aarch64_can_inline_p . In the case where CALLER and CALLEE are tri-bool options ( yes , no , do n't care ) and the default value is DEF , determine whether to reject inlining .


<s> bool aarch64_uimm12_shift ( HOST_WIDE_INT val ) { return ( ( val & ( ( ( HOST_WIDE_INT ) 0xfff ) << 0 ) ) == val || ( val & ( ( ( HOST_WIDE_INT ) 0xfff ) << 12 ) ) == val ) ; } </s>

Return true if val can be encoded as a 12-bit unsigned immediate with a left shift of 0 or 12 bits .


<s> static unsigned int aarch64_unspec_cond_code ( rtx_code code ) { switch ( code ) { case NE : return UNSPEC_COND_FCMNE ; case EQ : return UNSPEC_COND_FCMEQ ; case LT : return UNSPEC_COND_FCMLT ; case GT : return UNSPEC_COND_FCMGT ; case LE : return UNSPEC_COND_FCMLE ; case GE : return UNSPEC_COND_FCMGE ; case UNORDERED : return UNSPEC_COND_FCMUO ; default : gcc_unreachable ( ) ; } } </s>

Return the UNSPEC_COND_ * code for comparison CODE .


<s> static int aarch64_unspec_may_trap_p ( const_rtx x , unsigned flags ) { switch ( XINT ( x , 1 ) ) { case UNSPEC_GOTSMALLPIC : case UNSPEC_GOTSMALLPIC28K : case UNSPEC_GOTTINYPIC : return 0 ; default : break ; } return default_unspec_may_trap_p ( x , flags ) ; } </s>


<s> bool aarch64_use_pseudo_pic_reg ( void ) { return aarch64_cmodel == AARCH64_CMODEL_SMALL_SPIC ; } </s>

Return 1 if pseudo register should be created and used to hold GOT address for PIC code .


<s> bool aarch64_use_return_insn_p ( void ) { if ( ! reload_completed ) return false ; if ( crtl -> profile ) return false ; return known_eq ( cfun -> machine -> frame . frame_size , 0 ) ; } </s>

Return TRUE if we can use a simple_return insn . This function checks whether the callee saved stack is empty , which means no restore actions are need . The pro_and_epilogue will use this to check whether shrink-wrapping opt is feasible .


<s> int aarch64_uxt_size ( int shift , HOST_WIDE_INT mask ) { if ( shift >= 0 && shift <= 3 ) { int size ; for ( size = 8 ; size <= 32 ; size *= 2 ) { HOST_WIDE_INT bits = ( ( HOST_WIDE_INT ) 1U << size ) - 1 ; if ( mask == bits << shift ) return size ; } } return 0 ; } </s>

Return size in bits of an arithmetic operand which is shifted/scaled and masked such that it is suitable for a UXTB , UXTH , or UXTW extend operator .


"invalid feature modifier %qs in %<-march=%s%>"

Validate a command-line -march option . Parse the arch and extensions ( if any ) specified in STR and throw errors if appropriate . Put the results , if they are valid , in RES and ISA_FLAGS . Return whether the option is valid .


"invalid feature modifier %qs in %<-mcpu=%s%>"

Validate a command-line -mcpu option . Parse the cpu and extensions ( if any ) specified in STR and throw errors if appropriate . Put the results if they are valid in RES and ISA_FLAGS . Return whether the option is valid .


<s>
static
bool
aarch64_validate_mtune
(
const
char
*
str
,
const
struct
processor
*
*
res
)
{
enum
aarch64_parse_opt_result
parse_res
=
aarch64_parse_tune
(
str
,
res
)
;
if
(
parse_res
==
AARCH64_PARSE_OK
)
return
true
;
switch
(
parse_res
)
{
case
AARCH64_PARSE_MISSING_ARG
:
error
(
"missing cpu name in %<-mtune=%s%>"
,
str
)
;
break
;
case
AARCH64_PARSE_INVALID_ARG
:
error
(
"unknown value %qs for %<-mtune%>"
,
str
)
;
aarch64_print_hint_for_core
(
str
)
;
break
;
default
:
gcc_unreachable
(
)
;
}
return
false
;
}
</s>

Validate a command-line -mtune option . Parse the cpu specified in STR and throw errors if appropriate . Put the result , if it is valid , in RES . Return whether the option is valid .

aarch64
"missing cpu name in %<-mtune=%s%>"
"unknown value %qs for %<-mtune%>"


<s> static opt_machine_mode aarch64_vectorize_related_mode ( machine_mode vector_mode , scalar_mode element_mode , poly_uint64 nunits ) { unsigned int vec_flags = aarch64_classify_vector_mode ( vector_mode ) ; poly_uint64 sve_nunits ; if ( ( vec_flags & VEC_SVE_DATA ) && multiple_p ( BYTES_PER_SVE_VECTOR , GET_MODE_SIZE ( element_mode ) , & sve_nunits ) ) { machine_mode sve_mode ; if ( maybe_ne ( nunits , 0U ) ) { if ( multiple_p ( sve_nunits , nunits ) && aarch64_sve_data_mode ( element_mode , nunits ) . exists ( & sve_mode ) ) return sve_mode ; } else { sve_nunits = ordered_min ( sve_nunits , GET_MODE_SIZE ( vector_mode ) ) ; if ( aarch64_sve_data_mode ( element_mode , sve_nunits ) . exists ( & sve_mode ) ) return sve_mode ; } } if ( ( vec_flags & VEC_ADVSIMD ) && known_eq ( nunits , 0U ) && known_eq ( GET_MODE_BITSIZE ( vector_mode ) , 64U ) && maybe_ge ( GET_MODE_BITSIZE ( element_mode ) * GET_MODE_NUNITS ( vector_mode ) , 128U ) ) { machine_mode res = aarch64_simd_container_mode ( element_mode , 128 ) ; if ( VECTOR_MODE_P ( res ) ) return res ; } return default_vectorize_related_mode ( vector_mode , element_mode , nunits ) ; } </s>


<s> static bool aarch64_vectorize_vec_perm_const ( machine_mode vmode , rtx target , rtx op0 , rtx op1 , const vec_perm_indices & sel ) { struct expand_vec_perm_d d ; if ( sel . ninputs ( ) == 1 || ( op0 && rtx_equal_p ( op0 , op1 ) ) ) d . one_vector_p = true ; else if ( sel . all_from_input_p ( 0 ) ) { d . one_vector_p = true ; op1 = op0 ; } else if ( sel . all_from_input_p ( 1 ) ) { d . one_vector_p = true ; op0 = op1 ; } else d . one_vector_p = false ; d . perm . new_vector ( sel . encoding ( ) , d . one_vector_p ? 1 : 2 , sel . nelts_per_input ( ) ) ; d . vmode = vmode ; d . vec_flags = aarch64_classify_vector_mode ( d . vmode ) ; d . target = target ; d . op0 = op0 ; d . op1 = op1 ; d . testing_p = ! target ; if ( ! d . testing_p ) return aarch64_expand_vec_perm_const_1 ( & d ) ; rtx_insn * last = get_last_insn ( ) ; bool ret = aarch64_expand_vec_perm_const_1 ( & d ) ; gcc_assert ( last == get_last_insn ( ) ) ; return ret ; } </s>


<s> static bool aarch64_vector_data_mode_p ( machine_mode mode ) { return aarch64_classify_vector_mode ( mode ) & VEC_ANY_DATA ; } </s>

Return true if MODE is any of the data vector modes , including structure modes .


<s> static bool aarch64_vector_mode_supported_p ( machine_mode mode ) { unsigned int vec_flags = aarch64_classify_vector_mode ( mode ) ; return vec_flags != 0 && ( vec_flags & VEC_STRUCT ) == 0 ; } </s>

Implements target hook vector_mode_supported_p .


<s> int aarch64_vec_fpconst_pow_of_2 ( rtx x ) { int nelts ; if ( GET_CODE ( x ) != CONST_VECTOR || ! CONST_VECTOR_NUNITS ( x ) . is_constant ( & nelts ) ) return - 1 ; if ( GET_MODE_CLASS ( GET_MODE ( x ) ) != MODE_VECTOR_FLOAT ) return - 1 ; int firstval = aarch64_fpconst_pow_of_2 ( CONST_VECTOR_ELT ( x , 0 ) ) ; if ( firstval <= 0 ) return - 1 ; for ( int i = 1 ; i < nelts ; i ++ ) if ( aarch64_fpconst_pow_of_2 ( CONST_VECTOR_ELT ( x , i ) ) != firstval ) return - 1 ; return firstval ; } </s>

If X is a vector of equal CONST_DOUBLE values and that value is Y , return the aarch64_fpconst_pow_of_2 of Y . Otherwise return -1 .


<s> static bool aarch64_verify_type_context ( location_t loc , type_context_kind context , const_tree type , bool silent_p ) { return aarch64_sve :: verify_type_context ( loc , context , type , silent_p ) ; } </s>


"type %qT with %<[[no_unique_address]]%> members "

"type %qT when C++17 is enabled changed to match "

Return TRUE if an argument , whose type is described by TYPE and MODE , shall be passed or returned in simd/fp register ( s ) ( providing these parameter passing registers are available ) . Upon successful return , * COUNT returns the number of needed registers , * BASE_MODE returns the mode of the individual register and when IS_HAF is not NULL , * IS_HA indicates whether or not the argument is a homogeneous floating-point aggregate or a homogeneous short-vector aggregate .


<s> static poly_int64 aarch64_vl_bytes ( machine_mode mode , unsigned int vec_flags ) { if ( vec_flags & VEC_PARTIAL ) return GET_MODE_SIZE ( mode ) ; if ( vec_flags & VEC_SVE_DATA ) return BYTES_PER_SVE_VECTOR ; gcc_assert ( vec_flags & VEC_SVE_PRED ) ; return BYTES_PER_SVE_PRED ; } </s>

Return the number of defined bytes in one constituent vector of SVE mode MODE , which has vector flags VEC_FLAGS .


<s> opt_machine_mode aarch64_vq_mode ( scalar_mode mode ) { switch ( mode ) { case E_DFmode : return V2DFmode ; case E_SFmode : return V4SFmode ; case E_HFmode : return V8HFmode ; case E_BFmode : return V8BFmode ; case E_SImode : return V4SImode ; case E_HImode : return V8HImode ; case E_QImode : return V16QImode ; case E_DImode : return V2DImode ; default : return opt_machine_mode ( ) ; } } </s>

Return the 128-bit Advanced SIMD vector mode for element mode MODE , if it exists .


<s> unsigned int aarch64_widest_sve_pred_elt_size ( rtx_vector_builder & builder ) { unsigned int mask = GET_MODE_SIZE ( DImode ) ; mask |= builder . npatterns ( ) ; unsigned int nelts = builder . encoded_nelts ( ) ; for ( unsigned int i = 1 ; i < nelts ; ++ i ) if ( INTVAL ( builder . elt ( i ) ) != 0 ) { if ( i & 1 ) return 1 ; mask |= i ; } return mask & - mask ; } </s>

BUILDER contains a predicate constant of mode VNx16BI . Return the widest predicate element size it can have ( that is , the largest size for which each element would still be 0 or 1 ) .


<s> void pure_scalable_type_info :: add_piece ( const piece & p ) { if ( ! pieces . is_empty ( ) ) { piece & prev = pieces . last ( ) ; gcc_assert ( VECTOR_MODE_P ( p . mode ) && VECTOR_MODE_P ( prev . mode ) ) ; unsigned int nelems1 , nelems2 ; if ( prev . orig_mode == p . orig_mode && known_eq ( prev . offset + GET_MODE_SIZE ( prev . mode ) , p . offset ) && constant_multiple_p ( GET_MODE_NUNITS ( prev . mode ) , GET_MODE_NUNITS ( p . orig_mode ) , & nelems1 ) && constant_multiple_p ( GET_MODE_NUNITS ( p . mode ) , GET_MODE_NUNITS ( p . orig_mode ) , & nelems2 ) && targetm . array_mode ( p . orig_mode , nelems1 + nelems2 ) . exists ( & prev . mode ) ) { prev . num_zr += p . num_zr ; prev . num_pr += p . num_pr ; return ; } } pieces . quick_push ( p ) ; } </s>

Add P to the list of pieces in the type .


<s> pure_scalable_type_info :: analysis_result pure_scalable_type_info :: analyze ( const_tree type ) { gcc_assert ( pieces . is_empty ( ) ) ; if ( type == error_mark_node ) return NO_ABI_IDENTITY ; if ( TYPE_SIZE ( type ) && integer_zerop ( TYPE_SIZE ( type ) ) ) return NO_ABI_IDENTITY ; piece p = { } ; if ( aarch64_sve :: builtin_type_p ( type , & p . num_zr , & p . num_pr ) ) { machine_mode mode = TYPE_MODE_RAW ( type ) ; gcc_assert ( VECTOR_MODE_P ( mode ) && ( ! TARGET_SVE || aarch64_sve_mode_p ( mode ) ) ) ; p . mode = p . orig_mode = mode ; add_piece ( p ) ; return IS_PST ; } if ( TREE_CODE ( type ) == ARRAY_TYPE ) return analyze_array ( type ) ; if ( TREE_CODE ( type ) == RECORD_TYPE ) return analyze_record ( type ) ; return ISNT_PST ; } </s>

Count how many times each regno is referenced as base address for a memory access .


<s> pure_scalable_type_info :: analysis_result pure_scalable_type_info :: analyze_array ( const_tree type ) { pure_scalable_type_info element_info ; analysis_result result = element_info . analyze ( TREE_TYPE ( type ) ) ; if ( result != IS_PST ) return result ; tree nelts_minus_one = array_type_nelts ( type ) ; if ( ! tree_fits_uhwi_p ( nelts_minus_one ) ) return DOESNT_MATTER ; unsigned HOST_WIDE_INT count = tree_to_uhwi ( nelts_minus_one ) ; if ( count > MAX_PIECES ) return DOESNT_MATTER ; count += 1 ; if ( count * element_info . pieces . length ( ) > MAX_PIECES ) return DOESNT_MATTER ; poly_uint64 element_bytes ; if ( ! poly_int_tree_p ( TYPE_SIZE_UNIT ( TREE_TYPE ( type ) ) , & element_bytes ) ) gcc_unreachable ( ) ; gcc_assert ( ! element_info . pieces . is_empty ( ) ) ; for ( unsigned int i = 0 ; i < count ; ++ i ) for ( unsigned int j = 0 ; j < element_info . pieces . length ( ) ; ++ j ) { piece p = element_info . pieces [ j ] ; p . offset += i * element_bytes ; add_piece ( p ) ; } return IS_PST ; } </s>

Subroutine of analyze for handling ARRAY_TYPEs .


<s> pure_scalable_type_info :: analysis_result pure_scalable_type_info :: analyze_record ( const_tree type ) { for ( tree field = TYPE_FIELDS ( type ) ; field ; field = TREE_CHAIN ( field ) ) { if ( TREE_CODE ( field ) != FIELD_DECL ) continue ; if ( DECL_SIZE ( field ) && integer_zerop ( DECL_SIZE ( field ) ) ) continue ; pure_scalable_type_info field_info ; analysis_result subresult = field_info . analyze ( TREE_TYPE ( field ) ) ; if ( subresult == NO_ABI_IDENTITY ) continue ; if ( subresult != IS_PST ) return subresult ; tree bitpos = bit_position ( field ) ; gcc_assert ( poly_int_tree_p ( bitpos ) ) ; poly_widest_int wide_bytepos = exact_div ( wi :: to_poly_widest ( bitpos ) , BITS_PER_UNIT ) ; poly_uint64 bytepos ; if ( ! wide_bytepos . to_uhwi ( & bytepos ) || pieces . length ( ) + field_info . pieces . length ( ) > MAX_PIECES ) return DOESNT_MATTER ; gcc_assert ( ! field_info . pieces . is_empty ( ) ) ; for ( unsigned int i = 0 ; i < field_info . pieces . length ( ) ; ++ i ) { piece p = field_info . pieces [ i ] ; p . offset += bytepos ; add_piece ( p ) ; } } return pieces . is_empty ( ) ? NO_ABI_IDENTITY : IS_PST ; } </s>

Subroutine of analyze for handling RECORD_TYPEs .


<s> bool pure_scalable_type_info :: analyze_registers ( const_tree type ) { analysis_result result = analyze ( type ) ; gcc_assert ( result != DOESNT_MATTER ) ; return result == IS_PST ; } </s>

Analyze a type that is known not to be passed or returned in memory . Return true if it has an ABI identity and is a Pure Scalable Type .


<s> static bool currently_supported_simd_type ( tree t , tree b ) { if ( COMPLEX_FLOAT_TYPE_P ( t ) ) return false ; if ( TYPE_SIZE ( t ) != TYPE_SIZE ( b ) ) return false ; return supported_simd_type ( t ) ; } </s>

Return true for types that currently are supported as SIMD return or argument types .


<s> static bool dep_between_memop_and_curr ( rtx memop ) { rtx load_reg ; int opno ; gcc_assert ( GET_CODE ( memop ) == SET ) ; if ( ! REG_P ( SET_DEST ( memop ) ) ) return false ; load_reg = SET_DEST ( memop ) ; for ( opno = 1 ; opno < recog_data . n_operands ; opno ++ ) { rtx operand = recog_data . operand [ opno ] ; if ( REG_P ( operand ) && reg_overlap_mentioned_p ( load_reg , operand ) ) return true ; } return false ; } </s>

Check if there is a register dependency between a load and the insn for which we hold recog_data .


<s> static void emit_insr ( rtx target , rtx elem ) { machine_mode mode = GET_MODE ( target ) ; scalar_mode elem_mode = GET_MODE_INNER ( mode ) ; elem = force_reg ( elem_mode , elem ) ; insn_code icode = optab_handler ( vec_shl_insert_optab , mode ) ; gcc_assert ( icode != CODE_FOR_nothing ) ; emit_insn ( GEN_FCN ( icode ) ( target , target , elem ) ) ; } </s>

Emit RTL corresponding to : insr TARGET , ELEM .


<s> inline static rtx_insn * emit_set_insn ( rtx x , rtx y ) { return emit_insn ( gen_rtx_SET ( x , y ) ) ; } </s>

Emit an insn that 's a simple single-set . Both the operands must be known to be valid .


<s> static void emit_unspec2 ( rtx target , int code , rtx op0 , rtx op1 ) { emit_insn ( gen_rtx_SET ( target , gen_rtx_UNSPEC ( GET_MODE ( target ) , gen_rtvec ( 2 , op0 , op1 ) , code ) ) ) ; } </s>

Generate ( set TARGET ( unspec [ OP0 OP1 ] CODE ) ) .


<s> bool extract_base_offset_in_addr ( rtx mem , rtx * base , rtx * offset ) { rtx addr ; gcc_assert ( MEM_P ( mem ) ) ; addr = XEXP ( mem , 0 ) ; if ( REG_P ( addr ) ) { * base = addr ; * offset = const0_rtx ; return true ; } if ( GET_CODE ( addr ) == PLUS && REG_P ( XEXP ( addr , 0 ) ) && CONST_INT_P ( XEXP ( addr , 1 ) ) ) { * base = XEXP ( addr , 0 ) ; * offset = XEXP ( addr , 1 ) ; return true ; } * base = NULL_RTX ; * offset = NULL_RTX ; return false ; } </s>

If MEM is in the form of `` base+offset '' , extract the two parts of address and set to BASE and OFFSET , otherwise return false after clearing BASE and OFFSET .


<s> static enum sched_fusion_type fusion_load_store ( rtx_insn * insn , rtx * base , rtx * offset ) { rtx x , dest , src ; enum sched_fusion_type fusion = SCHED_FUSION_LD ; gcc_assert ( INSN_P ( insn ) ) ; x = PATTERN ( insn ) ; if ( GET_CODE ( x ) != SET ) return SCHED_FUSION_NONE ; src = SET_SRC ( x ) ; dest = SET_DEST ( x ) ; machine_mode dest_mode = GET_MODE ( dest ) ; if ( ! aarch64_mode_valid_for_sched_fusion_p ( dest_mode ) ) return SCHED_FUSION_NONE ; if ( GET_CODE ( src ) == SIGN_EXTEND ) { fusion = SCHED_FUSION_LD_SIGN_EXTEND ; src = XEXP ( src , 0 ) ; if ( GET_CODE ( src ) != MEM || GET_MODE ( src ) != SImode ) return SCHED_FUSION_NONE ; } else if ( GET_CODE ( src ) == ZERO_EXTEND ) { fusion = SCHED_FUSION_LD_ZERO_EXTEND ; src = XEXP ( src , 0 ) ; if ( GET_CODE ( src ) != MEM || GET_MODE ( src ) != SImode ) return SCHED_FUSION_NONE ; } if ( GET_CODE ( src ) == MEM && REG_P ( dest ) ) extract_base_offset_in_addr ( src , base , offset ) ; else if ( GET_CODE ( dest ) == MEM && ( REG_P ( src ) || src == const0_rtx ) ) { fusion = SCHED_FUSION_ST ; extract_base_offset_in_addr ( dest , base , offset ) ; } else return SCHED_FUSION_NONE ; if ( * base == NULL_RTX || * offset == NULL_RTX ) fusion = SCHED_FUSION_NONE ; return fusion ; } </s>

If INSN is a load or store of address in the form of [ base+offset ] , extract the two parts and set to BASE and OFFSET . IS_LOAD is set to TRUE if it 's a load . Return TRUE if INSN is such an instruction , otherwise return FALSE .


<s> rtx pure_scalable_type_info :: get_rtx ( machine_mode mode , unsigned int first_zr , unsigned int first_pr ) const { if ( mode == pieces [ 0 ] . mode ) { gcc_assert ( pieces . length ( ) == 1 ) ; return pieces [ 0 ] . get_rtx ( first_zr , first_pr ) ; } rtvec rtxes = rtvec_alloc ( pieces . length ( ) ) ; for ( unsigned int i = 0 ; i < pieces . length ( ) ; ++ i ) { rtx reg = pieces [ i ] . get_rtx ( first_zr , first_pr ) ; rtx offset = gen_int_mode ( pieces [ i ] . offset , Pmode ) ; RTVEC_ELT ( rtxes , i ) = gen_rtx_EXPR_LIST ( VOIDmode , reg , offset ) ; first_zr += pieces [ i ] . num_zr ; first_pr += pieces [ i ] . num_pr ; } return gen_rtx_PARALLEL ( mode , rtxes ) ; } </s>

Return the location of a PST that is known to be passed or returned in registers . FIRST_ZR is the first unused vector argument register and FIRST_PR is the first unused predicate argument register .


"the %qE attribute cannot be applied to an SVE function type"

Check whether an 'aarch64_vector_pcs ' attribute is valid .


"code model %qs not supported in ilp32 mode"

A checking mechanism for the implementation of the various code models .


<s> static void initialize_aarch64_tls_size ( struct gcc_options * opts ) { if ( aarch64_tls_size == 0 ) aarch64_tls_size = 24 ; switch ( opts -> x_aarch64_cmodel_var ) { case AARCH64_CMODEL_TINY : if ( aarch64_tls_size > 24 ) aarch64_tls_size = 24 ; break ; case AARCH64_CMODEL_SMALL : if ( aarch64_tls_size > 32 ) aarch64_tls_size = 32 ; break ; case AARCH64_CMODEL_LARGE : if ( aarch64_tls_size > 48 ) aarch64_tls_size = 48 ; break ; default : gcc_unreachable ( ) ; } return ; } </s>

A checking mechanism for the implementation of the tls size .


<s> static unsigned int num_occurences_in_str ( char c , char * str ) { unsigned int res = 0 ; while ( * str != '\0' ) { if ( * str == c ) res ++ ; str ++ ; } return res ; } </s>

Count how many times the character C appears in NULL-terminated string STR .


<s> unsigned int pure_scalable_type_info :: num_pr ( ) const { unsigned int res = 0 ; for ( unsigned int i = 0 ; i < pieces . length ( ) ; ++ i ) res += pieces [ i ] . num_pr ; return res ; } </s>

Return the total number of predicate registers required by the PST .


<s> unsigned int pure_scalable_type_info :: num_zr ( ) const { unsigned int res = 0 ; for ( unsigned int i = 0 ; i < pieces . length ( ) ; ++ i ) res += pieces [ i ] . num_zr ; return res ; } </s>

Return the total number of vector registers required by the PST .


<s> static inline bool offset_12bit_unsigned_scaled_p ( machine_mode mode , poly_int64 offset ) { HOST_WIDE_INT multiple ; return ( constant_multiple_p ( offset , GET_MODE_SIZE ( mode ) , & multiple ) && IN_RANGE ( multiple , 0 , 4095 ) ) ; } </s>

Return true if OFFSET is an unsigned 12-bit value multiplied by the size of MODE .


<s> static inline bool offset_4bit_signed_scaled_p ( machine_mode mode , poly_int64 offset ) { HOST_WIDE_INT multiple ; return ( constant_multiple_p ( offset , GET_MODE_SIZE ( mode ) , & multiple ) && IN_RANGE ( multiple , - 8 , 7 ) ) ; } </s>

Return true if OFFSET is a signed 4-bit value multiplied by the size of MODE .


<s> static inline bool offset_6bit_unsigned_scaled_p ( machine_mode mode , poly_int64 offset ) { HOST_WIDE_INT multiple ; return ( constant_multiple_p ( offset , GET_MODE_SIZE ( mode ) , & multiple ) && IN_RANGE ( multiple , 0 , 63 ) ) ; } </s>

Return true if OFFSET is an unsigned 6-bit value multiplied by the size of MODE .


<s> static inline bool offset_9bit_signed_scaled_p ( machine_mode mode , poly_int64 offset ) { HOST_WIDE_INT multiple ; return ( constant_multiple_p ( offset , GET_MODE_SIZE ( mode ) , & multiple ) && IN_RANGE ( multiple , - 256 , 255 ) ) ; } </s>

Return true if OFFSET is a signed 9-bit value multiplied by the size of MODE .


<s> inline bool pr_or_ffr_regnum_p ( unsigned int regno ) { return PR_REGNUM_P ( regno ) || regno == FFR_REGNUM || regno == FFRT_REGNUM ; } </s>

Return true if REGNO is P0-P15 or one of the special FFR-related registers .


<s> static char sizetochar ( int size ) { switch ( size ) { case 64 : return 'd' ; case 32 : return 's' ; case 16 : return 'h' ; case 8 : return 'b' ; default : gcc_unreachable ( ) ; } } </s>


<s> static bool supported_simd_type ( tree t ) { if ( SCALAR_FLOAT_TYPE_P ( t ) || INTEGRAL_TYPE_P ( t ) || POINTER_TYPE_P ( t ) ) { HOST_WIDE_INT s = tree_to_shwi ( TYPE_SIZE_UNIT ( t ) ) ; return s == 1 || s == 2 || s == 4 || s == 8 ; } return false ; } </s>

Return true for types that could be supported as SIMD return or argument types .


<s> static const char * svpattern_token ( enum aarch64_svpattern pattern ) { switch ( pattern ) { AARCH64_FOR_SVPATTERN ( CASE ) case AARCH64_NUM_SVPATTERNS : break ; } gcc_unreachable ( ) ; } </s>

Return the assembly token for svpattern value VALUE .


<s> static const char * svprfop_token ( enum aarch64_svprfop prfop ) { switch ( prfop ) { AARCH64_FOR_SVPRFOP ( CASE ) case AARCH64_NUM_SVPRFOPS : break ; } gcc_unreachable ( ) ; } </s>

Return the assembly token for svprfop value PRFOP .


<s> static enum tls_model tls_symbolic_operand_type ( rtx addr ) { enum tls_model tls_kind = TLS_MODEL_NONE ; if ( GET_CODE ( addr ) == CONST ) { poly_int64 addend ; rtx sym = strip_offset ( addr , & addend ) ; if ( GET_CODE ( sym ) == SYMBOL_REF ) tls_kind = SYMBOL_REF_TLS_MODEL ( sym ) ; } else if ( GET_CODE ( addr ) == SYMBOL_REF ) tls_kind = SYMBOL_REF_TLS_MODEL ( addr ) ; return tls_kind ; } </s>

Return the TLS model to use for SYMBOL .


<s> static bool use_rsqrt_p ( machine_mode mode ) { return ( ! flag_trapping_math && flag_unsafe_math_optimizations && ( ( aarch64_tune_params . approx_modes -> recip_sqrt & AARCH64_APPROX_MODE ( mode ) ) || flag_mrecip_low_precision_sqrt ) ) ; } </s>

Return true if it is safe to use the rsqrt optabs to optimize 1.0/sqrt .


<s> static bool virt_or_elim_regno_p ( unsigned regno ) { return ( ( regno >= FIRST_VIRTUAL_REGISTER && regno <= LAST_VIRTUAL_POINTER_REGISTER ) || regno == FRAME_POINTER_REGNUM || regno == ARG_POINTER_REGNUM ) ; } </s>

Return true if REGNO is a virtual pointer register , or an eliminable `` soft '' frame register . Like REGNO_PTR_FRAME_P except that we do n't include stack_pointer or hard_frame_pointer .


<s> void aarch64_register_fma_steering ( ) { opt_pass * pass_fma_steering = make_pass_fma_steering ( g ) ; struct register_pass_info fma_steering_info = { pass_fma_steering , "rnreg" , 1 , PASS_POS_INSERT_AFTER } ; register_pass ( & fma_steering_info ) ; } </s>

Register the FMA steering pass to the pass manager .


<s> void func_fma_steering :: analyze ( ) { int i , n_blocks , * bb_dfs_preorder ; basic_block bb ; rtx_insn * insn ; bb_dfs_preorder = XNEWVEC ( int , last_basic_block_for_fn ( cfun ) ) ; n_blocks = pre_and_rev_post_order_compute ( bb_dfs_preorder , NULL , false ) ; for ( i = 0 ; i < n_blocks ; i ++ ) { bb = BASIC_BLOCK_FOR_FN ( cfun , bb_dfs_preorder [ i ] ) ; FOR_BB_INSNS ( bb , insn ) { operand_rr_info * dest_op_info ; struct du_chain * chain ; unsigned dest_regno ; fma_forest * forest ; du_head_p head ; int i ; if ( ! is_fmul_fmac_insn ( insn , true ) ) continue ; dest_op_info = insn_rr [ INSN_UID ( insn ) ] . op_info ; dest_regno = REGNO ( SET_DEST ( PATTERN ( insn ) ) ) ; for ( i = 0 ; i < dest_op_info -> n_chains ; i ++ ) { if ( dest_op_info -> heads [ i ] -> regno != dest_regno ) continue ; head = dest_op_info -> heads [ i ] ; if ( ! head -> first ) head = regrename_chain_from_id ( head -> id ) ; forest = NULL ; for ( chain = head -> first ; chain ; chain = chain -> next_use ) { fma_node * * fma_slot ; if ( ! is_fmul_fmac_insn ( chain -> insn , true ) ) continue ; if ( chain -> loc != & SET_DEST ( PATTERN ( chain -> insn ) ) ) continue ; if ( chain -> insn == insn ) break ; fma_slot = this -> m_insn_fma_head_map -> get ( chain -> insn ) ; if ( fma_slot && ( * fma_slot ) -> get_children ( ) ) forest = ( * fma_slot ) -> get_forest ( ) ; } if ( chain ) break ; } gcc_assert ( i < dest_op_info -> n_chains ) ; this -> analyze_fma_fmul_insn ( forest , chain , head ) ; } } free ( bb_dfs_preorder ) ; if ( dump_file ) this -> dfs ( dump_forest_info , dump_tree_root_info , dump_tree_node_info , false ) ; } </s>

Count how many times each regno is referenced as base address for a memory access .


<s> void func_fma_steering :: dfs ( void ( * process_forest ) ( fma_forest * ) , void ( * process_root ) ( fma_forest * , fma_root_node * ) , void ( * process_node ) ( fma_forest * , fma_node * ) , bool free ) { vec < fma_node * > to_process ; std :: list < fma_forest * > :: iterator forest_iter ; to_process . create ( 0 ) ; for ( forest_iter = this -> m_fma_forests . begin ( ) ; forest_iter != this -> m_fma_forests . end ( ) ; forest_iter ++ ) { std :: list < fma_root_node * > :: iterator root_iter ; if ( process_forest ) process_forest ( * forest_iter ) ; for ( root_iter = ( * forest_iter ) -> get_roots ( ) -> begin ( ) ; root_iter != ( * forest_iter ) -> get_roots ( ) -> end ( ) ; root_iter ++ ) { if ( process_root ) process_root ( * forest_iter , * root_iter ) ; to_process . safe_push ( * root_iter ) ; } while ( ! to_process . is_empty ( ) ) { fma_node * node ; std :: list < fma_node * > :: iterator child_iter ; node = to_process . pop ( ) ; if ( process_node ) process_node ( * forest_iter , node ) ; if ( node -> get_children ( ) -> empty ( ) ) continue ; for ( child_iter = node -> get_children ( ) -> begin ( ) ; child_iter != node -> get_children ( ) -> end ( ) ; child_iter ++ ) to_process . safe_push ( * child_iter ) ; if ( free ) { if ( node -> root_p ( ) ) delete static_cast < fma_root_node * > ( node ) ; else delete node ; } } if ( free ) delete * forest_iter ; } to_process . release ( ) ; } </s>

Perform a depth-first search of the forests of fma_node in THIS- > m_fma_forests , calling PROCESS_FOREST ( ) on each fma_forest object in THIS- > m_fma_forests list , PROCESS_ROOT ( ) on each tree root and PROCESS_NODE ( ) on each node . If FREE is true , free all std : :list in the same dfs .


<s> static rtl_opt_pass * make_pass_fma_steering ( gcc :: context * ctxt ) { return new pass_fma_steering ( ctxt ) ; } </s>

Create a new fma steering pass instance .


<s> void fma_forest :: merge_forest ( fma_forest * other_forest ) { std :: list < fma_root_node * > * other_roots ; std :: list < fma_root_node * > :: iterator other_root_iter ; if ( this == other_forest ) return ; other_roots = other_forest -> m_roots ; for ( other_root_iter = other_roots -> begin ( ) ; other_root_iter != other_roots -> end ( ) ; other_root_iter ++ ) ( * other_root_iter ) -> set_forest ( this ) ; this -> m_globals -> remove_forest ( other_forest ) ; this -> m_roots -> splice ( this -> m_roots -> begin ( ) , * other_roots ) ; delete other_forest ; this -> m_nb_nodes += other_forest -> m_nb_nodes ; } </s>

Merge REF_FOREST and OTHER_FOREST together , making REF_FOREST the canonical fma_forest object to represent both .


"FMA or FMUL at insn %d but destination "

"register (%s) has different parity from expected to "

"Destination register of insn %d could not be "

"renamed. Dependent FMA insns will use this parity from "

Rename the destination register of a single FMUL or FMADD/FMSUB instruction represented by FMA_NODE to a register that respect the target parity for FOREST or with same parity of the instruction represented by its parent node if it has one .


<s>
static
bool
rename_single_chain
(
du_head_p
head
,
HARD_REG_SET
*
unavailable
)
{
int
best_new_reg
;
int
n_uses
=
0
;
struct
du_chain
*
tmp
;
int
reg
=
head
->
regno
;
enum
reg_class
super_class
=
NO_REGS
;
if
(
head
->
cannot_rename
)
return
false
;
if
(
fixed_regs
[
reg
]
||
global_regs
[
reg
]
||
(
frame_pointer_needed
&&
reg
==
HARD_FRAME_POINTER_REGNUM
)
)
return
false
;
for
(
tmp
=
head
->
first
;
tmp
;
tmp
=
tmp
->
next_use
)
{
if
(
DEBUG_INSN_P
(
tmp
->
insn
)
)
continue
;
n_uses
++
;
IOR_COMPL_HARD_REG_SET
(
*
unavailable
,
reg_class_contents
[
tmp
->
cl
]
)
;
super_class
=
reg_class_superunion
[
(
int
)
super_class
]
[
(
int
)
tmp
->
cl
]
;
}
if
(
n_uses
<
1
)
return
false
;
best_new_reg
=
find_rename_reg
(
head
,
super_class
,
unavailable
,
reg
,
false
)
;
if
(
dump_file
)
{
fprintf
(
dump_file
,
"Register %s in insn %d"
,
reg_names
[
reg
]
,
INSN_UID
(
head
->
first
->
insn
)
)
;
if
(
head
->
need_caller_save_reg
)
fprintf
(
dump_file
,
" crosses a call"
)
;
}
if
(
best_new_reg
==
reg
)
{
if
(
dump_file
)
fprintf
(
dump_file
,
"; no available better choice\n"
)
;
return
false
;
}
if
(
regrename_do_replace
(
head
,
best_new_reg
)
)
{
if
(
dump_file
)
fprintf
(
dump_file
,
", renamed as %s\n"
,
reg_names
[
best_new_reg
]
)
;
df_set_regs_ever_live
(
best_new_reg
,
true
)
;
}
else
{
if
(
dump_file
)
fprintf
(
dump_file
,
", renaming as %s failed\n"
,
reg_names
[
best_new_reg
]
)
;
return
false
;
}
return
true
;
}
</s>

Rename the register HEAD- > regno in all the insns in the chain HEAD to any register not in the set UNAVAILABLE . Adapted from rename_chains in regrename.c .


<s> void func_fma_steering :: analyze ( ) { int i , n_blocks , * bb_dfs_preorder ; basic_block bb ; rtx_insn * insn ; bb_dfs_preorder = XNEWVEC ( int , last_basic_block_for_fn ( cfun ) ) ; n_blocks = pre_and_rev_post_order_compute ( bb_dfs_preorder , NULL , false ) ; for ( i = 0 ; i < n_blocks ; i ++ ) { bb = BASIC_BLOCK_FOR_FN ( cfun , bb_dfs_preorder [ i ] ) ; FOR_BB_INSNS ( bb , insn ) { operand_rr_info * dest_op_info ; struct du_chain * chain = NULL ; unsigned dest_regno ; fma_forest * forest = NULL ; du_head_p head = NULL ; int i ; if ( ! is_fmul_fmac_insn ( insn , true ) ) continue ; dest_op_info = insn_rr [ INSN_UID ( insn ) ] . op_info ; dest_regno = REGNO ( SET_DEST ( PATTERN ( insn ) ) ) ; for ( i = 0 ; i < dest_op_info -> n_chains ; i ++ ) { if ( dest_op_info -> heads [ i ] -> regno != dest_regno ) continue ; head = dest_op_info -> heads [ i ] ; if ( ! head -> first ) head = regrename_chain_from_id ( head -> id ) ; forest = NULL ; for ( chain = head -> first ; chain ; chain = chain -> next_use ) { fma_node * * fma_slot ; if ( ! is_fmul_fmac_insn ( chain -> insn , true ) ) continue ; if ( chain -> loc != & SET_DEST ( PATTERN ( chain -> insn ) ) ) continue ; if ( chain -> insn == insn ) break ; fma_slot = this -> m_insn_fma_head_map -> get ( chain -> insn ) ; if ( fma_slot && ( * fma_slot ) -> get_children ( ) ) forest = ( * fma_slot ) -> get_forest ( ) ; } if ( chain ) break ; } gcc_assert ( i < dest_op_info -> n_chains ) ; this -> analyze_fma_fmul_insn ( forest , chain , head ) ; } } free ( bb_dfs_preorder ) ; if ( dump_file ) this -> dfs ( dump_forest_info , dump_tree_root_info , dump_tree_node_info , false ) ; } </s>


<s> void fma_node :: add_child ( fma_node * child ) { this -> m_children -> push_back ( child ) ; } </s>

Add a child to this node in the list of children .


<s> void func_fma_steering :: analyze ( ) { int i , n_blocks , * bb_dfs_preorder ; basic_block bb ; rtx_insn * insn ; bb_dfs_preorder = XNEWVEC ( int , last_basic_block_for_fn ( cfun ) ) ; n_blocks = pre_and_rev_post_order_compute ( bb_dfs_preorder , NULL , false ) ; for ( i = 0 ; i < n_blocks ; i ++ ) { bb = BASIC_BLOCK_FOR_FN ( cfun , bb_dfs_preorder [ i ] ) ; FOR_BB_INSNS ( bb , insn ) { operand_rr_info * dest_op_info ; struct du_chain * chain = NULL ; unsigned dest_regno ; fma_forest * forest = NULL ; du_head_p head = NULL ; int i ; if ( ! is_fmul_fmac_insn ( insn , true ) ) continue ; dest_op_info = insn_rr [ INSN_UID ( insn ) ] . op_info ; dest_regno = REGNO ( SET_DEST ( PATTERN ( insn ) ) ) ; for ( i = 0 ; i < dest_op_info -> n_chains ; i ++ ) { if ( dest_op_info -> heads [ i ] -> regno != dest_regno ) continue ; head = dest_op_info -> heads [ i ] ; if ( ! head -> first ) head = regrename_chain_from_id ( head -> id ) ; forest = NULL ; for ( chain = head -> first ; chain ; chain = chain -> next_use ) { fma_node * * fma_slot ; if ( ! is_fmul_fmac_insn ( chain -> insn , true ) ) continue ; if ( chain -> loc != & SET_DEST ( PATTERN ( chain -> insn ) ) ) continue ; if ( chain -> insn == insn ) break ; fma_slot = this -> m_insn_fma_head_map -> get ( chain -> insn ) ; if ( fma_slot && ( * fma_slot ) -> get_children ( ) ) forest = ( * fma_slot ) -> get_forest ( ) ; } if ( chain ) break ; } if ( i < dest_op_info -> n_chains ) this -> analyze_fma_fmul_insn ( forest , chain , head ) ; } } free ( bb_dfs_preorder ) ; if ( dump_file ) this -> dfs ( dump_forest_info , dump_tree_root_info , dump_tree_node_info , false ) ; } </s>


<s> void func_fma_steering :: analyze_fma_fmul_insn ( fma_forest * ref_forest , du_chain * chain , du_head_p head ) { fma_forest * forest ; fma_node * node = this -> get_fma_node ( chain -> insn ) ; if ( ! node ) { fma_root_node * root_node ; root_node = new fma_root_node ( this , chain , this -> m_next_forest_id ++ ) ; forest = root_node -> get_forest ( ) ; node = root_node ; this -> m_fma_forests . push_back ( forest ) ; } else forest = node -> get_forest ( ) ; node -> set_head ( head ) ; if ( ref_forest ) { ref_forest -> merge_forest ( forest ) ; return ; } for ( chain = head -> first ; chain ; chain = chain -> next_use ) { fma_node * child_fma ; rtx fma_rtx , * accum_rtx_p ; if ( ! is_fmul_fmac_insn ( chain -> insn , false ) ) continue ; fma_rtx = SET_SRC ( PATTERN ( chain -> insn ) ) ; if ( GET_CODE ( fma_rtx ) == NEG ) fma_rtx = XEXP ( fma_rtx , 0 ) ; accum_rtx_p = & XEXP ( fma_rtx , 2 ) ; if ( ! REG_P ( * accum_rtx_p ) ) accum_rtx_p = & XEXP ( * accum_rtx_p , 0 ) ; if ( accum_rtx_p != chain -> loc ) continue ; if ( this -> get_fma_node ( chain -> insn ) ) continue ; child_fma = new fma_node ( node , chain ) ; node -> add_child ( child_fma ) ; } } </s>

Allocate and initialize fma_node objects for the FMUL or FMADD/FMSUB instruction in CHAIN- > insn and its dependent FMADD/FMSUB instructions , all part of FOREST . For the children , the associated head is left untouched ( and thus null ) as this function will be called again when considering the chain where they are def . For the parent , the chain is given in HEAD .


<s> void func_fma_steering :: dfs ( void ( * process_forest ) ( fma_forest * ) , void ( * process_root ) ( fma_forest * , fma_root_node * ) , void ( * process_node ) ( fma_forest * , fma_node * ) , bool free ) { auto_vec < fma_node * > to_process ; auto_vec < fma_node * > to_free ; std :: list < fma_forest * > :: iterator forest_iter ; for ( forest_iter = this -> m_fma_forests . begin ( ) ; forest_iter != this -> m_fma_forests . end ( ) ; ++ forest_iter ) { std :: list < fma_root_node * > :: iterator root_iter ; if ( process_forest ) process_forest ( * forest_iter ) ; for ( root_iter = ( * forest_iter ) -> get_roots ( ) -> begin ( ) ; root_iter != ( * forest_iter ) -> get_roots ( ) -> end ( ) ; ++ root_iter ) { if ( process_root ) process_root ( * forest_iter , * root_iter ) ; to_process . safe_push ( * root_iter ) ; } while ( ! to_process . is_empty ( ) ) { fma_node * node ; std :: list < fma_node * > :: iterator child_iter ; node = to_process . pop ( ) ; if ( process_node ) process_node ( * forest_iter , node ) ; for ( child_iter = node -> get_children ( ) -> begin ( ) ; child_iter != node -> get_children ( ) -> end ( ) ; ++ child_iter ) to_process . safe_push ( * child_iter ) ; if ( free ) to_free . safe_push ( node ) ; } if ( free ) { delete * forest_iter ; while ( ! to_free . is_empty ( ) ) { fma_node * node = to_free . pop ( ) ; if ( node -> root_p ( ) ) delete static_cast < fma_root_node * > ( node ) ; else delete node ; } } } } </s>


<s>
void
fma_forest
::
dispatch
(
)
{
this
->
m_target_parity
=
this
->
m_roots
->
front
(
)
->
get_parity
(
)
;
int
fpu_balance
=
this
->
m_globals
->
get_fpu_balance
(
)
;
if
(
fpu_balance
!=
0
)
this
->
m_target_parity
=
(
fpu_balance
<
0
)
;
if
(
dump_file
)
fprintf
(
dump_file
,
"Target parity for forest #%d: %s\n"
,
this
->
m_id
,
this
->
m_target_parity
"odd"
:
"even"
)
;
}
</s>

Dispatch forest to the least utilized pipeline .


<s> static void dispatch_forest ( fma_forest * forest ) { forest -> dispatch ( ) ; } </s>

Wrapper around fma_forest : :dispatch for use as parameter of function pointer type in func_fma_steering : :dfs .


<s> static void dump_forest_info ( fma_forest * forest ) { forest -> dump_info ( ) ; } </s>

Wrapper around fma_forest : :dump_info for use as parameter of function pointer type in func_fma_steering : :dfs .


<s>
void
fma_root_node
::
dump_info
(
fma_forest
*
forest
)
{
gcc_assert
(
dump_file
)
;
if
(
this
==
forest
->
get_roots
(
)
->
front
(
)
)
fprintf
(
dump_file
,
"Instruction(s) at root of forest #%d:"
,
forest
->
get_id
(
)
)
;
fprintf
(
dump_file
,
" %d"
,
INSN_UID
(
this
->
m_insn
)
)
;
if
(
this
==
forest
->
get_roots
(
)
->
back
(
)
)
fprintf
(
dump_file
,
"\n"
)
;
}
</s>

Dump information about the roots of forest FOREST .

aarch64
"Instruction(s) at root of forest #%d:"
" %d"
"\n"


<s> static void dump_tree_node_info ( fma_forest * forest , fma_node * node ) { node -> dump_info ( forest ) ; } </s>

Wrapper around fma_node : :dump_info for use as parameter of function pointer type in func_fma_steering : :dfs .


<s> static void dump_tree_root_info ( fma_forest * forest , fma_root_node * node ) { node -> dump_info ( forest ) ; } </s>

Wrapper around fma_root_node : :dump_info for use as parameter of function pointer type in func_fma_steering : :dfs .


<s> virtual unsigned int execute ( function * ) { func_fma_steering * fma_steering = new func_fma_steering ; fma_steering -> execute_fma_steering ( ) ; delete fma_steering ; return 0 ; } </s>


<s> void func_fma_steering :: execute_fma_steering ( ) { df_set_flags ( DF_LR_RUN_DCE ) ; df_note_add_problem ( ) ; df_analyze ( ) ; df_set_flags ( DF_DEFER_INSN_RESCAN ) ; regrename_init ( true ) ; regrename_analyze ( NULL ) ; this -> analyze ( ) ; this -> rename_fma_trees ( ) ; regrename_finish ( ) ; } </s>


<s> fma_forest :: fma_forest ( func_fma_steering * fma_steer , fma_root_node * fma_root , int id ) { memset ( this , 0 , sizeof ( * this ) ) ; this -> m_globals = fma_steer ; this -> m_roots = new std :: list < fma_root_node * > ; this -> m_roots -> push_back ( fma_root ) ; this -> m_id = id ; } </s>


<s> void fma_forest :: fma_node_created ( fma_node * node ) { bool created = ! this -> m_globals -> put_node ( node ) ; gcc_assert ( created ) ; this -> m_nb_nodes ++ ; } </s>

Act on the creation of NODE by updating statistics in FOREST and adding an entry for it in the func_fma_steering hashmap .


<s> fma_node * func_fma_steering :: get_fma_node ( rtx_insn * insn ) { fma_node * * fma_slot ; fma_slot = this -> m_insn_fma_head_map -> get ( insn ) ; if ( fma_slot ) return * fma_slot ; return NULL ; } </s>

Return whether an fma_node object exists for instruction INSN and , if not , allocate one in * RET .


<s> fma_forest * fma_root_node :: get_forest ( ) { return this -> m_forest ; } </s>

Get the actual forest associated with a non root node as the one the node points to might have been merged into another one . In that case the pointer in the root nodes are updated so we return the forest pointer of a root node pointed to by the initial forest . Despite being a oneliner , this method is defined here as it references a method from fma_root_node .


<s> int fma_node :: get_parity ( ) { return this -> m_head -> regno % 2 ; } </s>

Return the parity of the destination register of the instruction represented by this node .


<s> static bool is_fmac_op ( enum attr_type t ) { return ( t == TYPE_FMACS ) || ( t == TYPE_FMACD ) || ( t == TYPE_NEON_FP_MLA_S ) ; } </s>

Return whether T is the attribute of a FMADD/FMSUB-like instruction .


<s> static bool is_fmul_fmac_insn ( rtx_insn * insn , bool fmul_ok ) { enum attr_type t ; if ( ! NONDEBUG_INSN_P ( insn ) ) return false ; if ( recog_memoized ( insn ) < 0 ) return false ; t = get_attr_type ( insn ) ; return is_fmac_op ( t ) || ( fmul_ok && is_fmul_op ( t ) ) ; } </s>

Return whether INSN is an FMUL ( if FMUL_OK is true ) or FMADD/FMSUB instruction .


<s> static bool is_fmul_op ( enum attr_type t ) { return ( t == TYPE_FMULS ) || ( t == TYPE_FMULD ) || ( t == TYPE_NEON_FP_MUL_S ) ; } </s>

Return whether T is the attribute of a FMUL instruction .


<s> rtl_opt_pass * make_pass_fma_steering ( gcc :: context * ctxt ) { return new pass_fma_steering ( ctxt ) ; } </s>

Create a new fma steering pass instance .


<s> void fma_forest :: merge_forest ( fma_forest * other_forest ) { std :: list < fma_root_node * > * other_roots ; std :: list < fma_root_node * > :: iterator other_root_iter ; if ( this == other_forest ) return ; other_roots = other_forest -> m_roots ; for ( other_root_iter = other_roots -> begin ( ) ; other_root_iter != other_roots -> end ( ) ; ++ other_root_iter ) ( * other_root_iter ) -> set_forest ( this ) ; this -> m_globals -> remove_forest ( other_forest ) ; this -> m_roots -> splice ( this -> m_roots -> begin ( ) , * other_roots ) ; this -> m_nb_nodes += other_forest -> m_nb_nodes ; delete other_forest ; } </s>

Merge REF_FOREST and OTHER_FOREST together , making REF_FOREST the canonical fma_forest object to represent both .


<s> bool func_fma_steering :: put_node ( fma_node * node ) { return this -> m_insn_fma_head_map -> put ( node -> get_insn ( ) , node ) ; } </s>

Memorize the mapping of this instruction to its fma_node object and return whether such a mapping existed .


"FMA or FMUL at insn %d but destination "

"register (%s) has different parity from expected to "

"Destination register of insn %d could not be "

"renamed. Dependent FMA insns will use this parity from "

Rename the destination register of a single FMUL or FMADD/FMSUB instruction represented by FMA_NODE to a register that respect the target parity for FOREST or with same parity of the instruction represented by its parent node if it has one .


<s> static void rename_fma_node ( fma_forest * forest , fma_node * node ) { node -> rename ( forest ) ; } </s>


"perfect balance of FMUL/FMA chains between the "

"%d more FMUL/FMA chains scheduled on the second "

"%d more FMUL/FMA chains scheduled on the first "

Perform the renaming of all chains with FMUL or FMADD/FMSUB involved with the objective of keeping FPU pipeline balanced in term of instructions and having FMADD/FMSUB with dependencies on previous FMUL or FMADD/FMSUB be scheduled on the same pipeline .


<s>
static
bool
rename_single_chain
(
du_head_p
head
,
HARD_REG_SET
*
unavailable
)
{
int
best_new_reg
;
int
n_uses
=
0
;
struct
du_chain
*
tmp
;
int
reg
=
head
->
regno
;
enum
reg_class
super_class
=
NO_REGS
;
if
(
head
->
cannot_rename
)
return
false
;
if
(
fixed_regs
[
reg
]
||
global_regs
[
reg
]
||
(
frame_pointer_needed
&&
reg
==
HARD_FRAME_POINTER_REGNUM
)
)
return
false
;
for
(
tmp
=
head
->
first
;
tmp
;
tmp
=
tmp
->
next_use
)
{
if
(
DEBUG_INSN_P
(
tmp
->
insn
)
)
continue
;
n_uses
++
;
*
unavailable
|=
~
reg_class_contents
[
tmp
->
cl
]
;
super_class
=
reg_class_superunion
[
(
int
)
super_class
]
[
(
int
)
tmp
->
cl
]
;
}
if
(
n_uses
<
1
)
return
false
;
best_new_reg
=
find_rename_reg
(
head
,
super_class
,
unavailable
,
reg
,
false
)
;
if
(
dump_file
)
{
fprintf
(
dump_file
,
"Register %s in insn %d"
,
reg_names
[
reg
]
,
INSN_UID
(
head
->
first
->
insn
)
)
;
if
(
head
->
call_abis
)
fprintf
(
dump_file
,
" crosses a call"
)
;
}
if
(
best_new_reg
==
reg
)
{
if
(
dump_file
)
fprintf
(
dump_file
,
"; no available better choice\n"
)
;
return
false
;
}
if
(
regrename_do_replace
(
head
,
best_new_reg
)
)
{
if
(
dump_file
)
fprintf
(
dump_file
,
", renamed as %s\n"
,
reg_names
[
best_new_reg
]
)
;
df_set_regs_ever_live
(
best_new_reg
,
true
)
;
}
else
{
if
(
dump_file
)
fprintf
(
dump_file
,
", renaming as %s failed\n"
,
reg_names
[
best_new_reg
]
)
;
return
false
;
}
return
true
;
}
</s>

Rename the register HEAD- > regno in all the insns in the chain HEAD to any register not in the set UNAVAILABLE . Adapted from rename_chains in regrename.c .


<s> bool fma_node :: root_p ( ) { return this -> m_root == this ; } </s>


<s> void func_fma_steering :: update_balance ( int parity ) { this -> m_fpu_balance = parity ? this -> m_fpu_balance + 1 : this -> m_fpu_balance - 1 ; } </s>

Update the current balance considering a node with the given PARITY .


<s> static size_t find_field ( const std :: string & field , const std :: string & key ) { size_t key_pos , sep_pos ; key_pos = field . find ( key ) ; if ( key_pos == std :: string :: npos ) return std :: string :: npos ; sep_pos = field . find ( ":" , key_pos + 1 ) ; if ( sep_pos == std :: string :: npos ) return std :: string :: npos ; return sep_pos ; } </s>

Returns the index of the ' : ' inside the FIELD which must be found after the value of KEY . Returns string : :npos if line does not contain a field .


<s> static unsigned parse_field ( const std :: string & field ) { const char * rest = strchr ( field . c_str ( ) , ':' ) ; if ( rest == NULL ) return - 1 ; char * after ; unsigned fint = strtol ( rest + 1 , & after , 16 ) ; if ( after == rest + 1 ) return - 1 ; return fint ; } </s>

Returns the hex integer that is after ' : ' for the FIELD . Returns -1 is returned if there was problem parsing the integer .


<s> static std :: string readline ( FILE * f ) { char * buf = NULL ; int size = 0 ; int last = 0 ; const int buf_size = 128 ; if ( feof ( f ) ) return std :: string ( ) ; do { size += buf_size ; buf = ( char * ) xrealloc ( buf , size ) ; gcc_assert ( buf ) ; if ( ! fgets ( buf + last , buf_size , f ) ) return std :: string ( ) ; last = strnlen ( buf , size ) - 1 ; } while ( ! feof ( f ) && buf [ last ] != '\n' ) ; std :: string result ( buf ) ; free ( buf ) ; return result ; } </s>

Read an entire line from F until '\n ' or EOF .


<s>
static
void
split_words
(
const
std
::
string
&
val
,
std
::
set
<
std
::
string
>
&
result
)
{
size_t
cur
,
prev
=
0
;
std
::
string
word
;
while
(
(
cur
=
val
.
find_first_of
(
" \n"
,
prev
)
)
!=
std
::
string
::
npos
)
{
word
=
val
.
substr
(
prev
,
cur
prev
)
;
if
(
!
word
.
empty
(
)
)
result
.
insert
(
word
)
;
prev
=
cur
+
1
;
}
if
(
prev
!=
cur
)
result
.
insert
(
val
.
substr
(
prev
)
)
;
}
</s>

Splits and returns a string based on whitespace and return it as part of a set . Empty strings are ignored .


<s> static bool contains_string_p ( const char * * arr , const char * str ) { bool res = false ; if ( arr [ 0 ] != NULL ) { res = strstr ( arr [ 0 ] , str ) != NULL ; if ( res ) return res ; if ( arr [ 1 ] != NULL ) return strstr ( arr [ 1 ] , str ) != NULL ; } return false ; } </s>

Return true iff ARR contains STR in one of its two elements .


<s> static bool valid_bL_string_p ( const char * * core , const char * bL_string ) { return strstr ( bL_string , core [ 0 ] ) != NULL && strstr ( bL_string , core [ 1 ] ) != NULL ; } </s>

Check wether the string CORE contains the same CPU part numbers as BL_STRING . For example CORE= '' { 0xd03 , 0xd07 } '' and BL_STRING= '' 0xd07.0xd03 '' should return true .


<s> static bool contains_core_p ( unsigned * arr , unsigned core ) { if ( arr [ 0 ] != INVALID_CORE ) { if ( arr [ 0 ] == core ) return true ; if ( arr [ 1 ] != INVALID_CORE ) return arr [ 1 ] == core ; } return false ; } </s>

Return true iff ARR contains CORE , in either of the two elements .


<s> static struct aarch64_arch_driver_info * get_arch_from_id ( const char * id ) { unsigned int i = 0 ; for ( i = 0 ; aarch64_arches [ i ] . id != NULL ; i ++ ) { if ( strcmp ( id , aarch64_arches [ i ] . id ) == 0 ) return & aarch64_arches [ i ] ; } return NULL ; } </s>

Return an aarch64_arch_driver_info for the architecture described by ID , or NULL if ID describes something we do n't know about .


<s> static unsigned parse_field ( const char * field ) { const char * rest = strchr ( field , ':' ) ; char * after ; unsigned fint = strtol ( rest + 1 , & after , 16 ) ; if ( after == rest + 1 ) return - 1 ; return fint ; } </s>


<s> static bool valid_bL_core_p ( unsigned int * core , unsigned int bL_core ) { return AARCH64_BIG_LITTLE ( core [ 0 ] , core [ 1 ] ) == bL_core || AARCH64_BIG_LITTLE ( core [ 1 ] , core [ 0 ] ) == bL_core ; } </s>

Check wether the CORE array is the same as the big.LITTLE BL_CORE . For an example CORE= { 0xd08 , 0xd03 } and BL_CORE=AARCH64_BIG_LITTLE ( 0xd08 , 0xd03 ) will return true .


<s> static enum reg_class init_unavailable ( tag_insn_info * insn_info , tag_map_t & tag_map , du_head_p head , HARD_REG_SET * unavailable ) { unsigned dest = head -> regno ; enum reg_class super_class = NO_REGS ; machine_mode mode = GET_MODE ( insn_info -> dest ) ; CLEAR_HARD_REG_SET ( * unavailable ) ; for ( struct du_chain * tmp = head -> first ; tmp ; tmp = tmp -> next_use ) { if ( DEBUG_INSN_P ( tmp -> insn ) ) continue ; IOR_COMPL_HARD_REG_SET ( * unavailable , reg_class_contents [ tmp -> cl ] ) ; super_class = reg_class_superunion [ ( int ) super_class ] [ ( int ) tmp -> cl ] ; } for ( unsigned i = 0 ; i < FIRST_PSEUDO_REGISTER ; i ++ ) if ( fixed_regs [ i ] || global_regs [ i ] ) add_to_hard_reg_set ( unavailable , mode , i ) ; arg_pair_t arg = arg_pair_t ( insn_info , unavailable ) ; tag_map . traverse < arg_pair_t * , check_colliding_tags > ( & arg ) ; ignore_all_aliases ( unavailable , mode , dest & 0xf ) ; return super_class ; } </s>

Initialize and build a set of hard register numbers UNAVAILABLE to avoid for renaming . INSN_INFO is the original insn , TAG_MAP is the map of the list of insns indexed by their tags , HEAD is the def/use chain head of the destination register of the original insn . The routine returns the super class of register classes that may be used during the renaming .


<s> static bool valid_src_p ( rtx src , rtx_insn * insn , struct loop * loop , bool * pre_post , rtx * base , rtx * offset , bool load_pair ) { subrtx_var_iterator :: array_type array ; rtx x = NULL_RTX ; FOR_EACH_SUBRTX_VAR ( iter , array , src , NONCONST ) if ( MEM_P ( * iter ) ) { x = * iter ; break ; } if ( ! x ) return false ; struct aarch64_address_info addr ; machine_mode mode = GET_MODE ( x ) ; if ( ! aarch64_classify_address ( & addr , XEXP ( x , 0 ) , mode , true ) ) return false ; unsigned regno = REGNO ( addr . base ) ; if ( global_regs [ regno ] || fixed_regs [ regno ] ) return false ; if ( addr . type == ADDRESS_REG_WB ) { unsigned code = GET_CODE ( XEXP ( x , 0 ) ) ; * pre_post = true ; * base = addr . base ; if ( code == PRE_MODIFY || code == POST_MODIFY ) * offset = addr . offset ; else { unsigned int_offset = GET_MODE_SIZE ( mode ) . to_constant ( ) ; if ( load_pair && ( code == POST_INC || code == POST_DEC ) ) int_offset *= 2 ; * offset = GEN_INT ( int_offset ) ; } return true ; } else if ( addr . type == ADDRESS_REG_IMM || addr . type == ADDRESS_REG_REG ) { if ( ! iv_p ( insn , addr . base , loop ) ) return false ; * base = addr . base ; * offset = addr . offset ; return true ; } return false ; } </s>

Return true if SRC is a strided load in the LOOP , false otherwise . If it is a strided load , set the BASE and OFFSET . Also , if this is a pre/post increment load , set PRE_POST to true .


<s> static void avoid_collisions ( tag_map_t & tag_map ) { do { map_changed = false ; tag_map . traverse < tag_map_t * , avoid_collisions_1 > ( & tag_map ) ; } while ( map_changed ) ; } </s>

For each set of collisions , attempt to rename the registers or insert a move to avoid the collision . We repeatedly traverse through TAG_MAP using AVOID_COLLISIONS_1 trying to rename registers to avoid collisions until a full traversal results in no change in the map .


"\t>> Failed to rename destination in insn %d\n\t>>"

The actual reallocation logic . For each vector of collisions V , try to resolve the collision by attempting to rename the destination register of all but one of the loads . This is a callback that is invoked for each name-value pair ( T , V ) in TAG_MAP . The function returns true whenever it returns unchanged and false otherwise to halt traversal .


<s> bool check_colliding_tags ( const rtx & t , const insn_info_list_t & v , arg_pair_t * arg ) { HARD_REG_SET * unavailable = arg -> second ; unsigned orig_tag = arg -> first -> tag ( ) ; unsigned tag = INTVAL ( t ) ; machine_mode mode = GET_MODE ( arg -> first -> dest ) ; if ( v . length ( ) == 0 ) return true ; if ( TAG_UPDATE_DEST ( tag , 0 ) == TAG_UPDATE_DEST ( orig_tag , 0 ) ) ignore_all_aliases ( unavailable , mode , TAG_GET_DEST ( tag ) ) ; return true ; } </s>

Callback to check which destination registers are unavailable to us for renaming because of the base and offset colliding . This is a callback that gets called for every name value pair ( T , V ) in the TAG_MAP . The ARG is an std : :pair of the tag_insn_info of the original insn and the hard register set UNAVAILABLE that is used to record hard register numbers that can not be used for the renaming . This always returns true since we want to traverse through the entire TAG_MAP .


<s>
bool
dump_insn_list
(
const
rtx
&
t
,
const
insn_info_list_t
&
insn_info
,
void
*
unused
ATTRIBUTE_UNUSED
)
{
gcc_assert
(
dump_file
)
;
fprintf
(
dump_file
,
"Tag 0x%lx ::\n"
,
INTVAL
(
t
)
)
;
for
(
unsigned
i
=
0
;
i
<
insn_info
.
length
(
)
;
i
++
)
dump_insn_slim
(
dump_file
,
insn_info
[
i
]
->
insn
)
;
fprintf
(
dump_file
,
"\n"
)
;
return
true
;
}
</s>

Callback invoked for each name-value pair ( T , INSN_INFO ) to dump the insn list INSN_INFO for tag T .


<s> virtual unsigned int execute ( function * ) { execute_tag_collision_avoidance ( ) ; return 0 ; } </s>


<s>
void
execute_tag_collision_avoidance
(
)
{
struct
loop
*
loop
;
df_set_flags
(
DF_RD_PRUNE_DEAD_DEFS
)
;
df_chain_add_problem
(
DF_UD_CHAIN
)
;
df_compute_regs_ever_live
(
true
)
;
df_note_add_problem
(
)
;
df_analyze
(
)
;
df_set_flags
(
DF_DEFER_INSN_RESCAN
)
;
regrename_init
(
true
)
;
regrename_analyze
(
)
;
compute_bb_for_insn
(
)
;
calculate_dominance_info
(
CDI_DOMINATORS
)
;
loop_optimizer_init
(
AVOID_CFG_MODIFICATIONS
)
;
FOR_EACH_LOOP
(
loop
,
LI_FROM_INNERMOST
)
{
tag_map_t
tag_map
(
512
)
;
record_loads
(
tag_map
,
loop
)
;
avoid_collisions
(
tag_map
)
;
if
(
dump_file
)
{
fprintf
(
dump_file
,
"Loop %d: Completed rename.\n"
,
loop
->
num
)
;
tag_map
.
traverse
<
void
*
,
dump_insn_list
>
(
)
;
}
tag_map
.
traverse
<
void
*
,
free_insn_info
>
(
)
;
}
loop_optimizer_finalize
(
)
;
free_dominance_info
(
CDI_DOMINATORS
)
;
regrename_finish
(
)
;
}
</s>

Tag collision avoidance pass for Falkor . The pass runs in two phases for each loop ; the first phase collects all loads that we consider as interesting for renaming into a tag-indexed map of lists . The second phase renames the destination register of the loads in an attempt to spread out the loads into different tags .


<s> bool free_insn_info ( const rtx & t ATTRIBUTE_UNUSED , insn_info_list_t * v , void * arg ATTRIBUTE_UNUSED ) { while ( v -> length ( ) > 0 ) delete v -> pop ( ) ; return true ; } </s>

Callback to free all tag_insn_info objects .


<s> static bool get_load_info ( rtx_insn * insn , struct loop * loop , rtx * dest , rtx * base , rtx * offset , bool * pre_post , bool * ldp ) { if ( ! INSN_P ( insn ) || recog_memoized ( insn ) < 0 ) return false ; rtx pat = PATTERN ( insn ) ; unsigned code = GET_CODE ( pat ) ; bool load_pair = ( code == PARALLEL ) ; if ( load_pair ) { pat = XVECEXP ( pat , 0 , 0 ) ; code = GET_CODE ( pat ) ; } if ( code != SET ) return false ; rtx dest_rtx = SET_DEST ( pat ) ; if ( ! REG_P ( dest_rtx ) ) return false ; unsigned regno = REGNO ( dest_rtx ) ; machine_mode mode = GET_MODE ( dest_rtx ) ; machine_mode inner_mode = GET_MODE_INNER ( mode ) ; if ( ! GET_MODE_SIZE ( mode ) . is_constant ( ) ) return false ; if ( GET_MODE_SIZE ( mode ) . to_constant ( ) != GET_MODE_SIZE ( inner_mode ) . to_constant ( ) ) return false ; if ( ( GET_MODE_SIZE ( mode ) . to_constant ( ) << load_pair ) > GET_MODE_SIZE ( OImode ) ) return false ; if ( regno == SP_REGNUM ) return false ; if ( valid_src_p ( SET_SRC ( pat ) , insn , loop , pre_post , base , offset , load_pair ) ) { * dest = dest_rtx ; * ldp = load_pair ; return true ; } return false ; } </s>

Return true if INSN is a strided load in LOOP . If it is a strided load , set the DEST , BASE and OFFSET . Also , if this is a pre/post increment load , set PRE_POST to true . The routine does checks on the destination of the insn and depends on STRIDED_LOAD_P to check the source and fill in the BASE and OFFSET .


<s> static void ignore_all_aliases ( HARD_REG_SET * unavailable , machine_mode mode , unsigned reg ) { add_to_hard_reg_set ( unavailable , mode , reg ) ; add_to_hard_reg_set ( unavailable , mode , reg + 16 ) ; add_to_hard_reg_set ( unavailable , mode , reg + 32 ) ; add_to_hard_reg_set ( unavailable , mode , reg + 48 ) ; } </s>

Add all aliases of the register to the unavailable register set . REG is the smallest register number that can then be used to reference its aliases . UNAVAILABLE is the hard register set to add the ignored register numbers to and MODE is the mode in which the registers would have been used .


<s> static enum reg_class init_unavailable ( tag_insn_info * insn_info , tag_map_t & tag_map , du_head_p head , HARD_REG_SET * unavailable ) { unsigned dest = head -> regno ; enum reg_class super_class = NO_REGS ; machine_mode mode = GET_MODE ( insn_info -> dest ) ; CLEAR_HARD_REG_SET ( * unavailable ) ; for ( struct du_chain * tmp = head -> first ; tmp ; tmp = tmp -> next_use ) { if ( DEBUG_INSN_P ( tmp -> insn ) ) continue ; * unavailable |= ~ reg_class_contents [ tmp -> cl ] ; super_class = reg_class_superunion [ ( int ) super_class ] [ ( int ) tmp -> cl ] ; } for ( unsigned i = 0 ; i < FIRST_PSEUDO_REGISTER ; i ++ ) if ( fixed_regs [ i ] || global_regs [ i ] ) add_to_hard_reg_set ( unavailable , mode , i ) ; arg_pair_t arg = arg_pair_t ( insn_info , unavailable ) ; tag_map . traverse < arg_pair_t * , check_colliding_tags > ( & arg ) ; ignore_all_aliases ( unavailable , mode , dest & 0xf ) ; return super_class ; } </s>


<s> static bool in_same_chain ( rtx_insn * insn , rtx_insn * cand , unsigned regno ) { struct du_chain * chain = NULL ; du_head_p head = NULL ; int i ; operand_rr_info * op_info = insn_rr [ INSN_UID ( insn ) ] . op_info ; for ( i = 0 ; i < op_info -> n_chains ; i ++ ) { if ( op_info -> heads [ i ] -> regno != regno ) continue ; head = op_info -> heads [ i ] ; if ( ! head -> first ) head = regrename_chain_from_id ( head -> id ) ; bool found_insn = false , found_cand = false ; for ( chain = head -> first ; chain ; chain = chain -> next_use ) { rtx * loc = & SET_DEST ( PATTERN ( chain -> insn ) ) ; if ( chain -> loc != loc ) continue ; if ( chain -> insn == insn ) found_insn = true ; if ( chain -> insn == cand ) found_cand = true ; if ( found_insn && found_cand ) return true ; } } return false ; } </s>

Return whether INSN and CAND are in the same def/use chain .


<s> static bool iv_p ( rtx_insn * insn , rtx reg , struct loop * loop ) { df_ref ause ; unsigned regno = REGNO ( reg ) ; if ( regno == SP_REGNUM ) return false ; for ( ause = DF_REG_USE_CHAIN ( regno ) ; ause ; ause = DF_REF_NEXT_REG ( ause ) ) { if ( ! DF_REF_INSN_INFO ( ause ) || ! NONDEBUG_INSN_P ( DF_REF_INSN ( ause ) ) ) continue ; if ( insn != DF_REF_INSN ( ause ) ) continue ; struct df_insn_info * insn_info = DF_INSN_INFO_GET ( insn ) ; df_ref def_rec ; FOR_EACH_INSN_INFO_DEF ( def_rec , insn_info ) { rtx_insn * insn = DF_REF_INSN ( def_rec ) ; basic_block bb = BLOCK_FOR_INSN ( insn ) ; if ( dominated_by_p ( CDI_DOMINATORS , bb , loop -> header ) && bb -> loop_father == loop ) { if ( recog_memoized ( insn ) < 0 ) continue ; rtx pat = PATTERN ( insn ) ; if ( GET_CODE ( pat ) != SET ) continue ; rtx x = SET_SRC ( pat ) ; if ( GET_CODE ( x ) == ZERO_EXTRACT || GET_CODE ( x ) == ZERO_EXTEND || GET_CODE ( x ) == SIGN_EXTEND ) x = XEXP ( x , 0 ) ; if ( MEM_P ( x ) ) continue ; if ( GET_CODE ( x ) == POST_INC || GET_CODE ( x ) == POST_DEC || GET_CODE ( x ) == PRE_INC || GET_CODE ( x ) == PRE_DEC ) return true ; else if ( BINARY_P ( x ) && ( CONST_INT_P ( XEXP ( x , 0 ) ) || CONST_INT_P ( XEXP ( x , 1 ) ) ) ) { rtx stride = ( CONST_INT_P ( XEXP ( x , 0 ) ) ? XEXP ( x , 0 ) : XEXP ( x , 1 ) ) ; if ( INTVAL ( stride ) < MAX_PREFETCH_STRIDE ) return true ; } } } return false ; } return false ; } </s>

Find the use def chain in which INSN exists and then see if there is a definition inside the loop and outside it . We use this as a simple approximation to determine whether the base register is an IV . The basic idea is to find INSN in the use-def chains for its base register and find all definitions that reach it . Of all these definitions , there should be at least one definition that is a simple addition of a constant value , either as a binary operation or a pre or post update . The function returns true if the base register is estimated to be an IV .


<s> rtl_opt_pass * make_pass_tag_collision_avoidance ( gcc :: context * ctxt ) { return new pass_tag_collision_avoidance ( ctxt ) ; } </s>


<s>
static
void
record_loads
(
tag_map_t
&
tag_map
,
struct
loop
*
loop
)
{
rtx_insn
*
insn
;
basic_block
*
body
,
bb
;
body
=
get_loop_body
(
loop
)
;
for
(
unsigned
i
=
0
;
i
<
loop
->
num_nodes
;
i
++
)
{
bb
=
body
[
i
]
;
FOR_BB_INSNS
(
bb
,
insn
)
{
rtx
base
=
NULL_RTX
;
rtx
dest
=
NULL_RTX
;
rtx
offset
=
NULL_RTX
;
bool
writeback
=
false
;
bool
ldp
=
false
;
if
(
!
INSN_P
(
insn
)
||
DEBUG_INSN_P
(
insn
)
)
continue
;
if
(
get_load_info
(
insn
,
loop
,
&
dest
,
&
base
,
&
offset
,
&
writeback
,
&
ldp
)
)
{
tag_insn_info
*
i
=
new
tag_insn_info
(
insn
,
dest
,
base
,
offset
,
writeback
,
ldp
)
;
rtx
tag
=
GEN_INT
(
i
->
tag
(
)
)
;
tag_map
.
get_or_insert
(
tag
)
.
safe_push
(
i
)
;
}
}
}
if
(
dump_file
)
{
fprintf
(
dump_file
,
"Loop %d: Tag map generated.\n"
,
loop
->
num
)
;
tag_map
.
traverse
<
void
*
,
dump_insn_list
>
(
)
;
}
tag_map
.
traverse
<
void
*
,
single_dest_per_chain
>
(
)
;
}
</s>

Record all loads in LOOP into TAG_MAP indexed by the falkor hardware prefetcher memory tags .