<s> for ( i = <NUM_LIT> ; i < nelt ; ++ i ) if ( dfirst . perm [ i ] == <NUM_LIT> ) { if ( GET_MODE_SIZE ( vmode ) == <NUM_LIT> && dfirst . perm [ i ^ ( nelt / <NUM_LIT> ) ] != <NUM_LIT> ) dfirst . perm [ i ] = dfirst . perm [ i ^ ( nelt / <NUM_LIT> ) ] ^ ( nelt / <NUM_LIT> ) ; else dfirst . perm [ i ] = i ; } else { if ( GET_MODE_SIZE ( vmode ) == <NUM_LIT> && dsecond . perm [ i ^ ( nelt / <NUM_LIT> ) ] != <NUM_LIT> ) dsecond . perm [ i ] = dsecond . perm [ i ^ ( nelt / <NUM_LIT> ) ] ^ ( nelt / <NUM_LIT> ) ; else dsecond . perm [ i ] = i ; } if ( ! d -> testing_p ) { if ( ! ident1 ) dfinal . op0 = dfirst . target = gen_reg_rtx ( d -> vmode ) ; if ( ! ident2 ) dfinal . op1 = dsecond . target = gen_reg_rtx ( d -> vmode ) ; } bool ok ; rtx_insn * seq1 = NULL , * seq2 = NULL ; if ( ! ident1 ) { start_sequence ( ) ; ok = expand_vec_perm_1 ( & dfirst ) ; seq1 = get_insns ( ) ; end_sequence ( ) ; if ( ! ok ) return false ; } if ( ! ident2 ) { start_sequence ( ) ; ok = expand_vec_perm_1 ( & dsecond ) ; seq2 = get_insns ( ) ;


<s> if ( elt >= nelt2 ) { gen = vmode == V16QImode ? gen_vec_interleave_highv16qi : gen_vec_interleave_highv8hi ; elt -= nelt2 ; } else gen = vmode == V16QImode ? gen_vec_interleave_lowv16qi : gen_vec_interleave_lowv8hi ; nelt2 /= <NUM_LIT> ; dest = gen_reg_rtx ( vmode ) ; emit_insn ( gen ( dest , op0 , op0 ) ) ; vmode = get_mode_wider_vector ( vmode ) ; op0 = gen_lowpart ( vmode , dest ) ; } while ( vmode != V4SImode ) ; memset ( perm2 , elt , <NUM_LIT> ) ; dest = gen_reg_rtx ( vmode ) ; ok = expand_vselect ( dest , op0 , perm2 , <NUM_LIT> , d -> testing_p ) ; gcc_assert ( ok ) ; emit_move_insn ( d -> target , gen_lowpart ( d -> vmode , dest ) ) ; return true ; case E_V8HFmode : case E_V8BFmode : if ( d -> testing_p ) return true ; rtx ( * maybe_gen ) ( machine_mode , int , rtx , rtx , rtx ) ; if ( elt >= nelt2 ) { maybe_gen = maybe_gen_vec_interleave_high ; elt -= nelt2 ; } else maybe_gen = maybe_gen_vec_interleave_low ; nelt2 /= <NUM_LIT> ; dest = gen_reg_rtx ( vmode ) ; emit_insn ( maybe_gen ( vmode , <NUM_LIT> , dest , op0 , op0 ) ) ; vmode = V4SImode ; op0 = gen_lowpart ( vmode , dest ) ;


<s> op0 = gen_lowpart ( vmode , dest ) ; memset ( perm2 , elt , <NUM_LIT> ) ; dest = gen_reg_rtx ( vmode ) ; ok = expand_vselect ( dest , op0 , perm2 , <NUM_LIT> , d -> testing_p ) ; gcc_assert ( ok ) ; emit_move_insn ( d -> target , gen_lowpart ( d -> vmode , dest ) ) ; return true ; case E_V8QImode : if ( d -> testing_p ) return true ; do { if ( elt >= nelt2 ) { gen = vmode == V8QImode ? gen_mmx_punpckhbw : gen_mmx_punpckhwd ; elt -= nelt2 ; } else gen = vmode == V8QImode ? gen_mmx_punpcklbw : gen_mmx_punpcklwd ; nelt2 /= <NUM_LIT> ; dest = gen_reg_rtx ( vmode ) ; emit_insn ( gen ( dest , op0 , op0 ) ) ; vmode = get_mode_wider_vector ( vmode ) ; op0 = gen_lowpart ( vmode , dest ) ; } while ( vmode != V2SImode ) ; memset ( perm2 , elt , <NUM_LIT> ) ; dest = gen_reg_rtx ( vmode ) ; ok = expand_vselect ( dest , op0 , perm2 , <NUM_LIT> , d -> testing_p ) ; gcc_assert ( ok ) ; emit_move_insn ( d -> target , gen_lowpart ( d -> vmode , dest ) ) ; return true ; case E_V8HImode : case E_V16QImode : if ( d -> testing_p ) return true ; do { if ( elt >= nelt2 ) { gen = vmode == V16QImode ? gen_vec_interleave_highv16qi : gen_vec_interleave_highv8hi ; elt -= nelt2 ; } else gen = vmode == V16QImode ? gen_vec_interleave_lowv16qi : gen_vec_interleave_lowv8hi ; nelt2 /= <NUM_LIT> ; dest = gen_reg_rtx ( vmode ) ; emit_insn ( gen ( dest , op0 , op0 ) ) ; vmode = get_mode_wider_vector ( vmode ) ; op0 = gen_lowpart ( vmode , dest ) ; } while ( vmode != V4SImode ) ; memset ( perm2 , elt , <NUM_LIT> ) ; dest = gen_reg_rtx ( vmode ) ; ok = expand_vselect ( dest , op0 , perm2 , <NUM_LIT> , d -> testing_p ) ; gcc_assert ( ok ) ; emit_move_insn ( d -> target , gen_lowpart ( d -> vmode , dest ) ) ; return true ; case E_V8HFmode : case E_V8BFmode : if ( d -> testing_p ) return true ; rtx ( * maybe_gen ) ( machine_mode , int , rtx , rtx , rtx ) ; if ( elt >= nelt2 ) { maybe_gen = maybe_gen_vec_interleave_high ; elt -= nelt2 ; } else maybe_gen = maybe_gen_vec_interleave_low ; nelt2 /= <NUM_LIT> ; dest = gen_reg_rtx ( vmode ) ; emit_insn ( maybe_gen ( vmode , <NUM_LIT> , dest , op0 , op0 ) ) ; vmode = V4SImode ; op0 = gen_lowpart ( vmode , dest ) ; memset ( perm2 , elt , <NUM_LIT> ) ;


<s> if ( d -> perm [ <NUM_LIT> ] == nelt ) x = gen_rtx_VEC_MERGE ( vmode , d -> op1 , d -> op0 , GEN_INT ( <NUM_LIT> ) ) ; else x = gen_rtx_VEC_MERGE ( vmode , d -> op0 , d -> op1 , GEN_INT ( <NUM_LIT> ) ) ; emit_insn ( gen_rtx_SET ( d -> target , x ) ) ; return true ;


<s> in_order = true ; for ( i = <NUM_LIT> ; i < nelt ; ++ i ) { unsigned e = dcopy . perm [ i ] ; if ( GET_MODE_SIZE ( d -> vmode ) == <NUM_LIT> && e >= nelt && ( e & ( nelt / <NUM_LIT> - <NUM_LIT> ) ) < min ) e = e - min - ( nelt / <NUM_LIT> ) ; else e = e - min ; if ( e != i ) in_order = false ; dcopy . perm [ i ] = e ; } dcopy . one_operand_p = true ; if ( single_insn_only_p && ! in_order ) return false ; if ( d -> testing_p ) { if ( in_order ) return true ; dcopy . op1 = dcopy . op0 ; return expand_vec_perm_1 ( & dcopy ) ; } shift = GEN_INT ( min * GET_MODE_UNIT_BITSIZE ( d -> vmode ) ) ; if ( GET_MODE_SIZE ( d -> vmode ) == <NUM_LIT> ) { target = gen_reg_rtx ( V1TImode ) ; emit_insn ( gen_ssse3_palignrv1ti ( target , gen_lowpart ( V1TImode , dcopy . op1 ) , gen_lowpart ( V1TImode , dcopy . op0 ) , shift ) ) ; } else { target = gen_reg_rtx ( V2TImode ) ; emit_insn ( gen_avx2_palignrv2ti ( target , gen_lowpart ( V2TImode , dcopy . op1 ) , gen_lowpart ( V2TImode , dcopy . op0 ) , shift ) ) ; }

dcopy . op0 = dcopy . op1 = gen_lowpart ( d -> vmode , target ) ;


<s> static bool expand_vec_perm_pblendv ( struct expand_vec_perm_d * d ) { unsigned i , which , nelt = d -> nelt ; struct expand_vec_perm_d dcopy , dcopy1 ; machine_mode vmode = d -> vmode ; bool ok ; if ( d -> one_operand_p ) return false ; if ( TARGET_AVX2 && GET_MODE_SIZE ( vmode ) == <NUM_LIT> ) ; else if ( TARGET_AVX && ( vmode == V4DFmode || vmode == V8SFmode ) ) ; else if ( TARGET_SSE4_1 && ( GET_MODE_SIZE ( vmode ) == <NUM_LIT> || ( TARGET_MMX_WITH_SSE && GET_MODE_SIZE ( vmode ) == <NUM_LIT> ) || GET_MODE_SIZE ( vmode ) == <NUM_LIT> ) ) ; else return false ; for ( i = <NUM_LIT> , which = <NUM_LIT> ; i < nelt ; ++ i ) { unsigned e = d -> perm [ i ] ; if ( e != i ) which |= ( e < nelt ? <NUM_LIT> : <NUM_LIT> ) ; } if ( which != <NUM_LIT> && which != <NUM_LIT> ) return false ; if ( d -> testing_p && GET_MODE_SIZE ( vmode ) == <NUM_LIT> ) return true ; dcopy = * d ; if ( which == <NUM_LIT> ) dcopy . op0 = dcopy . op1 = d -> op1 ; else dcopy . op0 = dcopy . op1 = d -> op0 ; if ( ! d -> testing_p ) dcopy . target = gen_reg_rtx ( vmode ) ; dcopy . one_operand_p = true ; for ( i = <NUM_LIT> ; i < nelt ; ++ i ) dcopy . perm [ i ] = d -> perm [ i ] & ( nelt - <NUM_LIT> ) ; ok = expand_vec_perm_1 ( & dcopy ) ; if ( GET_MODE_SIZE ( vmode ) != <NUM_LIT> && ! ok ) return false ; else gcc_assert ( ok ) ; if ( d -> testing_p ) return true ; dcopy1 = * d ; if ( which == <NUM_LIT> ) dcopy1 . op1 = dcopy . target ; else dcopy1 . op0 = dcopy . target ;

for ( i = <NUM_LIT> ; i < nelt ; ++ i ) dcopy1 . perm [ i ] = ( ( d -> perm [ i ] >= nelt ) ? ( nelt + i ) : i ) ;


<s> gen_vec_shr = vmode == E_V16QImode ? gen_vec_shr_v16qi : gen_vec_shr_v8hi ; gen_vec_shl = vmode == E_V16QImode ? gen_vec_shl_v16qi : gen_vec_shl_v8hi ; imode = GET_MODE_INNER ( vmode ) ; inner_size = GET_MODE_BITSIZE ( imode ) ; op0 = gen_reg_rtx ( vmode ) ; op1 = gen_reg_rtx ( vmode ) ; if ( start1 ) emit_insn ( gen_vec_shr ( op0 , d -> op0 , GEN_INT ( start1 * inner_size ) ) ) ; else emit_move_insn ( op0 , d -> op0 ) ; dop1 = d -> op1 ; if ( d -> one_operand_p ) dop1 = d -> op0 ; int shl_offset = end1 - start1 + <NUM_LIT> - start2 % nelt ; if ( shl_offset ) emit_insn ( gen_vec_shl ( op1 , dop1 , GEN_INT ( shl_offset * inner_size ) ) ) ; else emit_move_insn ( op1 , dop1 ) ; if ( clear_op0 || clear_op1 ) { rtx vec [ <NUM_LIT> ] ; rtx const_vec ; rtx clear ; for ( i = <NUM_LIT> ; i != nelt ; i ++ ) { if ( i < ( end1 - start1 + <NUM_LIT> ) ) vec [ i ] = gen_int_mode ( ( HOST_WIDE_INT_1U << inner_size ) - <NUM_LIT> , imode ) ;


<s> else { unsigned pair_idx = <NUM_LIT> , lone_idx = <NUM_LIT> , shift ; for ( i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) if ( ( d -> perm [ i ] > <NUM_LIT> && count == <NUM_LIT> ) || ( d -> perm [ i ] < <NUM_LIT> && count == <NUM_LIT> ) ) lone_idx = i ; gcc_assert ( count == ( lone_idx ? <NUM_LIT> : <NUM_LIT> ) ) ; shift = lone_idx & <NUM_LIT> ; pair_idx = <NUM_LIT> - lone_idx + <NUM_LIT> * shift ; perm1 [ <NUM_LIT> ] = perm1 [ <NUM_LIT> ] = ( count == <NUM_LIT> ) ? d -> perm [ lone_idx ] : d -> perm [ lone_idx ] - <NUM_LIT> ; perm1 [ <NUM_LIT> ] = perm1 [ <NUM_LIT> ] = ( count == <NUM_LIT> ) ? d -> perm [ pair_idx ] : d -> perm [ pair_idx ] + <NUM_LIT> ; if ( count == <NUM_LIT> ) std :: swap ( d -> op0 , d -> op1 ) ; ok = expand_vselect_vconcat ( tmp , d -> op0 , d -> op1 , perm1 , d -> nelt , false ) ;


<s> gcc_assert ( GET_MODE_SIZE ( o ) == GET_MODE_SIZE ( n ) ) ;


<s> gcc_assert ( GET_MODE_SIZE ( o ) == GET_MODE_SIZE ( n ) ) ; return n ;


<s> case MODE_INT : if ( mode != V32QImode ) { if ( ! MEM_P ( op0 ) ) { orig_op0 = op0 ; op0 = gen_reg_rtx ( V32QImode ) ; } else op0 = gen_lowpart ( V32QImode , op0 ) ; op1 = gen_lowpart ( V32QImode , op1 ) ; mode = V32QImode ; } break ; case MODE_VECTOR_FLOAT : break ; default : gcc_unreachable ( ) ; } switch ( mode ) { default : gcc_unreachable ( ) ; case E_V32QImode : extract = gen_avx_vextractf128v32qi ; mode = V16QImode ; break ; case E_V16BFmode : extract = gen_avx_vextractf128v16bf ; mode = V8BFmode ; break ; case E_V16HFmode : extract = gen_avx_vextractf128v16hf ; mode = V8HFmode ; break ; case E_V8SFmode : extract = gen_avx_vextractf128v8sf ; mode = V4SFmode ; break ; case E_V4DFmode : extract = gen_avx_vextractf128v4df ; mode = V2DFmode ; break ; } if ( MEM_P ( op1 ) ) { rtx r = gen_reg_rtx ( mode ) ; m = adjust_address ( op1 , mode , <NUM_LIT> ) ; emit_move_insn ( r , m ) ; m = adjust_address ( op1 , mode , <NUM_LIT> ) ; r = gen_rtx_VEC_CONCAT ( GET_MODE ( op0 ) , r , m ) ; emit_move_insn ( op0 , r ) ; } else if ( MEM_P ( op0 ) ) {

m = adjust_address ( op0 , mode , <NUM_LIT> ) ;


<s> case E_V16HFmode : extract = gen_avx_vextractf128v16hf ; mode = V8HFmode ; break ; case E_V8SFmode : extract = gen_avx_vextractf128v8sf ; mode = V4SFmode ; break ; case E_V4DFmode : extract = gen_avx_vextractf128v4df ; mode = V2DFmode ; break ; } if ( MEM_P ( op1 ) ) { rtx r = gen_reg_rtx ( mode ) ; m = adjust_address ( op1 , mode , <NUM_LIT> ) ; emit_move_insn ( r , m ) ; m = adjust_address ( op1 , mode , <NUM_LIT> ) ; r = gen_rtx_VEC_CONCAT ( GET_MODE ( op0 ) , r , m ) ; emit_move_insn ( op0 , r ) ; } else if ( MEM_P ( op0 ) ) {


<s> bool ix86_check_builtin_isa_match ( unsigned int fcode , HOST_WIDE_INT * pbisa , HOST_WIDE_INT * pbisa2 ) { HOST_WIDE_INT isa = ix86_isa_flags ; HOST_WIDE_INT isa2 = ix86_isa_flags2 ; HOST_WIDE_INT bisa = ix86_builtins_isa [ fcode ] . isa ; HOST_WIDE_INT bisa2 = ix86_builtins_isa [ fcode ] . isa2 ; if ( ( ( bisa & ( OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A ) ) == ( OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A ) ) && ( isa & ( OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A ) ) != <NUM_LIT> ) isa |= ( OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A ) ; if ( ( ( bisa & ( OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32 ) ) == ( OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32 ) ) && ( isa & ( OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32 ) ) != <NUM_LIT> ) isa |= ( OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32 ) ; if ( ( ( bisa & ( OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4 ) ) == ( OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4 ) ) && ( isa & ( OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4 ) ) != <NUM_LIT> ) isa |= ( OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4 ) ; if ( ( ( ( bisa & ( OPTION_MASK_ISA_AVX512VNNI | OPTION_MASK_ISA_AVX512VL ) ) == ( OPTION_MASK_ISA_AVX512VNNI | OPTION_MASK_ISA_AVX512VL ) ) || ( bisa2 & OPTION_MASK_ISA2_AVXVNNI ) != <NUM_LIT> ) && ( ( ( isa & ( OPTION_MASK_ISA_AVX512VNNI | OPTION_MASK_ISA_AVX512VL ) ) == ( OPTION_MASK_ISA_AVX512VNNI | OPTION_MASK_ISA_AVX512VL ) ) || ( isa2 & OPTION_MASK_ISA2_AVXVNNI ) != <NUM_LIT> ) ) { isa |= OPTION_MASK_ISA_AVX512VNNI | OPTION_MASK_ISA_AVX512VL ; isa2 |= OPTION_MASK_ISA2_AVXVNNI ; } if ( ( ( ( bisa & ( OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL ) ) == ( OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL ) ) || ( bisa2 & OPTION_MASK_ISA2_AVXIFMA ) != <NUM_LIT> ) && ( ( ( isa & ( OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL ) ) == ( OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL ) ) || ( isa2 & OPTION_MASK_ISA2_AVXIFMA ) != <NUM_LIT> ) ) { isa |= OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL ; isa2 |= OPTION_MASK_ISA2_AVXIFMA ; } if ( ( ( ( bisa & OPTION_MASK_ISA_AVX512VL ) != <NUM_LIT> && ( bisa2 & OPTION_MASK_ISA2_AVX512BF16 ) != <NUM_LIT> ) && ( bisa2 & OPTION_MASK_ISA2_AVXNECONVERT ) != <NUM_LIT> ) && ( ( ( isa & OPTION_MASK_ISA_AVX512VL ) != <NUM_LIT> && ( isa2 & OPTION_MASK_ISA2_AVX512BF16 ) != <NUM_LIT> ) || ( isa2 & OPTION_MASK_ISA2_AVXNECONVERT ) != <NUM_LIT> ) ) { isa |= OPTION_MASK_ISA_AVX512VL ; isa2 |= OPTION_MASK_ISA2_AVXNECONVERT | OPTION_MASK_ISA2_AVX512BF16 ; } if ( ( bisa & OPTION_MASK_ISA_MMX ) && ! TARGET_MMX && TARGET_MMX_WITH_SSE && fcode != IX86_BUILTIN_MASKMOVQ ) { bisa &= ~ OPTION_MASK_ISA_MMX ; bisa |= OPTION_MASK_ISA_SSE2 ; } if ( pbisa ) * pbisa = bisa ;


<s> HOST_WIDE_INT val_broadcast ; scalar_int_mode broadcast_mode ; if ( TARGET_AVX2 && ix86_broadcast ( val , GET_MODE_BITSIZE ( QImode ) , val_broadcast ) ) broadcast_mode = QImode ; else if ( TARGET_AVX2 && ix86_broadcast ( val , GET_MODE_BITSIZE ( HImode ) , val_broadcast ) ) broadcast_mode = HImode ; else if ( ix86_broadcast ( val , GET_MODE_BITSIZE ( SImode ) , val_broadcast ) ) broadcast_mode = SImode ; else if ( TARGET_64BIT && ix86_broadcast ( val , GET_MODE_BITSIZE ( DImode ) , val_broadcast ) ) broadcast_mode = DImode ; else return nullptr ; for ( int i = <NUM_LIT> ; i < CONST_WIDE_INT_NUNITS ( op ) ; i ++ ) if ( val != CONST_WIDE_INT_ELT ( op , i ) ) return nullptr ; unsigned int nunits = ( GET_MODE_SIZE ( mode ) / GET_MODE_SIZE ( broadcast_mode ) ) ;


<s> case E_DFmode : case E_XFmode : case E_QImode : case E_HImode : case E_SImode : simple : tmp = ix86_expand_compare ( code , op0 , op1 ) ; tmp = gen_rtx_IF_THEN_ELSE ( VOIDmode , tmp , gen_rtx_LABEL_REF ( VOIDmode , label ) , pc_rtx ) ; emit_jump_insn ( gen_rtx_SET ( pc_rtx , tmp ) ) ; return ; case E_DImode : if ( TARGET_64BIT ) goto simple ; case E_TImode : if ( ( code == EQ || code == NE ) && ! optimize_insn_for_size_p ( ) ) goto simple ; { rtx lo [ <NUM_LIT> ] , hi [ <NUM_LIT> ] ; rtx_code_label * label2 ; enum rtx_code code1 , code2 , code3 ; machine_mode submode ; if ( CONSTANT_P ( op0 ) && ! CONSTANT_P ( op1 ) ) { std :: swap ( op0 , op1 ) ; code = swap_condition ( code ) ; } split_double_mode ( mode , & op0 , <NUM_LIT> , lo + <NUM_LIT> , hi + <NUM_LIT> ) ; split_double_mode ( mode , & op1 , <NUM_LIT> , lo + <NUM_LIT> , hi + <NUM_LIT> ) ; submode = mode == DImode ? SImode : DImode ; if ( CONST_INT_P ( hi [ <NUM_LIT> ] ) ) switch ( code ) { case LT : case LTU : case GE : case GEU : if ( lo [ <NUM_LIT> ] == const0_rtx ) { ix86_expand_branch ( code , hi [ <NUM_LIT> ] , hi [ <NUM_LIT> ] , label ) ; return ; } break ; case LE : case LEU : case GT : case GTU : if ( lo [ <NUM_LIT> ] == constm1_rtx ) { ix86_expand_branch ( code , hi [ <NUM_LIT> ] , hi [ <NUM_LIT> ] , label ) ; return ; } break ; default : break ; } switch ( code ) { case LE : case LEU : case GT : case GTU : std :: swap ( lo [ <NUM_LIT> ] , lo [ <NUM_LIT> ] ) ; std :: swap ( hi [ <NUM_LIT> ] , hi [ <NUM_LIT> ] ) ;


<s> switch ( code ) { case LE : case LEU : case GT : case GTU : std :: swap ( lo [ <NUM_LIT> ] , lo [ <NUM_LIT> ] ) ; std :: swap ( hi [ <NUM_LIT> ] , hi [ <NUM_LIT> ] ) ; code = swap_condition ( code ) ; case LT : case LTU : case GE : case GEU : { bool uns = ( code == LTU || code == GEU ) ; rtx ( * sbb_insn ) ( machine_mode , rtx , rtx , rtx ) = uns ? gen_sub3_carry_ccc : gen_sub3_carry_ccgz ; if ( ! nonimmediate_operand ( lo [ <NUM_LIT> ] , submode ) ) lo [ <NUM_LIT> ] = force_reg ( submode , lo [ <NUM_LIT> ] ) ; if ( ! x86_64_general_operand ( lo [ <NUM_LIT> ] , submode ) ) lo [ <NUM_LIT> ] = force_reg ( submode , lo [ <NUM_LIT> ] ) ; if ( ! register_operand ( hi [ <NUM_LIT> ] , submode ) ) hi [ <NUM_LIT> ] = force_reg ( submode , hi [ <NUM_LIT> ] ) ; if ( ( uns && ! nonimmediate_operand ( hi [ <NUM_LIT> ] , submode ) ) || ( ! uns && ! x86_64_general_operand ( hi [ <NUM_LIT> ] , submode ) ) ) hi [ <NUM_LIT> ] = force_reg ( submode , hi [ <NUM_LIT> ] ) ; emit_insn ( gen_cmp_1 ( submode , lo [ <NUM_LIT> ] , lo [ <NUM_LIT> ] ) ) ; tmp = gen_rtx_SCRATCH ( submode ) ; emit_insn ( sbb_insn ( submode , tmp , hi [ <NUM_LIT> ] , hi [ <NUM_LIT> ] ) ) ; tmp = gen_rtx_REG ( uns ? CCCmode : CCGZmode , FLAGS_REG ) ; ix86_expand_branch ( code , tmp , const0_rtx , label ) ; return ; } default : break ; } label2 = gen_label_rtx ( ) ; code1 = code ; code2 = swap_condition ( code ) ; code3 = unsigned_condition ( code ) ; switch ( code ) { case LT : case GT : case LTU : case GTU : break ; case LE : code1 = LT ; code2 = GT ; break ; case GE : code1 = GT ; code2 = LT ; break ; case LEU : code1 = LTU ; code2 = GTU ; break ; case GEU : code1 = GTU ; code2 = LTU ; break ; case EQ : code1 = UNKNOWN ; code2 = NE ; break ; case NE : code2 = UNKNOWN ; break ; default : gcc_unreachable ( ) ; } if ( code1 != UNKNOWN ) ix86_expand_branch ( code1 , hi [ <NUM_LIT> ] , hi [ <NUM_LIT> ] , label ) ; if ( code2 != UNKNOWN ) ix86_expand_branch ( code2 , hi [ <NUM_LIT> ] , hi [ <NUM_LIT> ] , label2 ) ;

ix86_expand_branch ( code3 , lo [ <NUM_LIT> ] , lo [ <NUM_LIT> ] , label ) ;


<s> ret = simplify_const_unary_operation ( FLOAT_EXTEND , SFmode , val , BFmode ) ; if ( ret ) return ret ; ret = gen_reg_rtx ( SImode ) ;

emit_move_insn ( ret , GEN_INT ( INTVAL ( op ) & <NUM_LIT> ) ) ;


<s> dst = operands [ <NUM_LIT> ] ; src = operands [ <NUM_LIT> ] ; set = gen_rtx_fmt_e ( code , mode , src ) ; set = gen_rtx_SET ( dst , set ) ; if ( use_sse ) { rtx mask , use , clob ; mask = ix86_build_signbit_mask ( vmode , vector_mode , code == ABS ) ; use = gen_rtx_USE ( VOIDmode , mask ) ;

if ( vector_mode || mode == TFmode ) par = gen_rtvec ( <NUM_LIT> , set , use ) ;


<s> maskcmp = true ; cmp_mode = nbits > <NUM_LIT> ? int_mode_for_size ( nbits , <NUM_LIT> ) . require ( ) : E_QImode ; } else cmp_mode = cmp_ops_mode ; cmp_op0 = force_reg ( cmp_ops_mode , cmp_op0 ) ; int ( * op1_predicate ) ( rtx , machine_mode ) = VECTOR_MODE_P ( cmp_ops_mode ) ? vector_operand : nonimmediate_operand ; if ( ! op1_predicate ( cmp_op1 , cmp_ops_mode ) ) cmp_op1 = force_reg ( cmp_ops_mode , cmp_op1 ) ; if ( optimize || ( maskcmp && cmp_mode != mode ) || ( op_true && reg_overlap_mentioned_p ( dest , op_true ) ) || ( op_false && reg_overlap_mentioned_p ( dest , op_false ) ) ) dest = gen_reg_rtx ( maskcmp ? cmp_mode : mode ) ; if ( maskcmp ) { bool ok = ix86_expand_mask_vec_cmp ( dest , code , cmp_op0 , cmp_op1 ) ; gcc_assert ( ok ) ; return dest ; } x = gen_rtx_fmt_ee ( code , cmp_mode , cmp_op0 , cmp_op1 ) ; if ( cmp_mode != mode ) {


<s> case NE : check_unordered = true ; mode = CCZmode ; const_val = const1_rtx ; break ; default : gcc_unreachable ( ) ; } target = gen_reg_rtx ( SImode ) ; emit_move_insn ( target , const_val ) ; target = gen_rtx_SUBREG ( QImode , target , <NUM_LIT> ) ; if ( ( optimize && ! register_operand ( op0 , mode0 ) ) || ! insn_p -> operand [ <NUM_LIT> ] . predicate ( op0 , mode0 ) ) op0 = copy_to_mode_reg ( mode0 , op0 ) ; if ( ( optimize && ! register_operand ( op1 , mode1 ) ) || ! insn_p -> operand [ <NUM_LIT> ] . predicate ( op1 , mode1 ) ) op1 = copy_to_mode_reg ( mode1 , op1 ) ; pat = GEN_FCN ( icode ) ( op0 , op1 ) ; if ( ! pat ) return <NUM_LIT> ; set_dst = SET_DEST ( pat ) ; emit_insn ( pat ) ; return ix86_ssecom_setcc ( comparison , check_unordered , mode , set_dst , target ) ;


<s> case UNEQ : case UNLT : case UNLE : case LTGT : if ( ordered == non_signaling ) ordered = ! ordered ; break ; case EQ : check_unordered = true ; mode = CCZmode ; break ; case NE : gcc_assert ( ! ordered ) ; check_unordered = true ; mode = CCZmode ; const_val = const1_rtx ; break ; default : gcc_unreachable ( ) ; } target = gen_reg_rtx ( SImode ) ; emit_move_insn ( target , const_val ) ; target = gen_rtx_SUBREG ( QImode , target , <NUM_LIT> ) ; if ( ( optimize && ! register_operand ( op0 , mode0 ) ) || ! insn_p -> operand [ <NUM_LIT> ] . predicate ( op0 , mode0 ) ) op0 = copy_to_mode_reg ( mode0 , op0 ) ; if ( ( optimize && ! register_operand ( op1 , mode1 ) ) || ! insn_p -> operand [ <NUM_LIT> ] . predicate ( op1 , mode1 ) ) op1 = copy_to_mode_reg ( mode1 , op1 ) ; if ( non_signaling ) icode = ( icode == CODE_FOR_sse_comi_round ? CODE_FOR_sse_ucomi_round : CODE_FOR_sse2_ucomi_round ) ; pat = GEN_FCN ( icode ) ( op0 , op1 , op3 ) ; if ( ! pat ) return <NUM_LIT> ; if ( INTVAL ( op3 ) == NO_ROUND ) { pat = ix86_erase_embedded_rounding ( pat ) ; if ( ! pat ) return <NUM_LIT> ; set_dst = SET_DEST ( pat ) ; } else { gcc_assert ( GET_CODE ( pat ) == SET ) ; set_dst = SET_DEST ( pat ) ; } emit_insn ( pat ) ; return ix86_ssecom_setcc ( comparison , check_unordered , mode , set_dst , target ) ;


<s> machine_mode himode , qimode = GET_MODE ( dest ) ; rtx hop1 , hop2 , hdest ; rtx ( * gen_extend ) ( rtx , rtx ) ; rtx ( * gen_truncate ) ( rtx , rtx ) ; if ( qimode == E_V64QImode ) return false ; if ( ! TARGET_AVX512BW ) return false ; if ( ( qimode == V8QImode || qimode == V16QImode ) && ! TARGET_AVX512VL ) return false ; if ( qimode == V32QImode && ( TARGET_PREFER_AVX128 || TARGET_PREFER_AVX256 ) ) return false ; switch ( qimode ) { case E_V8QImode : himode = V8HImode ; gen_extend = gen_zero_extendv8qiv8hi2 ; gen_truncate = gen_truncv8hiv8qi2 ; break ; case E_V16QImode : himode = V16HImode ; gen_extend = gen_zero_extendv16qiv16hi2 ; gen_truncate = gen_truncv16hiv16qi2 ; break ; case E_V32QImode : himode = V32HImode ; gen_extend = gen_zero_extendv32qiv32hi2 ; gen_truncate = gen_truncv32hiv32qi2 ; break ; default : gcc_unreachable ( ) ;


<s> if ( GET_MODE_NUNITS ( GET_MODE ( x ) ) * <NUM_LIT> == n_elts ) { rtx ops [ <NUM_LIT> ] = { XVECEXP ( vals , <NUM_LIT> , <NUM_LIT> ) , XVECEXP ( vals , <NUM_LIT> , <NUM_LIT> ) } ; if ( inner_mode == QImode || inner_mode == HImode || inner_mode == TImode || inner_mode == HFmode || inner_mode == BFmode ) { unsigned int n_bits = n_elts * GET_MODE_SIZE ( inner_mode ) ; scalar_mode elt_mode = inner_mode == TImode ? DImode : SImode ; n_bits /= GET_MODE_SIZE ( elt_mode ) ; mode = mode_for_vector ( elt_mode , n_bits ) . require ( ) ; inner_mode = mode_for_vector ( elt_mode , n_bits / <NUM_LIT> ) . require ( ) ; ops [ <NUM_LIT> ] = gen_lowpart ( inner_mode , ops [ <NUM_LIT> ] ) ; ops [ <NUM_LIT> ] = gen_lowpart ( inner_mode , ops [ <NUM_LIT> ] ) ; subtarget = gen_reg_rtx ( mode ) ; } ix86_expand_vector_init_concat ( mode , subtarget , ops , <NUM_LIT> ) ; if ( subtarget != target ) emit_move_insn ( target , gen_lowpart ( GET_MODE ( target ) , subtarget ) ) ; return ; } gcc_unreachable ( ) ; } for ( i = <NUM_LIT> ; i < n_elts ; ++ i ) { x = XVECEXP ( vals , <NUM_LIT> , i ) ; if ( ! ( CONST_SCALAR_INT_P ( x ) || CONST_DOUBLE_P ( x ) || CONST_FIXED_P ( x ) ) ) n_var ++ , one_var = i ; else if ( x != CONST0_RTX ( inner_mode ) ) all_const_zero = false ; if ( i > <NUM_LIT> && ! rtx_equal_p ( x , XVECEXP ( vals , <NUM_LIT> , <NUM_LIT> ) ) ) all_same = false ; } if ( n_var == <NUM_LIT> ) { emit_move_insn ( target , gen_rtx_CONST_VECTOR ( mode , XVEC ( vals , <NUM_LIT> ) ) ) ; return ;


<s> int i ; rtx x ; if ( n_elts != XVECLEN ( vals , <NUM_LIT> ) ) { rtx subtarget = target ; x = XVECEXP ( vals , <NUM_LIT> , <NUM_LIT> ) ; gcc_assert ( GET_MODE_INNER ( GET_MODE ( x ) ) == inner_mode ) ; if ( GET_MODE_NUNITS ( GET_MODE ( x ) ) * <NUM_LIT> == n_elts ) { rtx ops [ <NUM_LIT> ] = { XVECEXP ( vals , <NUM_LIT> , <NUM_LIT> ) , XVECEXP ( vals , <NUM_LIT> , <NUM_LIT> ) } ; if ( inner_mode == QImode || inner_mode == HImode || inner_mode == TImode || inner_mode == HFmode || inner_mode == BFmode ) { unsigned int n_bits = n_elts * GET_MODE_SIZE ( inner_mode ) ; scalar_mode elt_mode = inner_mode == TImode ? DImode : SImode ; n_bits /= GET_MODE_SIZE ( elt_mode ) ; mode = mode_for_vector ( elt_mode , n_bits ) . require ( ) ; inner_mode = mode_for_vector ( elt_mode , n_bits / <NUM_LIT> ) . require ( ) ; ops [ <NUM_LIT> ] = gen_lowpart ( inner_mode , ops [ <NUM_LIT> ] ) ; ops [ <NUM_LIT> ] = gen_lowpart ( inner_mode , ops [ <NUM_LIT> ] ) ; subtarget = gen_reg_rtx ( mode ) ; } ix86_expand_vector_init_concat ( mode , subtarget , ops , <NUM_LIT> ) ; if ( subtarget != target ) emit_move_insn ( target , gen_lowpart ( GET_MODE ( target ) , subtarget ) ) ; return ; } gcc_unreachable ( ) ; } for ( i = <NUM_LIT> ; i < n_elts ; ++ i ) {


<s> case E_V32HFmode : half_mode = V16HFmode ; break ; case E_V32BFmode : half_mode = V16BFmode ; break ; case E_V16SImode : half_mode = V8SImode ; break ; case E_V16SFmode : half_mode = V8SFmode ; break ; case E_V8DImode : half_mode = V4DImode ; break ; case E_V8DFmode : half_mode = V4DFmode ; break ; case E_V16HFmode : half_mode = V8HFmode ; break ; case E_V16BFmode : half_mode = V8BFmode ; break ; case E_V8SImode : half_mode = V4SImode ; break ; case E_V8SFmode : half_mode = V4SFmode ; break ; case E_V4DImode : half_mode = V2DImode ; break ; case E_V4DFmode : half_mode = V2DFmode ; break ; case E_V4SImode : half_mode = V2SImode ; break ; case E_V4SFmode : half_mode = V2SFmode ; break ; case E_V2DImode : half_mode = DImode ; break ; case E_V2SImode : half_mode = SImode ; break ; case E_V2DFmode : half_mode = DFmode ; break ; case E_V2SFmode : half_mode = SFmode ; break ; default : gcc_unreachable ( ) ; } if ( ! register_operand ( ops [ <NUM_LIT> ] , half_mode ) ) ops [ <NUM_LIT> ] = force_reg ( half_mode , ops [ <NUM_LIT> ] ) ; if ( ! register_operand ( ops [ <NUM_LIT> ] , half_mode ) ) ops [ <NUM_LIT> ] = force_reg ( half_mode , ops [ <NUM_LIT> ] ) ; emit_insn ( gen_rtx_SET ( target , gen_rtx_VEC_CONCAT ( mode , ops [ <NUM_LIT> ] , ops [ <NUM_LIT> ] ) ) ) ; break ; case <NUM_LIT> : switch ( mode ) { case E_V4DImode : half_mode = V2DImode ; break ; case E_V4DFmode : half_mode = V2DFmode ; break ; case E_V4SImode : half_mode = V2SImode ; break ; case E_V4SFmode : half_mode = V2SFmode ; break ; default : gcc_unreachable ( ) ; } goto half ; case <NUM_LIT> : switch ( mode ) { case E_V8DImode : half_mode = V4DImode ; break ; case E_V8DFmode : half_mode = V4DFmode ; break ; case E_V8SImode : half_mode = V4SImode ; break ; case E_V8SFmode : half_mode = V4SFmode ; break ; default : gcc_unreachable ( ) ; } goto half ; case <NUM_LIT> : switch ( mode ) { case E_V16SImode : half_mode = V8SImode ; break ; case E_V16SFmode : half_mode = V8SFmode ; break ; default : gcc_unreachable ( ) ; } goto half ; half : i = n - <NUM_LIT> ; for ( j = <NUM_LIT> ; j != - <NUM_LIT> ; j -- ) { half [ j ] = gen_reg_rtx ( half_mode ) ; switch ( n >> <NUM_LIT> ) { case <NUM_LIT> : v = gen_rtvec ( <NUM_LIT> , ops [ i - <NUM_LIT> ] , ops [ i ] ) ; i -= <NUM_LIT> ; break ; case <NUM_LIT> :

v = gen_rtvec ( <NUM_LIT> , ops [ i - <NUM_LIT> ] , ops [ i - <NUM_LIT> ] , ops [ i - <NUM_LIT> ] , ops [ i ] ) ;


<s> rtx half [ <NUM_LIT> ] ; rtvec v ; int i , j ; switch ( n ) { case <NUM_LIT> : switch ( mode ) { case E_V32HFmode : half_mode = V16HFmode ; break ; case E_V32BFmode : half_mode = V16BFmode ; break ; case E_V16SImode : half_mode = V8SImode ; break ; case E_V16SFmode : half_mode = V8SFmode ; break ; case E_V8DImode : half_mode = V4DImode ; break ; case E_V8DFmode : half_mode = V4DFmode ; break ; case E_V16HFmode : half_mode = V8HFmode ; break ; case E_V16BFmode : half_mode = V8BFmode ; break ; case E_V8SImode : half_mode = V4SImode ; break ; case E_V8SFmode : half_mode = V4SFmode ; break ; case E_V4DImode : half_mode = V2DImode ; break ; case E_V4DFmode : half_mode = V2DFmode ; break ; case E_V4SImode : half_mode = V2SImode ; break ; case E_V4SFmode : half_mode = V2SFmode ; break ; case E_V2DImode : half_mode = DImode ; break ; case E_V2SImode : half_mode = SImode ; break ; case E_V2DFmode : half_mode = DFmode ; break ; case E_V2SFmode : half_mode = SFmode ; break ; default : gcc_unreachable ( ) ; } if ( ! register_operand ( ops [ <NUM_LIT> ] , half_mode ) ) ops [ <NUM_LIT> ] = force_reg ( half_mode , ops [ <NUM_LIT> ] ) ; if ( ! register_operand ( ops [ <NUM_LIT> ] , half_mode ) ) ops [ <NUM_LIT> ] = force_reg ( half_mode , ops [ <NUM_LIT> ] ) ; emit_insn ( gen_rtx_SET ( target , gen_rtx_VEC_CONCAT ( mode , ops [ <NUM_LIT> ] , ops [ <NUM_LIT> ] ) ) ) ; break ; case <NUM_LIT> : switch ( mode ) { case E_V4DImode :


<s> emit_insn ( gen_rtx_SET ( target , x ) ) ; return true ; } return false ; case E_V8QImode : case E_V4QImode : if ( ! mmx_ok ) return false ; goto widen ; case E_V8HImode : case E_V8HFmode : case E_V8BFmode : if ( TARGET_AVX2 ) return ix86_vector_duplicate_value ( mode , target , val ) ; if ( TARGET_SSE2 ) { struct expand_vec_perm_d dperm ; rtx tmp1 , tmp2 ; permute : memset ( & dperm , <NUM_LIT> , sizeof ( dperm ) ) ; dperm . target = target ; dperm . vmode = mode ; dperm . nelt = GET_MODE_NUNITS ( mode ) ; dperm . op0 = dperm . op1 = gen_reg_rtx ( mode ) ; dperm . one_operand_p = true ; if ( mode == V8HFmode || mode == V8BFmode ) { tmp1 = force_reg ( GET_MODE_INNER ( mode ) , val ) ; tmp2 = gen_reg_rtx ( mode ) ; emit_insn ( maybe_gen_vec_set_0 ( mode , tmp2 , CONST0_RTX ( mode ) , tmp1 ) ) ; tmp1 = gen_lowpart ( mode , tmp2 ) ; } else { tmp1 = gen_reg_rtx ( SImode ) ; emit_move_insn ( tmp1 , gen_lowpart ( SImode , val ) ) ; tmp2 = gen_reg_rtx ( V4SImode ) ; emit_insn ( gen_vec_setv4si_0 ( tmp2 , CONST0_RTX ( V4SImode ) , tmp1 ) ) ; tmp1 = gen_lowpart ( mode , tmp2 ) ; } emit_move_insn ( dperm . op0 , tmp1 ) ; ok = ( expand_vec_perm_1 ( & dperm ) || expand_vec_perm_broadcast_1 ( & dperm ) ) ; gcc_assert ( ok ) ; return ok ; } goto widen ; case E_V16QImode : if ( TARGET_AVX2 ) return ix86_vector_duplicate_value ( mode , target , val ) ; if ( TARGET_SSE2 ) goto permute ; goto widen ; widen : { machine_mode smode , wsmode , wvmode ; rtx x ; smode = GET_MODE_INNER ( mode ) ; wvmode = get_mode_wider_vector ( mode ) ; wsmode = GET_MODE_INNER ( wvmode ) ; val = convert_modes ( wsmode , smode , val , true ) ; if ( smode == QImode && ! TARGET_PARTIAL_REG_STALL ) emit_insn ( gen_insv_1 ( wsmode , val , val ) ) ; else { x = expand_simple_binop ( wsmode , ASHIFT , val , GEN_INT ( GET_MODE_BITSIZE ( smode ) ) , NULL_RTX , <NUM_LIT> , OPTAB_LIB_WIDEN ) ; val = expand_simple_binop ( wsmode , IOR , val , x , x , <NUM_LIT> , OPTAB_LIB_WIDEN ) ; } x = gen_reg_rtx ( wvmode ) ; ok = ix86_expand_vector_init_duplicate ( mmx_ok , wvmode , x , val ) ; gcc_assert ( ok ) ; emit_move_insn ( target , gen_lowpart ( GET_MODE ( target ) , x ) ) ; return ok ; } case E_V16HImode : case E_V16HFmode :


<s> quarter_mode = V16QImode ; half_mode = V32QImode ; goto quarter ; case E_V32HImode : quarter_mode = V8HImode ; half_mode = V16HImode ; goto quarter ; case E_V32HFmode : quarter_mode = V8HFmode ; half_mode = V16HFmode ; goto quarter ; case E_V32BFmode : quarter_mode = V8BFmode ; half_mode = V16BFmode ; goto quarter ; quarter : n = GET_MODE_NUNITS ( mode ) ; for ( i = <NUM_LIT> ; i < n ; i ++ ) ops [ i ] = XVECEXP ( vals , <NUM_LIT> , i ) ; op0 = gen_reg_rtx ( quarter_mode ) ; op1 = gen_reg_rtx ( quarter_mode ) ; op2 = gen_reg_rtx ( quarter_mode ) ; op3 = gen_reg_rtx ( quarter_mode ) ; op4 = gen_reg_rtx ( half_mode ) ; op5 = gen_reg_rtx ( half_mode ) ; ix86_expand_vector_init_interleave ( quarter_mode , op0 , ops , n >> <NUM_LIT> ) ; ix86_expand_vector_init_interleave ( quarter_mode , op1 , & ops [ n >> <NUM_LIT> ] , n >> <NUM_LIT> ) ; ix86_expand_vector_init_interleave ( quarter_mode , op2 , & ops [ n >> <NUM_LIT> ] , n >> <NUM_LIT> ) ; ix86_expand_vector_init_interleave ( quarter_mode , op3 , & ops [ ( n >> <NUM_LIT> ) | ( n >> <NUM_LIT> ) ] , n >> <NUM_LIT> ) ; emit_insn ( gen_rtx_SET ( op4 , gen_rtx_VEC_CONCAT ( half_mode , op0 , op1 ) ) ) ; emit_insn ( gen_rtx_SET ( op5 , gen_rtx_VEC_CONCAT ( half_mode , op2 , op3 ) ) ) ; emit_insn ( gen_rtx_SET ( target , gen_rtx_VEC_CONCAT ( mode , op4 , op5 ) ) ) ; return ; case E_V16QImode : if ( ! TARGET_SSE4_1 ) break ; case E_V8HImode : if ( ! TARGET_SSE2 ) break ; if ( ! TARGET_INTER_UNIT_MOVES_TO_VEC ) break ; case E_V8HFmode : case E_V8BFmode : n = GET_MODE_NUNITS ( mode ) ; for ( i = <NUM_LIT> ; i < n ; i ++ ) ops [ i ] = XVECEXP ( vals , <NUM_LIT> , i ) ; ix86_expand_vector_init_interleave ( mode , target , ops , n >> <NUM_LIT> ) ; return ; case E_V4HImode : case E_V8QImode : case E_V2HImode : case E_V4QImode : break ; default : gcc_unreachable ( ) ; }


<s> break ; case E_V8HImode : gen_load_even = gen_vec_setv8hi ; gen_interleave_first_low = gen_vec_interleave_lowv4si ; gen_interleave_second_low = gen_vec_interleave_lowv2di ; inner_mode = HImode ; first_imode = V4SImode ; second_imode = V2DImode ; third_imode = VOIDmode ; break ; case E_V16QImode : gen_load_even = gen_vec_setv16qi ; gen_interleave_first_low = gen_vec_interleave_lowv8hi ; gen_interleave_second_low = gen_vec_interleave_lowv4si ; inner_mode = QImode ; first_imode = V8HImode ; second_imode = V4SImode ; third_imode = V2DImode ; break ; default : gcc_unreachable ( ) ; } for ( i = <NUM_LIT> ; i < n ; i ++ ) { op = ops [ i + i ] ; if ( inner_mode == HFmode || inner_mode == BFmode ) { rtx even , odd ; machine_mode vec_mode = ( inner_mode == HFmode ) ? V8HFmode : V8BFmode ; op0 = gen_reg_rtx ( vec_mode ) ; even = lowpart_subreg ( vec_mode , force_reg ( inner_mode , op ) , inner_mode ) ; odd = lowpart_subreg ( vec_mode , force_reg ( inner_mode , ops [ i + i + <NUM_LIT> ] ) , inner_mode ) ; emit_insn ( gen_load_even ( op0 , even , odd ) ) ; } else { op0 = gen_reg_rtx ( SImode ) ; emit_move_insn ( op0 , gen_lowpart ( SImode , op ) ) ; op1 = gen_reg_rtx ( V4SImode ) ; op0 = gen_rtx_VEC_MERGE ( V4SImode , gen_rtx_VEC_DUPLICATE ( V4SImode , op0 ) , CONST0_RTX ( V4SImode ) , const1_rtx ) ; emit_insn ( gen_rtx_SET ( op1 , op0 ) ) ; op0 = gen_reg_rtx ( mode ) ; emit_move_insn ( op0 , gen_lowpart ( mode , op1 ) ) ; emit_insn ( gen_load_even ( op0 , force_reg ( inner_mode , ops [ i + i + <NUM_LIT> ] ) , const1_rtx ) ) ; } ops [ i ] = gen_reg_rtx ( first_imode ) ; emit_move_insn ( ops [ i ] , gen_lowpart ( first_imode , op0 ) ) ; } for ( i = j = <NUM_LIT> ; i < n ; i += <NUM_LIT> , j ++ ) {


<s> gen_interleave_first_low = gen_vec_interleave_lowv8hi ; gen_interleave_second_low = gen_vec_interleave_lowv4si ; inner_mode = QImode ; first_imode = V8HImode ; second_imode = V4SImode ; third_imode = V2DImode ; break ; default : gcc_unreachable ( ) ; } for ( i = <NUM_LIT> ; i < n ; i ++ ) { op = ops [ i + i ] ; if ( inner_mode == HFmode || inner_mode == BFmode ) { rtx even , odd ; machine_mode vec_mode = ( inner_mode == HFmode ) ? V8HFmode : V8BFmode ; op0 = gen_reg_rtx ( vec_mode ) ; even = lowpart_subreg ( vec_mode , force_reg ( inner_mode , op ) , inner_mode ) ; odd = lowpart_subreg ( vec_mode , force_reg ( inner_mode , ops [ i + i + <NUM_LIT> ] ) , inner_mode ) ; emit_insn ( gen_load_even ( op0 , even , odd ) ) ; } else { op0 = gen_reg_rtx ( SImode ) ; emit_move_insn ( op0 , gen_lowpart ( SImode , op ) ) ; op1 = gen_reg_rtx ( V4SImode ) ; op0 = gen_rtx_VEC_MERGE ( V4SImode , gen_rtx_VEC_DUPLICATE ( V4SImode , op0 ) , CONST0_RTX ( V4SImode ) , const1_rtx ) ; emit_insn ( gen_rtx_SET ( op1 , op0 ) ) ; op0 = gen_reg_rtx ( mode ) ; emit_move_insn ( op0 , gen_lowpart ( mode , op1 ) ) ; emit_insn ( gen_load_even ( op0 , force_reg ( inner_mode , ops [ i + i + <NUM_LIT> ] ) , const1_rtx ) ) ; } ops [ i ] = gen_reg_rtx ( first_imode ) ;

emit_move_insn ( ops [ i ] , gen_lowpart ( first_imode , op0 ) ) ;


<s> case E_V8HImode : case E_V4HImode : break ; case E_V16QImode : if ( TARGET_SSE4_1 ) break ; wmode = V8HImode ; goto widen ; case E_V8QImode : if ( TARGET_MMX_WITH_SSE && TARGET_SSE4_1 ) break ; wmode = V4HImode ; goto widen ; case E_V4QImode : if ( TARGET_SSE4_1 ) break ; wmode = V2HImode ; widen : x = XVECEXP ( vals , <NUM_LIT> , one_var ^ <NUM_LIT> ) ; if ( one_var & <NUM_LIT> ) { var = convert_modes ( HImode , QImode , var , true ) ; var = expand_simple_binop ( HImode , ASHIFT , var , GEN_INT ( <NUM_LIT> ) , NULL_RTX , <NUM_LIT> , OPTAB_LIB_WIDEN ) ; x = GEN_INT ( INTVAL ( x ) & <NUM_LIT> ) ; } else { var = convert_modes ( HImode , QImode , var , true ) ; x = gen_int_mode ( UINTVAL ( x ) << <NUM_LIT> , HImode ) ; } if ( x != const0_rtx ) var = expand_simple_binop ( HImode , IOR , var , x , var , <NUM_LIT> , OPTAB_LIB_WIDEN ) ; x = gen_reg_rtx ( wmode ) ; emit_move_insn ( x , gen_lowpart ( wmode , const_vec ) ) ; ix86_expand_vector_set ( mmx_ok , x , var , one_var >> <NUM_LIT> ) ;

emit_move_insn ( target , gen_lowpart ( mode , x ) ) ;


<s> op1 = scratch ; } tmp [ <NUM_LIT> ] = op0 ; tmp [ <NUM_LIT> ] = op1 ; ix86_expand_vector_move_misalign ( mode , tmp ) ; return ; } if ( mode == V1TImode && SUBREG_P ( op1 ) && GET_MODE ( SUBREG_REG ( op1 ) ) == TImode && TARGET_64BIT && TARGET_SSE && can_create_pseudo_p ( ) ) { rtx tmp = gen_reg_rtx ( V2DImode ) ; rtx lo = gen_reg_rtx ( DImode ) ; rtx hi = gen_reg_rtx ( DImode ) ; emit_move_insn ( lo , gen_lowpart ( DImode , SUBREG_REG ( op1 ) ) ) ; emit_move_insn ( hi , gen_highpart ( DImode , SUBREG_REG ( op1 ) ) ) ; emit_insn ( gen_vec_concatv2di ( tmp , lo , hi ) ) ; emit_move_insn ( op0 , gen_lowpart ( V1TImode , tmp ) ) ; return ; } if ( can_create_pseudo_p ( ) && ! ix86_hardreg_mov_ok ( op0 , op1 ) ) { rtx tmp = gen_reg_rtx ( GET_MODE ( op0 ) ) ; emit_move_insn ( tmp , op1 ) ; emit_move_insn ( op0 , tmp ) ; return ; } if ( can_create_pseudo_p ( ) && ! register_operand ( op0 , mode ) && ! register_operand ( op1 , mode ) ) { rtx tmp = gen_reg_rtx ( GET_MODE ( op0 ) ) ; emit_move_insn ( tmp , op1 ) ; emit_move_insn ( op0 , tmp ) ; return ; } emit_insn ( gen_rtx_SET ( op0 , op1 ) ) ;


<s> if ( tmp == nullptr ) op1 = validize_mem ( force_const_mem ( mode , op1 ) ) ; else op1 = tmp ; } } if ( can_create_pseudo_p ( ) && GET_MODE_SIZE ( mode ) >= <NUM_LIT> && VECTOR_MODE_P ( mode ) && ( MEM_P ( op1 ) && SYMBOL_REF_P ( XEXP ( op1 , <NUM_LIT> ) ) && CONSTANT_POOL_ADDRESS_P ( XEXP ( op1 , <NUM_LIT> ) ) ) ) { rtx first = ix86_broadcast_from_constant ( mode , op1 ) ; if ( first != nullptr ) { op1 = gen_reg_rtx ( mode ) ; if ( FLOAT_MODE_P ( mode ) || ( ! TARGET_64BIT && GET_MODE_INNER ( mode ) == DImode ) ) first = force_const_mem ( GET_MODE_INNER ( mode ) , first ) ; bool ok = ix86_expand_vector_init_duplicate ( false , mode , op1 , first ) ; gcc_assert ( ok ) ; emit_move_insn ( op0 , op1 ) ; return ; } } if ( can_create_pseudo_p ( ) && SSE_REG_MODE_P ( mode ) && ( ( MEM_P ( op0 ) && ( MEM_ALIGN ( op0 ) < align ) ) || ( MEM_P ( op1 ) && ( MEM_ALIGN ( op1 ) < align ) ) ) ) { rtx tmp [ <NUM_LIT> ] ; if ( ! register_operand ( op0 , mode ) && ! register_operand ( op1 , mode ) ) { rtx scratch = gen_reg_rtx ( mode ) ; emit_move_insn ( scratch , op1 ) ; op1 = scratch ;


<s> extract_hi = gen_vec_extract_hi_v32hi ; extract_lo = gen_vec_extract_lo_v32hi ; } else if ( mode == V32HFmode ) { half_mode = V16HFmode ; extract_hi = gen_vec_extract_hi_v32hf ; extract_lo = gen_vec_extract_lo_v32hf ; } else if ( mode == V32BFmode ) { half_mode = V16BFmode ; extract_hi = gen_vec_extract_hi_v32bf ; extract_lo = gen_vec_extract_lo_v32bf ; } else { half_mode = V32QImode ; extract_hi = gen_vec_extract_hi_v64qi ; extract_lo = gen_vec_extract_lo_v64qi ; } vhi = gen_reg_rtx ( half_mode ) ; vlo = gen_reg_rtx ( half_mode ) ; idx_hi = gen_reg_rtx ( GET_MODE ( idx ) ) ; emit_insn ( extract_hi ( vhi , target ) ) ; emit_insn ( extract_lo ( vlo , target ) ) ; vec [ <NUM_LIT> ] = idx_hi ; vec [ <NUM_LIT> ] = idx ; vec [ <NUM_LIT> ] = GEN_INT ( n_elts / <NUM_LIT> ) ; ix86_expand_binary_operator ( MINUS , GET_MODE ( idx ) , vec ) ; ix86_expand_vector_set_var ( vhi , val , idx_hi ) ; ix86_expand_vector_set_var ( vlo , val , idx ) ; emit_insn ( gen_rtx_SET ( target , gen_rtx_VEC_CONCAT ( mode , vlo , vhi ) ) ) ; return ; } if ( FLOAT_MODE_P ( GET_MODE_INNER ( mode ) ) ) { switch ( mode ) { case E_V2DFmode : cmp_mode = V2DImode ; break ; case E_V4DFmode : cmp_mode = V4DImode ; break ; case E_V8DFmode : cmp_mode = V8DImode ; break ; case E_V2SFmode : cmp_mode = V2SImode ; break ; case E_V4SFmode : cmp_mode = V4SImode ; break ; case E_V8SFmode : cmp_mode = V8SImode ; break ; case E_V16SFmode : cmp_mode = V16SImode ; break ; case E_V8HFmode : cmp_mode = V8HImode ; break ; case E_V16HFmode : cmp_mode = V16HImode ; break ; case E_V32HFmode : cmp_mode = V32HImode ; break ; case E_V8BFmode : cmp_mode = V8HImode ; break ; case E_V16BFmode : cmp_mode = V16HImode ; break ; case E_V32BFmode : cmp_mode = V32HImode ; break ; default : gcc_unreachable ( ) ;


<s> arg0 = CALL_EXPR_ARG ( exp , <NUM_LIT> ) ; arg1 = CALL_EXPR_ARG ( exp , <NUM_LIT> ) ; arg2 = CALL_EXPR_ARG ( exp , <NUM_LIT> ) ; tmode = TYPE_MODE ( TREE_TYPE ( arg0 ) ) ; mode1 = TYPE_MODE ( TREE_TYPE ( TREE_TYPE ( arg0 ) ) ) ; gcc_assert ( VECTOR_MODE_P ( tmode ) ) ;


<s> xor_constant = <NUM_LIT> << ( <NUM_LIT> - shift_amount - <NUM_LIT> ) ; and_constant = ( code == ASHIFT ? <NUM_LIT> - ( <NUM_LIT> << shift_amount ) : ( <NUM_LIT> << ( <NUM_LIT> - shift_amount ) ) - <NUM_LIT> ) ; switch ( qimode ) { case V16QImode : himode = V8HImode ; gen_shift = ( ( code == ASHIFT ) ? gen_ashlv8hi3 : ( code == ASHIFTRT ) ? gen_ashrv8hi3 : gen_lshrv8hi3 ) ; gen_and = gen_andv16qi3 ; gen_xor = gen_xorv16qi3 ; gen_sub = gen_subv16qi3 ; break ; case V32QImode : himode = V16HImode ; gen_shift = ( ( code == ASHIFT ) ? gen_ashlv16hi3 : ( code == ASHIFTRT ) ? gen_ashrv16hi3 : gen_lshrv16hi3 ) ; gen_and = gen_andv32qi3 ; gen_xor = gen_xorv32qi3 ; gen_sub = gen_subv32qi3 ; break ; case V64QImode : himode = V32HImode ; gen_shift = ( ( code == ASHIFT ) ? gen_ashlv32hi3 : ( code == ASHIFTRT ) ? gen_ashrv32hi3 : gen_lshrv32hi3 ) ; gen_and = gen_andv64qi3 ; gen_xor = gen_xorv64qi3 ; gen_sub = gen_subv64qi3 ; break ; default : gcc_unreachable ( ) ; } tmp = gen_reg_rtx ( himode ) ; vec_const_and = gen_reg_rtx ( qimode ) ; op1_subreg = lowpart_subreg ( himode , op1 , qimode ) ; emit_insn ( gen_shift ( tmp , op1_subreg , op2 ) ) ; emit_move_insn ( dest , simplify_gen_subreg ( qimode , tmp , himode , <NUM_LIT> ) ) ; emit_move_insn ( vec_const_and , ix86_build_const_vector ( qimode , true , gen_int_mode ( and_constant , QImode ) ) ) ; emit_insn ( gen_and ( dest , dest , vec_const_and ) ) ; if ( code == ASHIFTRT ) { vec_const_xor = gen_reg_rtx ( qimode ) ;

emit_move_insn ( vec_const_xor , ix86_build_const_vector ( qimode , true , gen_int_mode ( xor_constant , QImode ) ) ) ;


<s> rtx tmp , op1_subreg ; rtx ( * gen_shift ) ( rtx , rtx , rtx ) ; rtx ( * gen_and ) ( rtx , rtx , rtx ) ; rtx ( * gen_xor ) ( rtx , rtx , rtx ) ; rtx ( * gen_sub ) ( rtx , rtx , rtx ) ; if ( ! CONST_INT_P ( op2 ) ) return false ; qimode = GET_MODE ( dest ) ; shift_amount = INTVAL ( op2 ) ; if ( shift_amount > <NUM_LIT> ) return false ; gcc_assert ( code == ASHIFT || code == ASHIFTRT || code == LSHIFTRT ) ; xor_constant = <NUM_LIT> << ( <NUM_LIT> - shift_amount - <NUM_LIT> ) ; and_constant = ( code == ASHIFT ? <NUM_LIT> - ( <NUM_LIT> << shift_amount ) : ( <NUM_LIT> << ( <NUM_LIT> - shift_amount ) ) - <NUM_LIT> ) ; switch ( qimode ) { case V16QImode : himode = V8HImode ; gen_shift = ( ( code == ASHIFT ) ? gen_ashlv8hi3 : ( code == ASHIFTRT ) ? gen_ashrv8hi3 : gen_lshrv8hi3 ) ; gen_and = gen_andv16qi3 ; gen_xor = gen_xorv16qi3 ; gen_sub = gen_subv16qi3 ; break ; case V32QImode :


<s> rtx t1 = gen_reg_rtx ( SImode ) ; emit_insn ( gen_zero_extendhisi2 ( t1 , op ) ) ; emit_insn ( gen_ashlsi3 ( t1 , t1 , GEN_INT ( <NUM_LIT> ) ) ) ; op = gen_lowpart ( SFmode , t1 ) ; } * pop0 = op ; op = gen_lowpart ( HImode , op1 ) ; if ( CONST_INT_P ( op ) ) op = simplify_const_unary_operation ( FLOAT_EXTEND , SFmode , op1 , BFmode ) ; else { rtx t1 = gen_reg_rtx ( SImode ) ; emit_insn ( gen_zero_extendhisi2 ( t1 , op ) ) ; emit_insn ( gen_ashlsi3 ( t1 , t1 , GEN_INT ( <NUM_LIT> ) ) ) ; op = gen_lowpart ( SFmode , t1 ) ; } * pop1 = op ; return ix86_prepare_fp_compare_args ( code , pop0 , pop1 ) ; } if ( ! is_sse && ( unordered_compare || ( op_mode == XFmode && ! ( standard_80387_constant_p ( op0 ) == <NUM_LIT> || standard_80387_constant_p ( op1 ) == <NUM_LIT> ) && GET_CODE ( op1 ) != FLOAT ) || ix86_fp_comparison_strategy ( code ) == IX86_FPCMP_COMI ) ) { op0 = force_reg ( op_mode , op0 ) ; op1 = force_reg ( op_mode , op1 ) ; } else { if ( standard_80387_constant_p ( op0 ) == <NUM_LIT> || ( MEM_P ( op0 ) && ! ( standard_80387_constant_p ( op1 ) == <NUM_LIT> || MEM_P ( op1 ) ) ) ) { enum rtx_code new_code = ix86_fp_swap_condition ( code ) ; if ( new_code != UNKNOWN ) { std :: swap ( op0 , op1 ) ; code = new_code ; } } if ( ! REG_P ( op0 ) ) op0 = force_reg ( op_mode , op0 ) ; if ( CONSTANT_P ( op1 ) ) { int tmp = standard_80387_constant_p ( op1 ) ; if ( tmp == <NUM_LIT> ) op1 = validize_mem ( force_const_mem ( op_mode , op1 ) ) ; else if ( tmp == <NUM_LIT> ) { if ( TARGET_CMOVE ) op1 = force_reg ( op_mode , op1 ) ; } else op1 = force_reg ( op_mode , op1 ) ; } } if ( ix86_fp_comparison_cost ( code ) > ix86_fp_comparison_cost ( swap_condition ( code ) ) && ( REG_P ( op1 ) || can_create_pseudo_p ( ) ) ) {


<s> if ( ANY_QI_REG_P ( low [ <NUM_LIT> ] ) && ANY_QI_REG_P ( high [ <NUM_LIT> ] ) ) { rtx s , d , flags = gen_rtx_REG ( CCZmode , FLAGS_REG ) ; ix86_expand_clear ( low [ <NUM_LIT> ] ) ; ix86_expand_clear ( high [ <NUM_LIT> ] ) ; emit_insn ( gen_testqi_ccz_1 ( operands [ <NUM_LIT> ] , GEN_INT ( half_width ) ) ) ; d = gen_lowpart ( QImode , low [ <NUM_LIT> ] ) ; d = gen_rtx_STRICT_LOW_PART ( VOIDmode , d ) ; s = gen_rtx_EQ ( QImode , flags , const0_rtx ) ; emit_insn ( gen_rtx_SET ( d , s ) ) ; d = gen_lowpart ( QImode , high [ <NUM_LIT> ] ) ; d = gen_rtx_STRICT_LOW_PART ( VOIDmode , d ) ; s = gen_rtx_NE ( QImode , flags , const0_rtx ) ; emit_insn ( gen_rtx_SET ( d , s ) ) ; } else { rtx ( * gen_lshr3 ) ( rtx , rtx , rtx ) ; rtx ( * gen_and3 ) ( rtx , rtx , rtx ) ; rtx ( * gen_xor3 ) ( rtx , rtx , rtx ) ; HOST_WIDE_INT bits ; rtx x ; if ( mode == DImode ) { gen_lshr3 = gen_lshrsi3 ; gen_and3 = gen_andsi3 ; gen_xor3 = gen_xorsi3 ; bits = <NUM_LIT> ; } else { gen_lshr3 = gen_lshrdi3 ; gen_and3 = gen_anddi3 ; gen_xor3 = gen_xordi3 ; bits = <NUM_LIT> ; } if ( TARGET_PARTIAL_REG_STALL && ! optimize_insn_for_size_p ( ) ) x = gen_rtx_ZERO_EXTEND ( half_mode , operands [ <NUM_LIT> ] ) ; else x = gen_lowpart ( half_mode , operands [ <NUM_LIT> ] ) ; emit_insn ( gen_rtx_SET ( high [ <NUM_LIT> ] , x ) ) ; emit_insn ( gen_lshr3 ( high [ <NUM_LIT> ] , high [ <NUM_LIT> ] , GEN_INT ( bits ) ) ) ; emit_insn ( gen_and3 ( high [ <NUM_LIT> ] , high [ <NUM_LIT> ] , const1_rtx ) ) ; emit_move_insn ( low [ <NUM_LIT> ] , high [ <NUM_LIT> ] ) ; emit_insn ( gen_xor3 ( low [ <NUM_LIT> ] , low [ <NUM_LIT> ] , const1_rtx ) ) ; } emit_insn ( gen_ashl3 ( low [ <NUM_LIT> ] , low [ <NUM_LIT> ] , operands [ <NUM_LIT> ] ) ) ; emit_insn ( gen_ashl3 ( high [ <NUM_LIT> ] , high [ <NUM_LIT> ] , operands [ <NUM_LIT> ] ) ) ; return ; } if ( operands [ <NUM_LIT> ] == constm1_rtx ) {

emit_move_insn ( low [ <NUM_LIT> ] , constm1_rtx ) ;


<s> if ( count > half_width ) ix86_expand_ashl_const ( high [ <NUM_LIT> ] , count - half_width , mode ) ; } else { gen_shld = mode == DImode ? gen_x86_shld : gen_x86_64_shld ; if ( ! rtx_equal_p ( operands [ <NUM_LIT> ] , operands [ <NUM_LIT> ] ) ) emit_move_insn ( operands [ <NUM_LIT> ] , operands [ <NUM_LIT> ] ) ; emit_insn ( gen_shld ( high [ <NUM_LIT> ] , low [ <NUM_LIT> ] , GEN_INT ( count ) ) ) ; ix86_expand_ashl_const ( low [ <NUM_LIT> ] , count , mode ) ; } return ; } split_double_mode ( mode , operands , <NUM_LIT> , low , high ) ; half_mode = mode == DImode ? SImode : DImode ; gen_ashl3 = mode == DImode ? gen_ashlsi3 : gen_ashldi3 ; if ( operands [ <NUM_LIT> ] == const1_rtx ) { if ( ANY_QI_REG_P ( low [ <NUM_LIT> ] ) && ANY_QI_REG_P ( high [ <NUM_LIT> ] ) ) { rtx s , d , flags = gen_rtx_REG ( CCZmode , FLAGS_REG ) ; ix86_expand_clear ( low [ <NUM_LIT> ] ) ; ix86_expand_clear ( high [ <NUM_LIT> ] ) ; emit_insn ( gen_testqi_ccz_1 ( operands [ <NUM_LIT> ] , GEN_INT ( half_width ) ) ) ; d = gen_lowpart ( QImode , low [ <NUM_LIT> ] ) ; d = gen_rtx_STRICT_LOW_PART ( VOIDmode , d ) ; s = gen_rtx_EQ ( QImode , flags , const0_rtx ) ; emit_insn ( gen_rtx_SET ( d , s ) ) ; d = gen_lowpart ( QImode , high [ <NUM_LIT> ] ) ; d = gen_rtx_STRICT_LOW_PART ( VOIDmode , d ) ; s = gen_rtx_NE ( QImode , flags , const0_rtx ) ; emit_insn ( gen_rtx_SET ( d , s ) ) ;


<s> large = operands [ <NUM_LIT> ] ; zero_or_two31 = operands [ <NUM_LIT> ] ; input = operands [ <NUM_LIT> ] ; two31 = operands [ <NUM_LIT> ] ; vecmode = GET_MODE ( large ) ; value = gen_rtx_REG ( vecmode , REGNO ( operands [ <NUM_LIT> ] ) ) ; if ( MEM_P ( input ) ) { if ( vecmode == V4SFmode ) emit_insn ( gen_vec_setv4sf_0 ( value , CONST0_RTX ( V4SFmode ) , input ) ) ; else emit_insn ( gen_sse2_loadlpd ( value , CONST0_RTX ( V2DFmode ) , input ) ) ; } else { input = gen_rtx_REG ( vecmode , REGNO ( input ) ) ;

emit_move_insn ( value , CONST0_RTX ( vecmode ) ) ;


<s> else gen_divmod4_1 = unsigned_p ? gen_udivmodsi4_zext_2 : gen_divmodsi4_zext_2 ; } else gen_divmod4_1 = unsigned_p ? gen_udivmodsi4_zext_1 : gen_divmodsi4_zext_1 ; break ; case E_DImode : gen_divmod4_1 = unsigned_p ? gen_udivmoddi4_1 : gen_divmoddi4_1 ; break ; default : gcc_unreachable ( ) ; } end_label = gen_label_rtx ( ) ; qimode_label = gen_label_rtx ( ) ; scratch = gen_reg_rtx ( mode ) ; emit_move_insn ( scratch , operands [ <NUM_LIT> ] ) ; scratch = expand_simple_binop ( mode , IOR , scratch , operands [ <NUM_LIT> ] , scratch , <NUM_LIT> , OPTAB_DIRECT ) ; emit_insn ( gen_test_ccno_1 ( mode , scratch , GEN_INT ( - <NUM_LIT> ) ) ) ; tmp0 = gen_rtx_REG ( CCNOmode , FLAGS_REG ) ; tmp0 = gen_rtx_EQ ( VOIDmode , tmp0 , const0_rtx ) ; tmp0 = gen_rtx_IF_THEN_ELSE ( VOIDmode , tmp0 , gen_rtx_LABEL_REF ( VOIDmode , qimode_label ) , pc_rtx ) ; insn = emit_jump_insn ( gen_rtx_SET ( pc_rtx , tmp0 ) ) ; predict_jump ( REG_BR_PROB_BASE * <NUM_LIT> / <NUM_LIT> ) ; JUMP_LABEL ( insn ) = qimode_label ; emit_insn ( gen_divmod4_1 ( operands [ <NUM_LIT> ] , operands [ <NUM_LIT> ] , operands [ <NUM_LIT> ] , operands [ <NUM_LIT> ] ) ) ; emit_jump_insn ( gen_jump ( end_label ) ) ; emit_barrier ( ) ;


<s> rtx div , mod ; rtx_insn * insn ; rtx scratch , tmp0 , tmp1 , tmp2 ; rtx ( * gen_divmod4_1 ) ( rtx , rtx , rtx , rtx ) ; switch ( mode ) { case E_SImode : if ( GET_MODE ( operands [ <NUM_LIT> ] ) == SImode ) { if ( GET_MODE ( operands [ <NUM_LIT> ] ) == SImode ) gen_divmod4_1 = unsigned_p ? gen_udivmodsi4_1 : gen_divmodsi4_1 ; else gen_divmod4_1 = unsigned_p ? gen_udivmodsi4_zext_2 : gen_divmodsi4_zext_2 ; } else gen_divmod4_1 = unsigned_p ? gen_udivmodsi4_zext_1 : gen_divmodsi4_zext_1 ; break ; case E_DImode : gen_divmod4_1 = unsigned_p ? gen_udivmoddi4_1 : gen_divmoddi4_1 ; break ; default : gcc_unreachable ( ) ; } end_label = gen_label_rtx ( ) ; qimode_label = gen_label_rtx ( ) ; scratch = gen_reg_rtx ( mode ) ; emit_move_insn ( scratch , operands [ <NUM_LIT> ] ) ; scratch = expand_simple_binop ( mode , IOR , scratch , operands [ <NUM_LIT> ] , scratch , <NUM_LIT> , OPTAB_DIRECT ) ; emit_insn ( gen_test_ccno_1 ( mode , scratch , GEN_INT ( - <NUM_LIT> ) ) ) ; tmp0 = gen_rtx_REG ( CCNOmode , FLAGS_REG ) ; tmp0 = gen_rtx_EQ ( VOIDmode , tmp0 , const0_rtx ) ; tmp0 = gen_rtx_IF_THEN_ELSE ( VOIDmode , tmp0 , gen_rtx_LABEL_REF ( VOIDmode , qimode_label ) , pc_rtx ) ; insn = emit_jump_insn ( gen_rtx_SET ( pc_rtx , tmp0 ) ) ; predict_jump ( REG_BR_PROB_BASE * <NUM_LIT> / <NUM_LIT> ) ; JUMP_LABEL ( insn ) = qimode_label ; emit_insn ( gen_divmod4_1 ( operands [ <NUM_LIT> ] , operands [ <NUM_LIT> ] , operands [ <NUM_LIT> ] , operands [ <NUM_LIT> ] ) ) ; emit_jump_insn ( gen_jump ( end_label ) ) ; emit_barrier ( ) ; emit_label ( qimode_label ) ; tmp0 = lowpart_subreg ( HImode , scratch , mode ) ; tmp1 = lowpart_subreg ( HImode , operands [ <NUM_LIT> ] , mode ) ;

tmp2 = lowpart_subreg ( QImode , operands [ <NUM_LIT> ] , mode ) ;


<s> label = gen_label_rtx ( ) ; rtx tmp = gen_rtx_fmt_ee ( UNORDERED , VOIDmode , flag , const0_rtx ) ; tmp = gen_rtx_IF_THEN_ELSE ( VOIDmode , tmp , gen_rtx_LABEL_REF ( VOIDmode , label ) , pc_rtx ) ; emit_jump_insn ( gen_rtx_SET ( pc_rtx , tmp ) ) ; } if ( GET_MODE ( set_dst ) != mode ) { gcc_assert ( mode == CCAmode || mode == CCCmode || mode == CCOmode || mode == CCPmode || mode == CCSmode || mode == CCZmode ) ;

set_dst = gen_rtx_REG ( mode , FLAGS_REG ) ;


<s> gcc_assert ( ! op_true == ! op_false ) ; if ( ! op_true || ! ix86_valid_mask_cmp_mode ( cmp_mode ) ) return false ; if ( op_false == CONST0_RTX ( mode ) || op_true == CONST0_RTX ( mode ) || ( INTEGRAL_MODE_P ( mode ) && ( op_true == CONSTM1_RTX ( mode ) || op_false == CONSTM1_RTX ( mode ) ) ) ) return false ;


<s> if ( GET_MODE_SIZE ( mode ) == <NUM_LIT> ) return true ; gcc_assert ( ! op_true == ! op_false ) ; if ( ! op_true || ! ix86_valid_mask_cmp_mode ( cmp_mode ) ) return false ; if ( op_false == CONST0_RTX ( mode ) || op_true == CONST0_RTX ( mode ) || ( INTEGRAL_MODE_P ( mode ) && ( op_true == CONSTM1_RTX ( mode ) || op_false == CONSTM1_RTX ( mode ) ) ) ) return false ;


<s> static bool ix86_vector_duplicate_value ( machine_mode mode , rtx target , rtx val ) { bool ok ; rtx_insn * insn ; rtx dup ; recog_data_d recog_data_save = recog_data ; dup = gen_vec_duplicate ( mode , val ) ; insn = emit_insn ( gen_rtx_SET ( target , dup ) ) ; if ( recog_memoized ( insn ) < <NUM_LIT> ) { rtx_insn * seq ; machine_mode innermode = GET_MODE_INNER ( mode ) ; rtx reg ; start_sequence ( ) ; reg = force_reg ( innermode , val ) ; if ( GET_MODE ( reg ) != innermode ) reg = gen_lowpart ( innermode , reg ) ; SET_SRC ( PATTERN ( insn ) ) = gen_vec_duplicate ( mode , reg ) ; seq = get_insns ( ) ; end_sequence ( ) ; if ( seq ) emit_insn_before ( seq , insn ) ; ok = recog_memoized ( insn ) >= <NUM_LIT> ; gcc_assert ( ok ) ; } recog_data = recog_data_save ; return true ;


<s> if ( d -> perm [ <NUM_LIT> ] < nelt ) return ; for ( unsigned i = <NUM_LIT> ; i != nelt ; i ++ ) d -> perm [ i ] = ( d -> perm [ i ] + nelt ) % ( <NUM_LIT> * nelt ) ; std :: swap ( d -> op0 , d -> op1 ) ;


<s> else deleted_move_count ++ ; if ( ! rtx_equal_p ( dhi , hi ) ) emit_move_insn ( dhi , hi ) ; else deleted_move_count ++ ; } else if ( ! rtx_equal_p ( lo , dhi ) ) { if ( ! rtx_equal_p ( dhi , hi ) ) emit_move_insn ( dhi , hi ) ; else deleted_move_count ++ ; if ( ! rtx_equal_p ( dlo , lo ) ) emit_move_insn ( dlo , lo ) ; else deleted_move_count ++ ; } else if ( mode == TImode ) emit_insn ( gen_swapdi ( dlo , dhi ) ) ; else emit_insn ( gen_swapsi ( dlo , dhi ) ) ; if ( deleted_move_count == <NUM_LIT> ) emit_note ( NOTE_INSN_DELETED ) ;


<s> half_mode = HImode ; break ; case E_P2QImode : half_mode = QImode ; break ; default : gcc_unreachable ( ) ; } byte = GET_MODE_SIZE ( half_mode ) ; while ( num -- ) { rtx op = operands [ num ] ; if ( MEM_P ( op ) ) { if ( mem_op && rtx_equal_p ( op , mem_op ) ) { lo_half [ num ] = lo_half [ mem_num ] ; hi_half [ num ] = hi_half [ mem_num ] ; } else { mem_op = op ; mem_num = num ; lo_half [ num ] = adjust_address ( op , half_mode , <NUM_LIT> ) ; hi_half [ num ] = adjust_address ( op , half_mode , byte ) ; }


<s> static bool alg_usable_p ( enum stringop_alg alg , bool memset , bool have_as ) { if ( alg == no_stringop ) return false ; if ( alg == vector_loop ) return TARGET_SSE || TARGET_AVX ;

if ( alg == rep_prefix_1_byte || alg == rep_prefix_4_byte || alg == rep_prefix_8_byte ) {


<s> case E_V16HImode : mode = V8SImode ; break ; case E_V32HImode : mode = V16SImode ; break ; case E_V4SImode : mode = V2DImode ; break ; case E_V8SImode : mode = V4DImode ; break ; case E_V16SImode : mode = V8DImode ; break ; default : return false ; } for ( i = <NUM_LIT> ; i < d -> nelt ; i += <NUM_LIT> ) if ( ( d -> perm [ i ] & <NUM_LIT> ) || d -> perm [ i + <NUM_LIT> ] != d -> perm [ i ] + <NUM_LIT> ) return false ; nd -> vmode = mode ; nd -> nelt = d -> nelt / <NUM_LIT> ; for ( i = <NUM_LIT> ; i < nd -> nelt ; i ++ ) nd -> perm [ i ] = d -> perm [ <NUM_LIT> * i ] / <NUM_LIT> ; if ( GET_MODE_INNER ( mode ) != DImode ) canonicalize_vector_int_perm ( nd , nd ) ; if ( nd != d ) { nd -> one_operand_p = d -> one_operand_p ; nd -> testing_p = d -> testing_p ; if ( d -> op0 == d -> op1 ) nd -> op0 = nd -> op1 = gen_lowpart ( nd -> vmode , d -> op0 ) ; else { nd -> op0 = gen_lowpart ( nd -> vmode , d -> op0 ) ; nd -> op1 = gen_lowpart ( nd -> vmode , d -> op1 ) ; } if ( d -> testing_p ) nd -> target = gen_raw_REG ( nd -> vmode , LAST_VIRTUAL_REGISTER + <NUM_LIT> ) ; else nd -> target = gen_reg_rtx ( nd -> vmode ) ; }


<s> tmp = gen_reg_rtx ( Pmode ) ; unspec = gen_rtx_UNSPEC ( Pmode , gen_rtvec ( <NUM_LIT> , symbol ) , UNSPEC_PLTOFF ) ;

emit_move_insn ( tmp , gen_rtx_CONST ( Pmode , unspec ) ) ;


<s> int i , nunits = GET_MODE_NUNITS ( mode ) ; if ( GET_CODE ( op ) != CONST_VECTOR || nunits != CONST_VECTOR_NUNITS ( op ) ) return false ;

for ( i = <NUM_LIT> ; i < nunits ; i += <NUM_LIT> ) if ( CONST_VECTOR_ELT ( op , i ) != CONST_VECTOR_ELT ( op , i + <NUM_LIT> ) ) return false ;


<s> for ( i = <NUM_LIT> ; i < nunits ; i += <NUM_LIT> ) if ( CONST_VECTOR_ELT ( op , i ) != CONST_VECTOR_ELT ( op , i + <NUM_LIT> ) ) return false ; return true ;


<s> static machine_mode counter_mode ( rtx count_exp ) { if ( GET_MODE ( count_exp ) != VOIDmode ) return GET_MODE ( count_exp ) ; if ( ! CONST_INT_P ( count_exp ) ) return Pmode ;

if ( TARGET_64BIT && ( INTVAL ( count_exp ) & ~ <NUM_LIT> ) ) return DImode ;


<s> static machine_mode counter_mode ( rtx count_exp ) { if ( GET_MODE ( count_exp ) != VOIDmode ) return GET_MODE ( count_exp ) ; if ( ! CONST_INT_P ( count_exp ) ) return Pmode ; if ( TARGET_64BIT && ( INTVAL ( count_exp ) & ~ <NUM_LIT> ) ) return DImode ;


<s> int i ; bool any_alg_usable_p = false ; * noalign = false ; * dynamic_check = - <NUM_LIT> ; if ( optimize_function_for_size_p ( cfun ) || ( optimize_insn_for_size_p ( ) && ( max_size < <NUM_LIT> || ( expected_size != - <NUM_LIT> && expected_size < <NUM_LIT> ) ) ) ) optimize_for_speed = false ; else optimize_for_speed = true ; cost = optimize_for_speed ? ix86_cost : & ix86_size_cost ; if ( memset ) algs = & cost -> memset [ TARGET_64BIT != <NUM_LIT> ] ; else algs = & cost -> memcpy [ TARGET_64BIT != <NUM_LIT> ] ; for ( i = <NUM_LIT> ; i < MAX_STRINGOP_ALGS ; i ++ ) { enum stringop_alg candidate = algs -> size [ i ] . alg ; bool usable = alg_usable_p ( candidate , memset , have_as ) ; any_alg_usable_p |= usable ; if ( candidate != libcall && candidate && usable ) max = algs -> size [ i ] . max ; } if ( ( ( max > <NUM_LIT> && ( unsigned HOST_WIDE_INT ) max >= max_size ) || max == - <NUM_LIT> ) && expected_size == - <NUM_LIT> ) expected_size = min_size / <NUM_LIT> + max_size / <NUM_LIT> ; if ( ix86_stringop_alg != no_stringop && alg_usable_p ( ix86_stringop_alg , memset , have_as ) ) return ix86_stringop_alg ; else if ( ! optimize_for_speed ) { * noalign = true ; if ( ! count || ( count & <NUM_LIT> ) || ( memset && ! zero_memset ) ) return alg_usable_p ( rep_prefix_1_byte , memset , have_as ) ? rep_prefix_1_byte : loop_1_byte ;

else return alg_usable_p ( rep_prefix_4_byte , memset , have_as ) ? rep_prefix_4_byte : loop ;


<s> if ( TARGET_CPU_P ( PENTIUMPRO ) && ( alg == rep_prefix_4_byte || alg == rep_prefix_1_byte ) ) desired_align = <NUM_LIT> ; if ( optimize_size ) desired_align = <NUM_LIT> ; if ( desired_align < align ) desired_align = align ; if ( expected_size != - <NUM_LIT> && expected_size < <NUM_LIT> ) desired_align = align ;


<s> desired_align = GET_MODE_SIZE ( move_mode ) ; if ( TARGET_CPU_P ( PENTIUMPRO ) && ( alg == rep_prefix_4_byte || alg == rep_prefix_1_byte ) ) desired_align = <NUM_LIT> ; if ( optimize_size ) desired_align = <NUM_LIT> ;

if ( desired_align < align ) desired_align = align ;


<s> rtx dst = destmem , src = * srcmem , tempreg ; enum insn_code code ; machine_mode move_mode ; int piece_size , i ; piece_size = <NUM_LIT> << floor_log2 ( size_to_move ) ; while ( ! int_mode_for_size ( piece_size * BITS_PER_UNIT , <NUM_LIT> ) . exists ( & move_mode ) || ( code = optab_handler ( mov_optab , move_mode ) ) == CODE_FOR_nothing ) { gcc_assert ( piece_size > <NUM_LIT> ) ; piece_size >>= <NUM_LIT> ; } if ( GET_MODE_SIZE ( move_mode ) > GET_MODE_SIZE ( word_mode ) ) { int nunits = GET_MODE_SIZE ( move_mode ) / GET_MODE_SIZE ( word_mode ) ; if ( ! mode_for_vector ( word_mode , nunits ) . exists ( & move_mode ) || ( code = optab_handler ( mov_optab , move_mode ) ) == CODE_FOR_nothing ) { move_mode = word_mode ; piece_size = GET_MODE_SIZE ( move_mode ) ; code = optab_handler ( mov_optab , move_mode ) ; }


<s> machine_mode move_mode ; int piece_size , i ; piece_size = <NUM_LIT> << floor_log2 ( size_to_move ) ; while ( ! int_mode_for_size ( piece_size * BITS_PER_UNIT , <NUM_LIT> ) . exists ( & move_mode ) || ( code = optab_handler ( mov_optab , move_mode ) ) == CODE_FOR_nothing ) { gcc_assert ( piece_size > <NUM_LIT> ) ; piece_size >>= <NUM_LIT> ; } if ( GET_MODE_SIZE ( move_mode ) > GET_MODE_SIZE ( word_mode ) ) { int nunits = GET_MODE_SIZE ( move_mode ) / GET_MODE_SIZE ( word_mode ) ; if ( ! mode_for_vector ( word_mode , nunits ) . exists ( & move_mode ) || ( code = optab_handler ( mov_optab , move_mode ) ) == CODE_FOR_nothing ) { move_mode = word_mode ; piece_size = GET_MODE_SIZE ( move_mode ) ; code = optab_handler ( mov_optab , move_mode ) ; } } gcc_assert ( code != CODE_FOR_nothing ) ; dst = adjust_automodify_address_nv ( dst , move_mode , destptr , <NUM_LIT> ) ; src = adjust_automodify_address_nv ( src , move_mode , srcptr , <NUM_LIT> ) ; gcc_assert ( size_to_move % piece_size == <NUM_LIT> ) ;

for ( i = <NUM_LIT> ; i < size_to_move ; i += piece_size ) {


<s> gcc_assert ( code != CODE_FOR_nothing && promoted_val != NULL_RTX ) ; dst = adjust_automodify_address_nv ( dst , move_mode , destptr , <NUM_LIT> ) ; gcc_assert ( size_to_move % piece_size == <NUM_LIT> ) ; for ( i = <NUM_LIT> ; i < size_to_move ; i += piece_size ) { if ( piece_size <= GET_MODE_SIZE ( word_mode ) ) { emit_insn ( gen_strset ( destptr , dst , promoted_val ) ) ; dst = adjust_automodify_address_nv ( dst , move_mode , destptr , piece_size ) ; continue ; } emit_insn ( GEN_FCN ( code ) ( dst , promoted_val ) ) ; emit_move_insn ( destptr , plus_constant ( Pmode , copy_rtx ( destptr ) , piece_size ) ) ; dst = adjust_automodify_address_nv ( dst , move_mode , destptr , piece_size ) ; } return dst ;


<s> rtx src , dest ; if ( CONST_INT_P ( count ) ) { HOST_WIDE_INT countval = INTVAL ( count ) ; HOST_WIDE_INT epilogue_size = countval % max_size ; int i ; gcc_assert ( ( max_size & ( max_size - <NUM_LIT> ) ) == <NUM_LIT> ) ; for ( i = max_size ; i >= <NUM_LIT> ; i >>= <NUM_LIT> ) { if ( epilogue_size & i ) destmem = emit_memmov ( destmem , & srcmem , destptr , srcptr , i ) ; } return ; } if ( max_size > <NUM_LIT> ) { count = expand_simple_binop ( GET_MODE ( count ) , AND , count , GEN_INT ( max_size - <NUM_LIT> ) , count , <NUM_LIT> , OPTAB_DIRECT ) ; expand_set_or_cpymem_via_loop ( destmem , srcmem , destptr , srcptr , NULL , count , QImode , <NUM_LIT> , <NUM_LIT> , false ) ; return ; } if ( TARGET_SINGLE_STRINGOP ) { if ( max_size > <NUM_LIT> ) { rtx_code_label * label = ix86_expand_aligntest ( count , <NUM_LIT> , true ) ; src = change_address ( srcmem , SImode , srcptr ) ; dest = change_address ( destmem , SImode , destptr ) ; emit_insn ( gen_strmov ( destptr , dest , srcptr , src ) ) ; emit_label ( label ) ; LABEL_NUSES ( label ) = <NUM_LIT> ; } if ( max_size > <NUM_LIT> ) { rtx_code_label * label = ix86_expand_aligntest ( count , <NUM_LIT> , true ) ; src = change_address ( srcmem , HImode , srcptr ) ; dest = change_address ( destmem , HImode , destptr ) ; emit_insn ( gen_strmov ( destptr , dest , srcptr , src ) ) ; emit_label ( label ) ; LABEL_NUSES ( label ) = <NUM_LIT> ; } if ( max_size > <NUM_LIT> ) { rtx_code_label * label = ix86_expand_aligntest ( count , <NUM_LIT> , true ) ; src = change_address ( srcmem , QImode , srcptr ) ; dest = change_address ( destmem , QImode , destptr ) ; emit_insn ( gen_strmov ( destptr , dest , srcptr , src ) ) ; emit_label ( label ) ; LABEL_NUSES ( label ) = <NUM_LIT> ; } } else { rtx offset = force_reg ( Pmode , const0_rtx ) ; rtx tmp ; if ( max_size > <NUM_LIT> ) { rtx_code_label * label = ix86_expand_aligntest ( count , <NUM_LIT> , true ) ; src = change_address ( srcmem , SImode , srcptr ) ; dest = change_address ( destmem , SImode , destptr ) ; emit_move_insn ( dest , src ) ; tmp = expand_simple_binop ( Pmode , PLUS , offset , GEN_INT ( <NUM_LIT> ) , NULL , true , OPTAB_LIB_WIDEN ) ; if ( tmp != offset ) emit_move_insn ( offset , tmp ) ; emit_label ( label ) ;


<s> HOST_WIDE_INT countval = INTVAL ( count ) ; HOST_WIDE_INT epilogue_size = countval % max_size ; int i ; gcc_assert ( ( max_size & ( max_size - <NUM_LIT> ) ) == <NUM_LIT> ) ; for ( i = max_size ; i >= <NUM_LIT> ; i >>= <NUM_LIT> ) { if ( epilogue_size & i ) { if ( vec_value && i > GET_MODE_SIZE ( GET_MODE ( value ) ) ) destmem = emit_memset ( destmem , destptr , vec_value , i ) ; else destmem = emit_memset ( destmem , destptr , value , i ) ; } } return ; } if ( max_size > <NUM_LIT> ) { expand_setmem_epilogue_via_loop ( destmem , destptr , value , count , max_size ) ; return ; } if ( max_size > <NUM_LIT> ) { rtx_code_label * label = ix86_expand_aligntest ( count , <NUM_LIT> , true ) ; if ( TARGET_64BIT ) { dest = change_address ( destmem , DImode , destptr ) ; emit_insn ( gen_strset ( destptr , dest , value ) ) ; dest = adjust_automodify_address_nv ( dest , DImode , destptr , <NUM_LIT> ) ; emit_insn ( gen_strset ( destptr , dest , value ) ) ; } else { dest = change_address ( destmem , SImode , destptr ) ; emit_insn ( gen_strset ( destptr , dest , value ) ) ; dest = adjust_automodify_address_nv ( dest , SImode , destptr , <NUM_LIT> ) ;


<s> static void expand_setmem_epilogue_via_loop ( rtx destmem , rtx destptr , rtx value , rtx count , int max_size ) { count = expand_simple_binop ( counter_mode ( count ) , AND , count , GEN_INT ( max_size - <NUM_LIT> ) , count , <NUM_LIT> , OPTAB_DIRECT ) ; expand_set_or_cpymem_via_loop ( destmem , NULL , destptr , NULL , gen_lowpart ( QImode , value ) , count , QImode , <NUM_LIT> , max_size / <NUM_LIT> , true ) ;


<s> gcc_assert ( srcp != NULL ) ; src = * srcp ; orig_src = src ; } for ( piece_size = <NUM_LIT> ; piece_size <= desired_align && copied_bytes < align_bytes ; piece_size <<= <NUM_LIT> ) { if ( align_bytes & piece_size ) { if ( issetmem ) { if ( vec_value && piece_size > GET_MODE_SIZE ( GET_MODE ( value ) ) ) dst = emit_memset ( dst , destreg , vec_value , piece_size ) ; else dst = emit_memset ( dst , destreg , value , piece_size ) ; } else dst = emit_memmov ( dst , & src , destreg , srcreg , piece_size ) ; copied_bytes += piece_size ; } } if ( MEM_ALIGN ( dst ) < ( unsigned int ) desired_align * BITS_PER_UNIT ) set_mem_align ( dst , desired_align * BITS_PER_UNIT ) ; if ( MEM_SIZE_KNOWN_P ( orig_dst ) ) set_mem_size ( dst , MEM_SIZE ( orig_dst ) - align_bytes ) ; if ( ! issetmem ) { int src_align_bytes = get_mem_align_offset ( src , desired_align * BITS_PER_UNIT ) ; if ( src_align_bytes >= <NUM_LIT> ) src_align_bytes = desired_align - src_align_bytes ; if ( src_align_bytes >= <NUM_LIT> ) { unsigned int src_align ; for ( src_align = desired_align ; src_align >= <NUM_LIT> ; src_align >>= <NUM_LIT> ) { if ( ( src_align_bytes & ( src_align - <NUM_LIT> ) ) == ( align_bytes & ( src_align - <NUM_LIT> ) ) ) break ;


<s> static rtx expand_set_or_cpymem_prologue ( rtx destmem , rtx srcmem , rtx destptr , rtx srcptr , rtx value , rtx vec_value , rtx count , int align , int desired_alignment , bool issetmem ) { int i ; for ( i = <NUM_LIT> ; i < desired_alignment ; i <<= <NUM_LIT> ) { if ( align <= i ) { rtx_code_label * label = ix86_expand_aligntest ( destptr , i , false ) ; if ( issetmem ) { if ( vec_value && i > GET_MODE_SIZE ( GET_MODE ( value ) ) ) destmem = emit_memset ( destmem , destptr , vec_value , i ) ; else destmem = emit_memset ( destmem , destptr , value , i ) ; } else destmem = emit_memmov ( destmem , & srcmem , destptr , srcptr , i ) ; ix86_adjust_counter ( count , i ) ; emit_label ( label ) ;


<s> if ( ! * done_label ) * done_label = gen_label_rtx ( ) ; emit_cmp_and_jump_insns ( * count , GEN_INT ( size2 ) , GE , <NUM_LIT> , GET_MODE ( * count ) , <NUM_LIT> , loop_label ) ; size2 >>= <NUM_LIT> ; for ( ; size2 > <NUM_LIT> ; size2 >>= <NUM_LIT> ) expand_small_cpymem_or_setmem ( destmem , srcmem , * destptr , * srcptr , value , vec_value , * count , size2 , * done_label , issetmem ) ; emit_cmp_and_jump_insns ( * count , const0_rtx , EQ , <NUM_LIT> , GET_MODE ( * count ) , <NUM_LIT> , * done_label ) ; destmem = change_address ( destmem , QImode , * destptr ) ; if ( issetmem ) emit_move_insn ( destmem , gen_lowpart ( QImode , value ) ) ; else { srcmem = change_address ( srcmem , QImode , * srcptr ) ; emit_move_insn ( destmem , srcmem ) ; } label = ix86_expand_aligntest ( * count , <NUM_LIT> , false ) ; destmem = change_address ( destmem , HImode , * destptr ) ; destmem = offset_address ( destmem , * count , <NUM_LIT> ) ; destmem = offset_address ( destmem , GEN_INT ( - <NUM_LIT> ) , <NUM_LIT> ) ; if ( issetmem ) emit_move_insn ( destmem , gen_lowpart ( HImode , value ) ) ; else { srcmem = change_address ( srcmem , HImode , * srcptr ) ; srcmem = offset_address ( srcmem , * count , <NUM_LIT> ) ; srcmem = offset_address ( srcmem , GEN_INT ( - <NUM_LIT> ) , <NUM_LIT> ) ; emit_move_insn ( destmem , srcmem ) ; } emit_label ( label ) ; LABEL_NUSES ( label ) = <NUM_LIT> ; emit_jump_insn ( gen_jump ( * done_label ) ) ; emit_barrier ( ) ; } else gcc_assert ( * min_size >= ( unsigned HOST_WIDE_INT ) size || UINTVAL ( * count ) >= ( unsigned HOST_WIDE_INT ) size ) ; if ( loop_label ) { emit_label ( loop_label ) ; LABEL_NUSES ( loop_label ) = <NUM_LIT> ; } if ( ! issetmem ) srcmem = change_address ( srcmem , mode , * srcptr ) ; destmem = change_address ( destmem , mode , * destptr ) ; modesize = GEN_INT ( GET_MODE_SIZE ( mode ) ) ; for ( n = <NUM_LIT> ; prolog_size < desired_align - align ; n ++ ) { if ( issetmem ) emit_move_insn ( destmem , mode_value ) ; else { emit_move_insn ( destmem , srcmem ) ; srcmem = offset_address ( srcmem , modesize , GET_MODE_SIZE ( mode ) ) ; } destmem = offset_address ( destmem , modesize , GET_MODE_SIZE ( mode ) ) ; prolog_size += GET_MODE_SIZE ( mode ) ; } destmem = offset_address ( destmem , * count , <NUM_LIT> ) ; destmem = offset_address ( destmem , GEN_INT ( - size - prolog_size ) , <NUM_LIT> ) ; if ( issetmem ) emit_move_insn ( destmem , mode_value ) ;


<s> if ( ! issetmem ) { srcmem = offset_address ( srcmem , copy_rtx ( tmp ) , piece_size_n ) ; srcmem = adjust_address ( srcmem , mode , <NUM_LIT> ) ; if ( ! TARGET_64BIT && <NUM_LIT> ) { for ( i = <NUM_LIT> ; i < unroll ; i ++ ) { if ( i ) { destmem = adjust_address ( copy_rtx ( destmem ) , mode , GET_MODE_SIZE ( mode ) ) ; srcmem = adjust_address ( copy_rtx ( srcmem ) , mode , GET_MODE_SIZE ( mode ) ) ; } emit_move_insn ( destmem , srcmem ) ; } } else { rtx tmpreg [ <NUM_LIT> ] ; gcc_assert ( unroll <= <NUM_LIT> ) ; for ( i = <NUM_LIT> ; i < unroll ; i ++ ) { tmpreg [ i ] = gen_reg_rtx ( mode ) ; if ( i ) srcmem = adjust_address ( copy_rtx ( srcmem ) , mode , GET_MODE_SIZE ( mode ) ) ; emit_move_insn ( tmpreg [ i ] , srcmem ) ; } for ( i = <NUM_LIT> ; i < unroll ; i ++ ) { if ( i ) destmem = adjust_address ( copy_rtx ( destmem ) , mode , GET_MODE_SIZE ( mode ) ) ; emit_move_insn ( destmem , tmpreg [ i ] ) ; } } } else for ( i = <NUM_LIT> ; i < unroll ; i ++ ) { if ( i ) destmem = adjust_address ( copy_rtx ( destmem ) , mode , GET_MODE_SIZE ( mode ) ) ; emit_move_insn ( destmem , value ) ; } tmp = expand_simple_binop ( iter_mode , PLUS , iter , piece_size , iter , true , OPTAB_LIB_WIDEN ) ; if ( tmp != iter ) emit_move_insn ( iter , tmp ) ; emit_cmp_and_jump_insns ( iter , size , LT , NULL_RTX , iter_mode , true , top_label ) ; if ( expected_size != - <NUM_LIT> ) { expected_size /= GET_MODE_SIZE ( mode ) * unroll ; if ( expected_size == <NUM_LIT> ) predict_jump ( <NUM_LIT> ) ; else if ( expected_size > REG_BR_PROB_BASE ) predict_jump ( REG_BR_PROB_BASE - <NUM_LIT> ) ; else predict_jump ( REG_BR_PROB_BASE - ( REG_BR_PROB_BASE + expected_size / <NUM_LIT> ) / expected_size ) ; } else predict_jump ( REG_BR_PROB_BASE * <NUM_LIT> / <NUM_LIT> ) ; iter = ix86_zero_extend_to_Pmode ( iter ) ; tmp = expand_simple_binop ( Pmode , PLUS , destptr , iter , destptr , true , OPTAB_LIB_WIDEN ) ; if ( tmp != destptr ) emit_move_insn ( destptr , tmp ) ; if ( ! issetmem ) {

tmp = expand_simple_binop ( Pmode , PLUS , srcptr , iter , srcptr , true , OPTAB_LIB_WIDEN ) ;


<s> for ( i = <NUM_LIT> ; i < unroll ; i ++ ) { if ( i ) { destmem = adjust_address ( copy_rtx ( destmem ) , mode , GET_MODE_SIZE ( mode ) ) ; srcmem = adjust_address ( copy_rtx ( srcmem ) , mode , GET_MODE_SIZE ( mode ) ) ; } emit_move_insn ( destmem , srcmem ) ; } } else { rtx tmpreg [ <NUM_LIT> ] ; gcc_assert ( unroll <= <NUM_LIT> ) ; for ( i = <NUM_LIT> ; i < unroll ; i ++ ) { tmpreg [ i ] = gen_reg_rtx ( mode ) ; if ( i ) srcmem = adjust_address ( copy_rtx ( srcmem ) , mode , GET_MODE_SIZE ( mode ) ) ; emit_move_insn ( tmpreg [ i ] , srcmem ) ; } for ( i = <NUM_LIT> ; i < unroll ; i ++ ) { if ( i ) destmem = adjust_address ( copy_rtx ( destmem ) , mode , GET_MODE_SIZE ( mode ) ) ; emit_move_insn ( destmem , tmpreg [ i ] ) ; } } } else for ( i = <NUM_LIT> ; i < unroll ; i ++ ) { if ( i ) destmem = adjust_address ( copy_rtx ( destmem ) , mode , GET_MODE_SIZE ( mode ) ) ; emit_move_insn ( destmem , value ) ; } tmp = expand_simple_binop ( iter_mode , PLUS , iter , piece_size , iter , true , OPTAB_LIB_WIDEN ) ; if ( tmp != iter ) emit_move_insn ( iter , tmp ) ;

emit_cmp_and_jump_insns ( iter , size , LT , NULL_RTX , iter_mode , true , top_label ) ;


<s> rtx srcexp ; rtx countreg ; HOST_WIDE_INT rounded_count ; if ( mode == QImode && CONST_INT_P ( count ) && ! ( INTVAL ( count ) & <NUM_LIT> ) && ! TARGET_PREFER_KNOWN_REP_MOVSB_STOSB && ( ! issetmem || orig_value == const0_rtx ) ) mode = SImode ; if ( destptr != XEXP ( destmem , <NUM_LIT> ) || GET_MODE ( destmem ) != BLKmode ) destmem = adjust_automodify_address_nv ( destmem , BLKmode , destptr , <NUM_LIT> ) ; countreg = ix86_zero_extend_to_Pmode ( scale_counter ( count , GET_MODE_SIZE ( mode ) ) ) ; if ( mode != QImode ) { destexp = gen_rtx_ASHIFT ( Pmode , countreg , GEN_INT ( exact_log2 ( GET_MODE_SIZE ( mode ) ) ) ) ; destexp = gen_rtx_PLUS ( Pmode , destexp , destptr ) ; } else destexp = gen_rtx_PLUS ( Pmode , destptr , countreg ) ; if ( ( ! issetmem || orig_value == const0_rtx ) && CONST_INT_P ( count ) ) { rounded_count = ROUND_DOWN ( INTVAL ( count ) , ( HOST_WIDE_INT ) GET_MODE_SIZE ( mode ) ) ; destmem = shallow_copy_rtx ( destmem ) ; set_mem_size ( destmem , rounded_count ) ; } else if ( MEM_SIZE_KNOWN_P ( destmem ) ) clear_mem_size ( destmem ) ; if ( issetmem ) { value = force_reg ( mode , gen_lowpart ( mode , value ) ) ; emit_insn ( gen_rep_stos ( destptr , countreg , destmem , value , destexp ) ) ; } else { if ( srcptr != XEXP ( srcmem , <NUM_LIT> ) || GET_MODE ( srcmem ) != BLKmode ) srcmem = adjust_automodify_address_nv ( srcmem , BLKmode , srcptr , <NUM_LIT> ) ; if ( mode != QImode ) { srcexp = gen_rtx_ASHIFT ( Pmode , countreg , GEN_INT ( exact_log2 ( GET_MODE_SIZE ( mode ) ) ) ) ;

srcexp = gen_rtx_PLUS ( Pmode , srcexp , srcptr ) ;


<s> } else { if ( size >= <NUM_LIT> ) mode = TARGET_AVX ? V32QImode : TARGET_SSE ? V16QImode : DImode ; else if ( size >= <NUM_LIT> ) mode = TARGET_SSE ? V16QImode : DImode ; srcmem = change_address ( srcmem , mode , srcptr ) ; } destmem = change_address ( destmem , mode , destptr ) ; modesize = GEN_INT ( GET_MODE_SIZE ( mode ) ) ; gcc_assert ( GET_MODE_SIZE ( mode ) <= size ) ; for ( n = <NUM_LIT> ; n * GET_MODE_SIZE ( mode ) < size ; n ++ ) { if ( issetmem ) emit_move_insn ( destmem , gen_lowpart ( mode , value ) ) ; else { emit_move_insn ( destmem , srcmem ) ; srcmem = offset_address ( srcmem , modesize , GET_MODE_SIZE ( mode ) ) ; } destmem = offset_address ( destmem , modesize , GET_MODE_SIZE ( mode ) ) ; } destmem = offset_address ( destmem , count , <NUM_LIT> ) ; destmem = offset_address ( destmem , GEN_INT ( - <NUM_LIT> * size ) , GET_MODE_SIZE ( mode ) ) ; if ( ! issetmem ) { srcmem = offset_address ( srcmem , count , <NUM_LIT> ) ; srcmem = offset_address ( srcmem , GEN_INT ( - <NUM_LIT> * size ) , GET_MODE_SIZE ( mode ) ) ; } for ( n = <NUM_LIT> ; n * GET_MODE_SIZE ( mode ) < size ; n ++ ) { if ( issetmem ) emit_move_insn ( destmem , gen_lowpart ( mode , value ) ) ; else { emit_move_insn ( destmem , srcmem ) ; srcmem = offset_address ( srcmem , modesize , GET_MODE_SIZE ( mode ) ) ; } destmem = offset_address ( destmem , modesize , GET_MODE_SIZE ( mode ) ) ; } emit_jump_insn ( gen_jump ( done_label ) ) ; emit_barrier ( ) ; emit_label ( label ) ; LABEL_NUSES ( label ) = <NUM_LIT> ;


<s> case E_V8HImode : gen = gen_avx2_pbroadcastv8hi ; break ; case E_V16SFmode : if ( TARGET_AVX512F ) gen = gen_avx512f_vec_dupv16sf_1 ; break ; case E_V8SFmode : gen = gen_avx2_vec_dupv8sf_1 ; break ; case E_V8DFmode : if ( TARGET_AVX512F ) gen = gen_avx512f_vec_dupv8df_1 ; break ; case E_V8DImode : if ( TARGET_AVX512F ) gen = gen_avx512f_vec_dupv8di_1 ; break ; default : break ; } if ( gen != NULL ) { if ( ! d -> testing_p ) emit_insn ( gen ( d -> target , d -> op0 ) ) ; return true ; } } if ( expand_vselect ( d -> target , d -> op0 , nd . perm , nelt , d -> testing_p ) ) return true ; for ( i = <NUM_LIT> ; i < nelt ; i += <NUM_LIT> ) { nd . perm [ i ] = d -> perm [ i ] & mask ; nd . perm [ i + <NUM_LIT> ] = ( d -> perm [ i + <NUM_LIT> ] & mask ) + nelt ; } if ( expand_vselect_vconcat ( d -> target , d -> op0 , d -> op0 , nd . perm , nelt , d -> testing_p ) ) return true ; if ( nelt >= <NUM_LIT> ) { for ( i = <NUM_LIT> ; i < nelt ; i += <NUM_LIT> ) { nd . perm [ i + <NUM_LIT> ] = d -> perm [ i + <NUM_LIT> ] & mask ; nd . perm [ i + <NUM_LIT> ] = d -> perm [ i + <NUM_LIT> ] & mask ; nd . perm [ i + <NUM_LIT> ] = ( d -> perm [ i + <NUM_LIT> ] & mask ) + nelt ; nd . perm [ i + <NUM_LIT> ] = ( d -> perm [ i + <NUM_LIT> ] & mask ) + nelt ; }


<s> dsecond . one_operand_p = true ; for ( i = <NUM_LIT> ; i < nelt ; i ++ ) if ( d -> perm [ i ] >= nelt ) { dsecond . perm [ i / <NUM_LIT> + ( i >= lane ? lane / <NUM_LIT> : <NUM_LIT> ) ] = d -> perm [ i ] - nelt ; if ( d -> perm [ i ] - nelt != i / <NUM_LIT> + ( i >= lane ? lane / <NUM_LIT> : <NUM_LIT> ) ) ident2 = false ; dsecond . perm [ i / <NUM_LIT> + ( i >= lane ? lane : lane / <NUM_LIT> ) ] = d -> perm [ i ] - nelt ; } else { dfirst . perm [ i / <NUM_LIT> + ( i >= lane ? lane / <NUM_LIT> : <NUM_LIT> ) ] = d -> perm [ i ] ; if ( d -> perm [ i ] != i / <NUM_LIT> + ( i >= lane ? lane / <NUM_LIT> : <NUM_LIT> ) ) ident1 = false ; dfirst . perm [ i / <NUM_LIT> + ( i >= lane ? lane : lane / <NUM_LIT> ) ] = d -> perm [ i ] ; } if ( two_insn && ! ident1 && ! ident2 ) return false ; if ( ! d -> testing_p ) { if ( ! ident1 ) dfinal . op0 = dfirst . target = gen_reg_rtx ( d -> vmode ) ; if ( ! ident2 ) dfinal . op1 = dsecond . target = gen_reg_rtx ( d -> vmode ) ; if ( d -> perm [ <NUM_LIT> ] >= nelt ) std :: swap ( dfinal . op0 , dfinal . op1 ) ; } bool ok ; rtx_insn * seq1 = NULL , * seq2 = NULL ; if ( ! ident1 ) { start_sequence ( ) ; ok = expand_vec_perm_1 ( & dfirst ) ; seq1 = get_insns ( ) ; end_sequence ( ) ; if ( ! ok ) return false ; } if ( ! ident2 ) { start_sequence ( ) ; ok = expand_vec_perm_1 ( & dsecond ) ; seq2 = get_insns ( ) ; end_sequence ( ) ; if ( ! ok ) return false ; } if ( d -> testing_p ) return true ; for ( i = <NUM_LIT> ; i < nelt ; i ++ ) {


<s> if ( d -> perm [ i ] != i + nelt ) ident2 = false ; } else { dsecond . perm [ i ] = <NUM_LIT> ; dfirst . perm [ i ] = d -> perm [ i ] ; if ( d -> perm [ i ] != i ) ident1 = false ; } if ( two_insn && ! ident1 && ! ident2 ) return false ; for ( i = <NUM_LIT> ; i < nelt ; ++ i ) if ( dfirst . perm [ i ] == <NUM_LIT> ) { if ( GET_MODE_SIZE ( vmode ) == <NUM_LIT> && dfirst . perm [ i ^ ( nelt / <NUM_LIT> ) ] != <NUM_LIT> ) dfirst . perm [ i ] = dfirst . perm [ i ^ ( nelt / <NUM_LIT> ) ] ^ ( nelt / <NUM_LIT> ) ; else dfirst . perm [ i ] = i ; } else { if ( GET_MODE_SIZE ( vmode ) == <NUM_LIT> && dsecond . perm [ i ^ ( nelt / <NUM_LIT> ) ] != <NUM_LIT> ) dsecond . perm [ i ] = dsecond . perm [ i ^ ( nelt / <NUM_LIT> ) ] ^ ( nelt / <NUM_LIT> ) ; else dsecond . perm [ i ] = i ; } if ( ! d -> testing_p ) { if ( ! ident1 ) dfinal . op0 = dfirst . target = gen_reg_rtx ( d -> vmode ) ; if ( ! ident2 ) dfinal . op1 = dsecond . target = gen_reg_rtx ( d -> vmode ) ; } bool ok ; rtx_insn * seq1 = NULL , * seq2 = NULL ; if ( ! ident1 ) { start_sequence ( ) ; ok = expand_vec_perm_1 ( & dfirst ) ; seq1 = get_insns ( ) ; end_sequence ( ) ; if ( ! ok ) return false ; } if ( ! ident2 ) { start_sequence ( ) ; ok = expand_vec_perm_1 ( & dsecond ) ; seq2 = get_insns ( ) ; end_sequence ( ) ; if ( ! ok ) return false ; } if ( d -> testing_p ) return true ; for ( i = <NUM_LIT> ; i < nelt ; ++ i ) dfinal . perm [ i ] = ( d -> perm [ i ] >= nelt ? i + nelt : i ) ;


<s> if ( d -> testing_p ) return true ; dfirst = * d ; dsecond = * d ; dthird = * d ; dfirst . perm [ <NUM_LIT> ] = ( d -> perm [ <NUM_LIT> ] & ~ <NUM_LIT> ) ; dfirst . perm [ <NUM_LIT> ] = ( d -> perm [ <NUM_LIT> ] & ~ <NUM_LIT> ) + <NUM_LIT> ; dfirst . perm [ <NUM_LIT> ] = ( d -> perm [ <NUM_LIT> ] & ~ <NUM_LIT> ) ; dfirst . perm [ <NUM_LIT> ] = ( d -> perm [ <NUM_LIT> ] & ~ <NUM_LIT> ) + <NUM_LIT> ; dsecond . perm [ <NUM_LIT> ] = ( d -> perm [ <NUM_LIT> ] & ~ <NUM_LIT> ) ; dsecond . perm [ <NUM_LIT> ] = ( d -> perm [ <NUM_LIT> ] & ~ <NUM_LIT> ) + <NUM_LIT> ; dsecond . perm [ <NUM_LIT> ] = ( d -> perm [ <NUM_LIT> ] & ~ <NUM_LIT> ) ; dsecond . perm [ <NUM_LIT> ] = ( d -> perm [ <NUM_LIT> ] & ~ <NUM_LIT> ) + <NUM_LIT> ; dthird . perm [ <NUM_LIT> ] = ( d -> perm [ <NUM_LIT> ] % <NUM_LIT> ) ; dthird . perm [ <NUM_LIT> ] = ( d -> perm [ <NUM_LIT> ] % <NUM_LIT> ) + <NUM_LIT> ;


<s> static bool expand_vec_perm_broadcast ( struct expand_vec_perm_d * d ) { unsigned i , elt , nelt = d -> nelt ; if ( ! d -> one_operand_p ) return false ; elt = d -> perm [ <NUM_LIT> ] ; for ( i = <NUM_LIT> ; i < nelt ; ++ i ) if ( d -> perm [ i ] != elt ) return false ; return expand_vec_perm_broadcast_1 ( d ) ;


<s> for ( i = <NUM_LIT> ; i < nelt ; ++ i ) if ( d -> perm [ i ] != elt ) return false ;

return expand_vec_perm_broadcast_1 ( d ) ;


<s> emit_insn ( gen ( dest , op0 , op0 ) ) ; vmode = get_mode_wider_vector ( vmode ) ; op0 = gen_lowpart ( vmode , dest ) ; memset ( perm2 , elt , <NUM_LIT> ) ; dest = gen_reg_rtx ( vmode ) ; ok = expand_vselect ( dest , op0 , perm2 , <NUM_LIT> , d -> testing_p ) ; gcc_assert ( ok ) ; emit_move_insn ( d -> target , gen_lowpart ( d -> vmode , dest ) ) ; return true ; case E_V8QImode : if ( d -> testing_p ) return true ; do { if ( elt >= nelt2 ) { gen = vmode == V8QImode ? gen_mmx_punpckhbw : gen_mmx_punpckhwd ; elt -= nelt2 ; } else gen = vmode == V8QImode ? gen_mmx_punpcklbw : gen_mmx_punpcklwd ; nelt2 /= <NUM_LIT> ; dest = gen_reg_rtx ( vmode ) ; emit_insn ( gen ( dest , op0 , op0 ) ) ; vmode = get_mode_wider_vector ( vmode ) ; op0 = gen_lowpart ( vmode , dest ) ; } while ( vmode != V2SImode ) ; memset ( perm2 , elt , <NUM_LIT> ) ; dest = gen_reg_rtx ( vmode ) ; ok = expand_vselect ( dest , op0 , perm2 , <NUM_LIT> , d -> testing_p ) ; gcc_assert ( ok ) ; emit_move_insn ( d -> target , gen_lowpart ( d -> vmode , dest ) ) ; return true ; case E_V8HImode : case E_V16QImode : if ( d -> testing_p ) return true ;


<s> odd = d -> perm [ <NUM_LIT> ] ; if ( odd != <NUM_LIT> && odd != <NUM_LIT> ) return false ; for ( i = <NUM_LIT> ; i < nelt ; ++ i ) if ( d -> perm [ i ] != <NUM_LIT> * i + odd ) return false ;

if ( d -> vmode == E_V32HImode && d -> testing_p && ! TARGET_AVX512BW ) return false ;


<s> for ( i = <NUM_LIT> ; i < nelt ; ++ i ) if ( d -> perm [ i ] != <NUM_LIT> * i + odd ) return false ; if ( d -> vmode == E_V32HImode && d -> testing_p && ! TARGET_AVX512BW ) return false ;

return expand_vec_perm_even_odd_1 ( d , odd ) ;


<s> gen_shift = gen_lshrv2si3 ; break ; case E_V8HImode : if ( ! TARGET_SSE4_1 ) return false ; c = <NUM_LIT> ; s = <NUM_LIT> ; half_mode = V4SImode ; gen_and = gen_andv4si3 ; gen_pack = gen_sse4_1_packusdw ; gen_shift = gen_lshrv4si3 ; break ; case E_V8QImode : c = <NUM_LIT> ; s = <NUM_LIT> ; half_mode = V4HImode ; gen_and = gen_andv4hi3 ; gen_pack = gen_mmx_packuswb ; gen_shift = gen_lshrv4hi3 ; break ; case E_V16QImode : c = <NUM_LIT> ; s = <NUM_LIT> ; half_mode = V8HImode ; gen_and = gen_andv8hi3 ; gen_pack = gen_sse2_packuswb ; gen_shift = gen_lshrv8hi3 ; break ; case E_V16HImode : if ( ! TARGET_AVX2 ) return false ; c = <NUM_LIT> ; s = <NUM_LIT> ; half_mode = V8SImode ; gen_and = gen_andv8si3 ; gen_pack = gen_avx2_packusdw ; gen_shift = gen_lshrv8si3 ; end_perm = true ; break ; case E_V32QImode : if ( ! TARGET_AVX2 ) return false ; c = <NUM_LIT> ; s = <NUM_LIT> ; half_mode = V16HImode ; gen_and = gen_andv16hi3 ; gen_pack = gen_avx2_packuswb ; gen_shift = gen_lshrv16hi3 ; end_perm = true ; break ; default : return false ; } odd = d -> perm [ <NUM_LIT> ] ; if ( odd > <NUM_LIT> ) return false ; for ( i = <NUM_LIT> ; i < nelt ; ++ i ) if ( d -> perm [ i ] != <NUM_LIT> * i + odd ) return false ; if ( d -> testing_p ) return true ; dop0 = gen_reg_rtx ( half_mode ) ; dop1 = gen_reg_rtx ( half_mode ) ; if ( odd == <NUM_LIT> ) { t = gen_const_vec_duplicate ( half_mode , GEN_INT ( c ) ) ; t = force_reg ( half_mode , t ) ; emit_insn ( gen_and ( dop0 , t , gen_lowpart ( half_mode , d -> op0 ) ) ) ; emit_insn ( gen_and ( dop1 , t , gen_lowpart ( half_mode , d -> op1 ) ) ) ; } else { emit_insn ( gen_shift ( dop0 , gen_lowpart ( half_mode , d -> op0 ) , GEN_INT ( s ) ) ) ; emit_insn ( gen_shift ( dop1 , gen_lowpart ( half_mode , d -> op1 ) , GEN_INT ( s ) ) ) ; } if ( TARGET_AVX2 && end_perm ) { op = gen_reg_rtx ( d -> vmode ) ; t = gen_reg_rtx ( V4DImode ) ; emit_insn ( gen_pack ( op , dop0 , dop1 ) ) ;


<s> if ( odd > <NUM_LIT> ) return false ; for ( i = <NUM_LIT> ; i < nelt ; ++ i ) if ( d -> perm [ i ] != <NUM_LIT> * i + odd ) return false ; if ( d -> testing_p ) return true ; if ( odd ) { t1 = gen_reg_rtx ( V32HImode ) ; t2 = gen_reg_rtx ( V32HImode ) ; emit_insn ( gen_lshrv32hi3 ( t1 , gen_lowpart ( V32HImode , d -> op0 ) , GEN_INT ( <NUM_LIT> ) ) ) ; emit_insn ( gen_lshrv32hi3 ( t2 , gen_lowpart ( V32HImode , d -> op1 ) , GEN_INT ( <NUM_LIT> ) ) ) ; }


<s> else return false ; nelt = d -> nelt ; if ( d -> perm [ <NUM_LIT> ] != <NUM_LIT> && d -> perm [ <NUM_LIT> ] != nelt / <NUM_LIT> ) return false ; for ( i = <NUM_LIT> ; i < nelt ; i += <NUM_LIT> ) if ( d -> perm [ i ] != d -> perm [ <NUM_LIT> ] + i / <NUM_LIT> || d -> perm [ i + <NUM_LIT> ] != d -> perm [ <NUM_LIT> ] + i / <NUM_LIT> + nelt ) return false ; if ( d -> testing_p ) return true ; switch ( d -> vmode ) { case E_V32QImode : if ( d -> perm [ <NUM_LIT> ] ) gen = gen_vec_interleave_highv32qi ; else gen = gen_vec_interleave_lowv32qi ; break ;


<s> if ( d -> perm [ <NUM_LIT> ] != nelt && d -> perm [ <NUM_LIT> ] != <NUM_LIT> ) return false ; for ( i = <NUM_LIT> ; i < nelt ; ++ i ) if ( d -> perm [ i ] != i + nelt - d -> perm [ <NUM_LIT> ] ) return false ; if ( d -> testing_p ) return true ; if ( d -> perm [ <NUM_LIT> ] == nelt ) x = gen_rtx_VEC_MERGE ( vmode , d -> op1 , d -> op0 , GEN_INT ( <NUM_LIT> ) ) ; else x = gen_rtx_VEC_MERGE ( vmode , d -> op0 , d -> op1 , GEN_INT ( <NUM_LIT> ) ) ; emit_insn ( gen_rtx_SET ( d -> target , x ) ) ; return true ;


<s> unsigned min , max , minswap , maxswap ; bool in_order , ok , swap = false ; rtx shift , target ; struct expand_vec_perm_d dcopy ; if ( ( ! TARGET_SSSE3 || GET_MODE_SIZE ( d -> vmode ) != <NUM_LIT> ) && ( ! TARGET_AVX2 || GET_MODE_SIZE ( d -> vmode ) != <NUM_LIT> ) ) return false ; min = <NUM_LIT> * nelt ; max = <NUM_LIT> ; minswap = <NUM_LIT> * nelt ; maxswap = <NUM_LIT> ; for ( i = <NUM_LIT> ; i < nelt ; ++ i ) { unsigned e = d -> perm [ i ] ; unsigned eswap = d -> perm [ i ] ^ nelt ; if ( GET_MODE_SIZE ( d -> vmode ) == <NUM_LIT> ) { e = ( e & ( ( nelt / <NUM_LIT> ) - <NUM_LIT> ) ) | ( ( e & nelt ) >> <NUM_LIT> ) ; eswap = e ^ ( nelt / <NUM_LIT> ) ; } if ( e < min ) min = e ; if ( e > max ) max = e ; if ( eswap < minswap ) minswap = eswap ; if ( eswap > maxswap ) maxswap = eswap ; } if ( min == <NUM_LIT> || max - min >= ( GET_MODE_SIZE ( d -> vmode ) == <NUM_LIT> ? nelt / <NUM_LIT> : nelt ) ) { if ( d -> one_operand_p || minswap == <NUM_LIT> || maxswap - minswap >= ( GET_MODE_SIZE ( d -> vmode ) == <NUM_LIT> ? nelt / <NUM_LIT> : nelt ) ) return false ; swap = true ; min = minswap ; max = maxswap ; } if ( d -> testing_p && GET_MODE_SIZE ( d -> vmode ) == <NUM_LIT> && ! single_insn_only_p ) return true ; dcopy = * d ; if ( swap ) {


<s> bool ok ; if ( d -> one_operand_p ) return false ; if ( TARGET_AVX2 && GET_MODE_SIZE ( vmode ) == <NUM_LIT> ) ; else if ( TARGET_AVX && ( vmode == V4DFmode || vmode == V8SFmode ) ) ; else if ( TARGET_SSE4_1 && ( GET_MODE_SIZE ( vmode ) == <NUM_LIT> || GET_MODE_SIZE ( vmode ) == <NUM_LIT> || GET_MODE_SIZE ( vmode ) == <NUM_LIT> ) ) ; else return false ; for ( i = <NUM_LIT> , which = <NUM_LIT> ; i < nelt ; ++ i ) { unsigned e = d -> perm [ i ] ; if ( e != i ) which |= ( e < nelt ? <NUM_LIT> : <NUM_LIT> ) ; } if ( which != <NUM_LIT> && which != <NUM_LIT> ) return false ; if ( d -> testing_p && GET_MODE_SIZE ( vmode ) == <NUM_LIT> ) return true ; dcopy = * d ; if ( which == <NUM_LIT> ) dcopy . op0 = dcopy . op1 = d -> op1 ; else dcopy . op0 = dcopy . op1 = d -> op0 ; if ( ! d -> testing_p ) dcopy . target = gen_reg_rtx ( vmode ) ; dcopy . one_operand_p = true ; for ( i = <NUM_LIT> ; i < nelt ; ++ i ) dcopy . perm [ i ] = d -> perm [ i ] & ( nelt - <NUM_LIT> ) ; ok = expand_vec_perm_1 ( & dcopy ) ; if ( GET_MODE_SIZE ( vmode ) != <NUM_LIT> && ! ok ) return false ; else gcc_assert ( ok ) ; if ( d -> testing_p ) return true ; dcopy1 = * d ; if ( which == <NUM_LIT> ) dcopy1 . op1 = dcopy . target ; else dcopy1 . op0 = dcopy . target ; for ( i = <NUM_LIT> ; i < nelt ; ++ i ) dcopy1 . perm [ i ] = ( ( d -> perm [ i ] >= nelt ) ? ( nelt + i ) : i ) ; ok = expand_vec_perm_blend ( & dcopy1 ) ;


<s> nelt = d -> nelt ; eltsz = GET_MODE_UNIT_SIZE ( d -> vmode ) ; m128 = GEN_INT ( - <NUM_LIT> ) ; for ( i = <NUM_LIT> ; i < nelt ; ++ i ) { unsigned j , k , e = d -> perm [ i ] ; unsigned which = ( e >= nelt ) ; if ( e >= nelt ) e -= nelt ; for ( j = <NUM_LIT> ; j < eltsz ; ++ j ) { rperm [ which ] [ i * eltsz + j ] = GEN_INT ( e * eltsz + j ) ; rperm [ <NUM_LIT> - which ] [ i * eltsz + j ] = m128 ; } for ( k = i * eltsz + j ; k < <NUM_LIT> ; ++ k ) rperm [ <NUM_LIT> ] [ k ] = rperm [ <NUM_LIT> ] [ k ] = m128 ;


<s> rtx rperm [ <NUM_LIT> ] [ <NUM_LIT> ] , vperm , l , h , op , m128 ; unsigned int i , nelt , eltsz ; machine_mode mode ; rtx ( * gen ) ( rtx , rtx , rtx ) ; if ( ! TARGET_SSSE3 || ( GET_MODE_SIZE ( d -> vmode ) != <NUM_LIT> && GET_MODE_SIZE ( d -> vmode ) != <NUM_LIT> && GET_MODE_SIZE ( d -> vmode ) != <NUM_LIT> ) ) return false ; gcc_assert ( ! d -> one_operand_p ) ; if ( d -> testing_p ) return true ; switch ( GET_MODE_SIZE ( d -> vmode ) ) { case <NUM_LIT> : mode = V4QImode ; gen = gen_mmx_pshufbv4qi3 ; break ; case <NUM_LIT> : mode = V8QImode ; gen = gen_mmx_pshufbv8qi3 ; break ; case <NUM_LIT> : mode = V16QImode ; gen = gen_ssse3_pshufbv16qi3 ; break ; default : gcc_unreachable ( ) ; } nelt = d -> nelt ; eltsz = GET_MODE_UNIT_SIZE ( d -> vmode ) ; m128 = GEN_INT ( - <NUM_LIT> ) ; for ( i = <NUM_LIT> ; i < nelt ; ++ i ) { unsigned j , k , e = d -> perm [ i ] ; unsigned which = ( e >= nelt ) ; if ( e >= nelt ) e -= nelt ; for ( j = <NUM_LIT> ; j < eltsz ; ++ j ) { rperm [ which ] [ i * eltsz + j ] = GEN_INT ( e * eltsz + j ) ; rperm [ <NUM_LIT> - which ] [ i * eltsz + j ] = m128 ; }

for ( k = i * eltsz + j ; k < <NUM_LIT> ; ++ k ) rperm [ <NUM_LIT> ] [ k ] = rperm [ <NUM_LIT> ] [ k ] = m128 ;


<s> unsigned char perm2 [ MAX_VECT_LEN ] ; unsigned i ; bool ok ; if ( d -> vmode != V8HImode || ! d -> one_operand_p ) return false ; for ( i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) if ( d -> perm [ i ] >= <NUM_LIT> ) return false ; for ( i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) if ( d -> perm [ i ] < <NUM_LIT> ) return false ; if ( d -> testing_p ) return true ; memcpy ( perm2 , d -> perm , <NUM_LIT> ) ; for ( i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) perm2 [ i ] = i ; ok = expand_vselect ( d -> target , d -> op0 , perm2 , <NUM_LIT> , d -> testing_p ) ; gcc_assert ( ok ) ; memcpy ( perm2 + <NUM_LIT> , d -> perm + <NUM_LIT> , <NUM_LIT> ) ; for ( i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) perm2 [ i ] = i ;

ok = expand_vselect ( d -> target , d -> target , perm2 , <NUM_LIT> , d -> testing_p ) ;


<s> if ( ( perm & <NUM_LIT> ) == ( <NUM_LIT> << <NUM_LIT> ) ) continue ; if ( ( perm & <NUM_LIT> ) == <NUM_LIT> ) continue ; if ( ( perm & <NUM_LIT> ) == ( <NUM_LIT> << <NUM_LIT> ) ) continue ; } for ( i = <NUM_LIT> ; i < nelt ; i ++ ) { j = d -> perm [ i ] / nelt2 ; if ( j == ( ( perm >> ( <NUM_LIT> * ( i >= nelt2 ) ) ) & <NUM_LIT> ) ) dsecond . perm [ i ] = nelt + ( i & nelt2 ) + ( d -> perm [ i ] & ( nelt2 - <NUM_LIT> ) ) ; else if ( j == ( unsigned ) ( i >= nelt2 ) + <NUM_LIT> * ( perm >= <NUM_LIT> ) ) dsecond . perm [ i ] = d -> perm [ i ] & ( nelt - <NUM_LIT> ) ; else break ; } if ( i == nelt ) { start_sequence ( ) ; ok = expand_vec_perm_1 ( & dsecond ) ; end_sequence ( ) ; } else ok = false ; if ( ok ) { if ( d -> testing_p ) return true ; dsecond . testing_p = false ; dfirst = * d ; dfirst . target = gen_reg_rtx ( d -> vmode ) ; for ( i = <NUM_LIT> ; i < nelt ; i ++ ) dfirst . perm [ i ] = ( i & ( nelt2 - <NUM_LIT> ) ) + ( ( perm >> ( <NUM_LIT> * ( i >= nelt2 ) ) ) & <NUM_LIT> ) * nelt2 ; canonicalize_perm ( & dfirst ) ; ok = expand_vec_perm_1 ( & dfirst ) ; gcc_assert ( ok ) ; if ( perm >= <NUM_LIT> ) dsecond . op0 = dsecond . op1 ;


<s> j = d -> perm [ i ] / nelt2 ; if ( j == ( ( perm >> ( <NUM_LIT> * ( i >= nelt2 ) ) ) & <NUM_LIT> ) ) dsecond . perm [ i ] = nelt + ( i & nelt2 ) + ( d -> perm [ i ] & ( nelt2 - <NUM_LIT> ) ) ; else if ( j == ( unsigned ) ( i >= nelt2 ) + <NUM_LIT> * ( perm >= <NUM_LIT> ) ) dsecond . perm [ i ] = d -> perm [ i ] & ( nelt - <NUM_LIT> ) ; else break ; } if ( i == nelt ) { start_sequence ( ) ; ok = expand_vec_perm_1 ( & dsecond ) ; end_sequence ( ) ; } else ok = false ; if ( ok ) { if ( d -> testing_p ) return true ; dsecond . testing_p = false ; dfirst = * d ; dfirst . target = gen_reg_rtx ( d -> vmode ) ; for ( i = <NUM_LIT> ; i < nelt ; i ++ ) dfirst . perm [ i ] = ( i & ( nelt2 - <NUM_LIT> ) ) + ( ( perm >> ( <NUM_LIT> * ( i >= nelt2 ) ) ) & <NUM_LIT> ) * nelt2 ; canonicalize_perm ( & dfirst ) ; ok = expand_vec_perm_1 ( & dfirst ) ; gcc_assert ( ok ) ; if ( perm >= <NUM_LIT> ) dsecond . op0 = dsecond . op1 ; dsecond . op1 = dfirst . target ; ok = expand_vec_perm_1 ( & dsecond ) ; gcc_assert ( ok ) ; return true ;


<s> j = ( d -> perm [ i ] & nelt2 ) ? i | nelt2 : i & ~ nelt2 ; if ( dfirst . perm [ j ] != <NUM_LIT> && dfirst . perm [ j ] != d -> perm [ i ] ) return false ; dfirst . perm [ j ] = d -> perm [ i ] ; if ( j != i ) msk |= ( <NUM_LIT> << i ) ; } for ( i = <NUM_LIT> ; i < nelt ; i ++ ) if ( dfirst . perm [ i ] == <NUM_LIT> ) dfirst . perm [ i ] = i ; if ( ! d -> testing_p ) dfirst . target = gen_reg_rtx ( dfirst . vmode ) ; start_sequence ( ) ; ok = expand_vec_perm_1 ( & dfirst ) ; seq = get_insns ( ) ; end_sequence ( ) ; if ( ! ok ) return false ; if ( d -> testing_p ) return true ; emit_insn ( seq ) ; dsecond = * d ; dsecond . op0 = dfirst . target ; dsecond . op1 = dfirst . target ; dsecond . one_operand_p = true ; dsecond . target = gen_reg_rtx ( dsecond . vmode ) ; for ( i = <NUM_LIT> ; i < nelt ; i ++ ) dsecond . perm [ i ] = i ^ nelt2 ; ok = expand_vec_perm_1 ( & dsecond ) ; gcc_assert ( ok ) ; blend = d -> vmode == V8SFmode ? gen_avx_blendps256 : gen_avx_blendpd256 ; emit_insn ( blend ( d -> target , dfirst . target , dsecond . target , GEN_INT ( msk ) ) ) ;


<s> unsigned i ; if ( ! TARGET_AVX || d -> vmode != V8SFmode || ! d -> one_operand_p ) return false ; for ( i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) { unsigned e = d -> perm [ i ] ; if ( i < <NUM_LIT> ? e >= <NUM_LIT> : e < <NUM_LIT> ) return false ; } if ( d -> testing_p ) return true ; for ( i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) { unsigned e = d -> perm [ i ] ; if ( e >= <NUM_LIT> + <NUM_LIT> ) e -= <NUM_LIT> ; else if ( e >= <NUM_LIT> ) e -= <NUM_LIT> ;


<s> dremap . target = gen_reg_rtx ( V4DImode ) ; dremap . op0 = gen_lowpart ( V4DImode , d -> op0 ) ; dremap . op1 = dremap . op0 ; dremap . one_operand_p = true ; for ( i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) { unsigned int cnt = <NUM_LIT> ; for ( j = <NUM_LIT> ; j < <NUM_LIT> ; ++ j ) if ( ( contents [ i ] & ( <NUM_LIT> << j ) ) != <NUM_LIT> ) dremap . perm [ <NUM_LIT> * i + cnt ++ ] = j ; for ( ; cnt < <NUM_LIT> ; ++ cnt ) dremap . perm [ <NUM_LIT> * i + cnt ] = <NUM_LIT> ; } dfinal = * d ; dfinal . op0 = gen_lowpart ( dfinal . vmode , dremap . target ) ; dfinal . op1 = dfinal . op0 ; dfinal . one_operand_p = true ;

for ( i = <NUM_LIT> , j = <NUM_LIT> ; i < nelt ; ++ i ) {


<s> for ( j = <NUM_LIT> ; j < <NUM_LIT> ; ++ j ) if ( ( contents [ i ] & ( <NUM_LIT> << j ) ) != <NUM_LIT> ) dremap . perm [ <NUM_LIT> * i + cnt ++ ] = j ; for ( ; cnt < <NUM_LIT> ; ++ cnt ) dremap . perm [ <NUM_LIT> * i + cnt ] = <NUM_LIT> ; } dfinal = * d ; dfinal . op0 = gen_lowpart ( dfinal . vmode , dremap . target ) ; dfinal . op1 = dfinal . op0 ; dfinal . one_operand_p = true ; for ( i = <NUM_LIT> , j = <NUM_LIT> ; i < nelt ; ++ i ) { if ( i == nelt2 ) j = <NUM_LIT> ; dfinal . perm [ i ] = ( d -> perm [ i ] & ( nelt4 - <NUM_LIT> ) ) | ( j ? nelt2 : <NUM_LIT> ) ; if ( ( d -> perm [ i ] / nelt4 ) == dremap . perm [ j ] ) ; else if ( ( d -> perm [ i ] / nelt4 ) == dremap . perm [ j + <NUM_LIT> ] ) dfinal . perm [ i ] |= nelt4 ; else gcc_unreachable ( ) ;


<s> if ( d -> testing_p ) return true ; nelt = d -> nelt ; eltsz = GET_MODE_UNIT_SIZE ( d -> vmode ) ; m128 = GEN_INT ( - <NUM_LIT> ) ; for ( i = <NUM_LIT> ; i < nelt ; ++ i ) { unsigned j , e = d -> perm [ i ] & ( nelt / <NUM_LIT> - <NUM_LIT> ) ; unsigned which = ( ( d -> perm [ i ] ^ i ) & ( nelt / <NUM_LIT> ) ) * eltsz ; for ( j = <NUM_LIT> ; j < eltsz ; ++ j ) { rperm [ ! ! which ] [ ( i * eltsz + j ) ^ which ] = GEN_INT ( e * eltsz + j ) ; rperm [ ! which ] [ ( i * eltsz + j ) ^ ( which ^ <NUM_LIT> ) ] = m128 ; } } vperm = gen_rtx_CONST_VECTOR ( V32QImode , gen_rtvec_v ( <NUM_LIT> , rperm [ <NUM_LIT> ] ) ) ; vperm = force_reg ( V32QImode , vperm ) ; h = gen_reg_rtx ( V32QImode ) ; op = gen_lowpart ( V32QImode , d -> op0 ) ; emit_insn ( gen_avx2_pshufbv32qi3 ( h , op , vperm ) ) ; hp = gen_reg_rtx ( V4DImode ) ; op = gen_lowpart ( V4DImode , h ) ; emit_insn ( gen_avx2_permv4di_1 ( hp , op , const2_rtx , GEN_INT ( <NUM_LIT> ) , const0_rtx , const1_rtx ) ) ; vperm = gen_rtx_CONST_VECTOR ( V32QImode , gen_rtvec_v ( <NUM_LIT> , rperm [ <NUM_LIT> ] ) ) ; vperm = force_reg ( V32QImode , vperm ) ;