On Mon, Jun 18, 2018 at 5:01 PM Richard Sandiford
<richard.sandif...@arm.com> wrote:
>
> This patch makes pattern recognisers do their own checking for vector
> types and target support. Previously some recognisers did this
> themselves and some left it to vect_pattern_recog_1.
>
> Doing this means we can get rid of the type_in argument, which was
> ignored if the recogniser did its own checking. It also means
> we create fewer junk statements.
>
> Tested on aarch64-linux-gnu and x86_64-linux-gnu. OK to install?
OK.
Richard.
> Richard
>
>
> 2018-06-18 Richard Sandiford <richard.sandif...@arm.com>
>
> gcc/
> * tree-vectorizer.h (NUM_PATTERNS, vect_recog_func_ptr): Move to
> tree-vect-patterns.c.
> * tree-vect-patterns.c (vect_supportable_direct_optab_p): New
> function.
> (vect_recog_dot_prod_pattern): Use it. Remove the type_in argument.
> (vect_recog_sad_pattern): Likewise.
> (vect_recog_widen_sum_pattern): Likewise.
> (vect_recog_pow_pattern): Likewise. Check for a null vectype.
> (vect_recog_widen_shift_pattern): Remove the type_in argument.
> (vect_recog_rotate_pattern): Likewise.
> (vect_recog_mult_pattern): Likewise.
> (vect_recog_vector_vector_shift_pattern): Likewise.
> (vect_recog_divmod_pattern): Likewise.
> (vect_recog_mixed_size_cond_pattern): Likewise.
> (vect_recog_bool_pattern): Likewise.
> (vect_recog_mask_conversion_pattern): Likewise.
> (vect_try_gather_scatter_pattern): Likewise.
> (vect_recog_widen_mult_pattern): Likewise. Check for a null vectype.
> (vect_recog_over_widening_pattern): Likewise.
> (vect_recog_gather_scatter_pattern): Likewise.
> (vect_recog_func_ptr): Move from tree-vectorizer.h
> (vect_vect_recog_func_ptrs): Move further down the file.
> (vect_recog_func): Likewise. Remove the third argument.
> (NUM_PATTERNS): Define based on vect_vect_recog_func_ptrs.
> (vect_pattern_recog_1): Expect the pattern function to do any
> necessary target tests. Also expect it to provide a vector type.
> Remove the type_in handling.
>
> Index: gcc/tree-vectorizer.h
> ===================================================================
> --- gcc/tree-vectorizer.h 2018-06-14 12:27:34.104084473 +0100
> +++ gcc/tree-vectorizer.h 2018-06-18 15:43:52.951038712 +0100
> @@ -1616,8 +1616,6 @@ extern int vect_get_place_in_interleavin
> /* Pattern recognition functions.
> Additional pattern recognition functions can (and will) be added
> in the future. */
> -typedef gimple *(* vect_recog_func_ptr) (vec<gimple *> *, tree *, tree *);
> -#define NUM_PATTERNS 15
> void vect_pattern_recog (vec_info *);
>
> /* In tree-vectorizer.c. */
> Index: gcc/tree-vect-patterns.c
> ===================================================================
> --- gcc/tree-vect-patterns.c 2018-06-18 15:43:52.000000000 +0100
> +++ gcc/tree-vect-patterns.c 2018-06-18 15:43:52.951038712 +0100
> @@ -47,66 +47,6 @@ Software Foundation; either version 3, o
> #include "omp-simd-clone.h"
> #include "predict.h"
>
> -/* Pattern recognition functions */
> -static gimple *vect_recog_widen_sum_pattern (vec<gimple *> *, tree *,
> - tree *);
> -static gimple *vect_recog_widen_mult_pattern (vec<gimple *> *, tree *,
> - tree *);
> -static gimple *vect_recog_dot_prod_pattern (vec<gimple *> *, tree *,
> - tree *);
> -static gimple *vect_recog_sad_pattern (vec<gimple *> *, tree *,
> - tree *);
> -static gimple *vect_recog_pow_pattern (vec<gimple *> *, tree *, tree *);
> -static gimple *vect_recog_over_widening_pattern (vec<gimple *> *, tree *,
> - tree *);
> -static gimple *vect_recog_widen_shift_pattern (vec<gimple *> *,
> - tree *, tree *);
> -static gimple *vect_recog_rotate_pattern (vec<gimple *> *, tree *, tree *);
> -static gimple *vect_recog_vector_vector_shift_pattern (vec<gimple *> *,
> - tree *, tree *);
> -static gimple *vect_recog_divmod_pattern (vec<gimple *> *,
> - tree *, tree *);
> -
> -static gimple *vect_recog_mult_pattern (vec<gimple *> *,
> - tree *, tree *);
> -
> -static gimple *vect_recog_mixed_size_cond_pattern (vec<gimple *> *,
> - tree *, tree *);
> -static gimple *vect_recog_bool_pattern (vec<gimple *> *, tree *, tree *);
> -static gimple *vect_recog_mask_conversion_pattern (vec<gimple *> *, tree *,
> tree *);
> -static gimple *vect_recog_gather_scatter_pattern (vec<gimple *> *, tree *,
> - tree *);
> -
> -struct vect_recog_func
> -{
> - vect_recog_func_ptr fn;
> - const char *name;
> -};
> -
> -/* Note that ordering matters - the first pattern matching on a stmt
> - is taken which means usually the more complex one needs to preceed
> - the less comples onex (widen_sum only after dot_prod or sad for example).
> */
> -static vect_recog_func vect_vect_recog_func_ptrs[NUM_PATTERNS] = {
> - { vect_recog_widen_mult_pattern, "widen_mult" },
> - { vect_recog_dot_prod_pattern, "dot_prod" },
> - { vect_recog_sad_pattern, "sad" },
> - { vect_recog_widen_sum_pattern, "widen_sum" },
> - { vect_recog_pow_pattern, "pow" },
> - { vect_recog_widen_shift_pattern, "widen_shift" },
> - { vect_recog_over_widening_pattern, "over_widening" },
> - { vect_recog_rotate_pattern, "rotate" },
> - { vect_recog_vector_vector_shift_pattern, "vector_vector_shift" },
> - { vect_recog_divmod_pattern, "divmod" },
> - { vect_recog_mult_pattern, "mult" },
> - { vect_recog_mixed_size_cond_pattern, "mixed_size_cond" },
> - { vect_recog_bool_pattern, "bool" },
> - /* This must come before mask conversion, and includes the parts
> - of mask conversion that are needed for gather and scatter
> - internal functions. */
> - { vect_recog_gather_scatter_pattern, "gather_scatter" },
> - { vect_recog_mask_conversion_pattern, "mask_conversion" }
> -};
> -
> /* Report that we've found an instance of pattern PATTERN in
> statement STMT. */
>
> @@ -134,6 +74,49 @@ new_pattern_def_seq (stmt_vec_info stmt_
> append_pattern_def_seq (stmt_info, stmt);
> }
>
> +/* Return true if the target supports a vector version of CODE,
> + where CODE is known to map to a direct optab. ITYPE specifies
> + the type of (some of) the scalar inputs and OTYPE specifies the
> + type of the scalar result.
> +
> + If CODE allows the inputs and outputs to have different type
> + (such as for WIDEN_SUM_EXPR), it is the input mode rather
> + than the output mode that determines the appropriate target pattern.
> + Operand 0 of the target pattern then specifies the mode that the output
> + must have.
> +
> + When returning true, set *VECOTYPE_OUT to the vector version of OTYPE.
> + Also set *VECITYPE_OUT to the vector version of ITYPE if VECITYPE_OUT
> + is nonnull. */
> +
> +static bool
> +vect_supportable_direct_optab_p (tree otype, tree_code code,
> + tree itype, tree *vecotype_out,
> + tree *vecitype_out = NULL)
> +{
> + tree vecitype = get_vectype_for_scalar_type (itype);
> + if (!vecitype)
> + return false;
> +
> + tree vecotype = get_vectype_for_scalar_type (otype);
> + if (!vecotype)
> + return false;
> +
> + optab optab = optab_for_tree_code (code, vecitype, optab_default);
> + if (!optab)
> + return false;
> +
> + insn_code icode = optab_handler (optab, TYPE_MODE (vecitype));
> + if (icode == CODE_FOR_nothing
> + || insn_data[icode].operand[0].mode != TYPE_MODE (vecotype))
> + return false;
> +
> + *vecotype_out = vecotype;
> + if (vecitype_out)
> + *vecitype_out = vecitype;
> + return true;
> +}
> +
> /* Check whether STMT2 is in the same loop or basic block as STMT1.
> Which of the two applies depends on whether we're currently doing
> loop-based or basic-block-based vectorization, as determined by
> @@ -293,8 +276,6 @@ vect_reassociating_reduction_p (stmt_vec
>
> Output:
>
> - * TYPE_IN: The type of the input arguments to the pattern.
> -
> * TYPE_OUT: The type of the output of this pattern.
>
> * Return value: A new stmt that will be used to replace the sequence of
> @@ -310,8 +291,7 @@ vect_reassociating_reduction_p (stmt_vec
> inner-loop nested in an outer-loop that us being vectorized). */
>
> static gimple *
> -vect_recog_dot_prod_pattern (vec<gimple *> *stmts, tree *type_in,
> - tree *type_out)
> +vect_recog_dot_prod_pattern (vec<gimple *> *stmts, tree *type_out)
> {
> gimple *stmt, *last_stmt = (*stmts)[0];
> tree oprnd0, oprnd1;
> @@ -448,8 +428,9 @@ vect_recog_dot_prod_pattern (vec<gimple
> vect_pattern_detected ("vect_recog_dot_prod_pattern", last_stmt);
>
> half_type = TREE_TYPE (oprnd00);
> - *type_in = half_type;
> - *type_out = type;
> + if (!vect_supportable_direct_optab_p (type, DOT_PROD_EXPR, half_type,
> + type_out))
> + return NULL;
>
> var = vect_recog_temp_ssa_var (type, NULL);
> pattern_stmt = gimple_build_assign (var, DOT_PROD_EXPR,
> @@ -489,8 +470,6 @@ vect_recog_dot_prod_pattern (vec<gimple
>
> Output:
>
> - * TYPE_IN: The type of the input arguments to the pattern.
> -
> * TYPE_OUT: The type of the output of this pattern.
>
> * Return value: A new stmt that will be used to replace the sequence of
> @@ -499,8 +478,7 @@ vect_recog_dot_prod_pattern (vec<gimple
> */
>
> static gimple *
> -vect_recog_sad_pattern (vec<gimple *> *stmts, tree *type_in,
> - tree *type_out)
> +vect_recog_sad_pattern (vec<gimple *> *stmts, tree *type_out)
> {
> gimple *last_stmt = (*stmts)[0];
> tree sad_oprnd0, sad_oprnd1;
> @@ -641,8 +619,9 @@ vect_recog_sad_pattern (vec<gimple *> *s
>
> vect_pattern_detected ("vect_recog_sad_pattern", last_stmt);
>
> - *type_in = TREE_TYPE (sad_oprnd0);
> - *type_out = sum_type;
> + if (!vect_supportable_direct_optab_p (sum_type, SAD_EXPR,
> + TREE_TYPE (sad_oprnd0), type_out))
> + return NULL;
>
> tree var = vect_recog_temp_ssa_var (sum_type, NULL);
> gimple *pattern_stmt = gimple_build_assign (var, SAD_EXPR, sad_oprnd0,
> @@ -778,8 +757,6 @@ vect_handle_widen_op_by_const (gimple *s
>
> Output:
>
> - * TYPE_IN: The type of the input arguments to the pattern.
> -
> * TYPE_OUT: The type of the output of this pattern.
>
> * Return value: A new stmt that will be used to replace the sequence of
> @@ -790,8 +767,7 @@ vect_handle_widen_op_by_const (gimple *s
> */
>
> static gimple *
> -vect_recog_widen_mult_pattern (vec<gimple *> *stmts,
> - tree *type_in, tree *type_out)
> +vect_recog_widen_mult_pattern (vec<gimple *> *stmts, tree *type_out)
> {
> gimple *last_stmt = stmts->pop ();
> gimple *def_stmt0, *def_stmt1;
> @@ -933,8 +909,9 @@ vect_recog_widen_mult_pattern (vec<gimpl
> &dummy_int, &dummy_vec))
> return NULL;
>
> - *type_in = vectype;
> *type_out = get_vectype_for_scalar_type (type);
> + if (!*type_out)
> + return NULL;
>
> /* Pattern supported. Create a stmt to be used to replace the pattern: */
> var = vect_recog_temp_ssa_var (itype, NULL);
> @@ -989,8 +966,6 @@ vect_recog_widen_mult_pattern (vec<gimpl
>
> Output:
>
> - * TYPE_IN: The type of the input arguments to the pattern.
> -
> * TYPE_OUT: The type of the output of this pattern.
>
> * Return value: A new stmt that will be used to replace the sequence of
> @@ -1001,8 +976,7 @@ vect_recog_widen_mult_pattern (vec<gimpl
> */
>
> static gimple *
> -vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_in,
> - tree *type_out)
> +vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_out)
> {
> gimple *last_stmt = (*stmts)[0];
> tree base, exp;
> @@ -1072,12 +1046,13 @@ vect_recog_pow_pattern (vec<gimple *> *s
> if (node->simd_clones == NULL)
> return NULL;
> }
> + *type_out = get_vectype_for_scalar_type (TREE_TYPE (base));
> + if (!*type_out)
> + return NULL;
> stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
> tree def = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
> gimple *g = gimple_build_assign (def, MULT_EXPR, exp, logc);
> new_pattern_def_seq (stmt_vinfo, g);
> - *type_in = TREE_TYPE (base);
> - *type_out = NULL_TREE;
> tree res = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
> g = gimple_build_call (exp_decl, 1, def);
> gimple_call_set_lhs (g, res);
> @@ -1091,15 +1066,15 @@ vect_recog_pow_pattern (vec<gimple *> *s
> /* We now have a pow or powi builtin function call with a constant
> exponent. */
>
> - *type_out = NULL_TREE;
> -
> /* Catch squaring. */
> if ((tree_fits_shwi_p (exp)
> && tree_to_shwi (exp) == 2)
> || (TREE_CODE (exp) == REAL_CST
> && real_equal (&TREE_REAL_CST (exp), &dconst2)))
> {
> - *type_in = TREE_TYPE (base);
> + if (!vect_supportable_direct_optab_p (TREE_TYPE (base), MULT_EXPR,
> + TREE_TYPE (base), type_out))
> + return NULL;
>
> var = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
> stmt = gimple_build_assign (var, MULT_EXPR, base, base);
> @@ -1110,9 +1085,9 @@ vect_recog_pow_pattern (vec<gimple *> *s
> if (TREE_CODE (exp) == REAL_CST
> && real_equal (&TREE_REAL_CST (exp), &dconsthalf))
> {
> - *type_in = get_vectype_for_scalar_type (TREE_TYPE (base));
> - if (*type_in
> - && direct_internal_fn_supported_p (IFN_SQRT, *type_in,
> + *type_out = get_vectype_for_scalar_type (TREE_TYPE (base));
> + if (*type_out
> + && direct_internal_fn_supported_p (IFN_SQRT, *type_out,
> OPTIMIZE_FOR_SPEED))
> {
> gcall *stmt = gimple_build_call_internal (IFN_SQRT, 1, base);
> @@ -1150,8 +1125,6 @@ vect_recog_pow_pattern (vec<gimple *> *s
>
> Output:
>
> - * TYPE_IN: The type of the input arguments to the pattern.
> -
> * TYPE_OUT: The type of the output of this pattern.
>
> * Return value: A new stmt that will be used to replace the sequence of
> @@ -1167,8 +1140,7 @@ vect_recog_pow_pattern (vec<gimple *> *s
> inner-loop nested in an outer-loop that us being vectorized). */
>
> static gimple *
> -vect_recog_widen_sum_pattern (vec<gimple *> *stmts, tree *type_in,
> - tree *type_out)
> +vect_recog_widen_sum_pattern (vec<gimple *> *stmts, tree *type_out)
> {
> gimple *stmt, *last_stmt = (*stmts)[0];
> tree oprnd0, oprnd1;
> @@ -1223,14 +1195,15 @@ vect_recog_widen_sum_pattern (vec<gimple
> if (!type_conversion_p (oprnd0, last_stmt, true, &half_type, &stmt,
> &promotion)
> || !promotion)
> - return NULL;
> + return NULL;
>
> oprnd0 = gimple_assign_rhs1 (stmt);
>
> vect_pattern_detected ("vect_recog_widen_sum_pattern", last_stmt);
>
> - *type_in = half_type;
> - *type_out = type;
> + if (!vect_supportable_direct_optab_p (type, WIDEN_SUM_EXPR, half_type,
> + type_out))
> + return NULL;
>
> var = vect_recog_temp_ssa_var (type, NULL);
> pattern_stmt = gimple_build_assign (var, WIDEN_SUM_EXPR, oprnd0, oprnd1);
> @@ -1473,8 +1446,7 @@ vect_operation_fits_smaller_type (gimple
> demotion operation. We also check that S3 and S4 have only one use. */
>
> static gimple *
> -vect_recog_over_widening_pattern (vec<gimple *> *stmts,
> - tree *type_in, tree *type_out)
> +vect_recog_over_widening_pattern (vec<gimple *> *stmts, tree *type_out)
> {
> gimple *stmt = stmts->pop ();
> gimple *pattern_stmt = NULL, *new_def_stmt, *prev_stmt = NULL,
> @@ -1562,14 +1534,15 @@ vect_recog_over_widening_pattern (vec<gi
> if (TYPE_UNSIGNED (new_type) != TYPE_UNSIGNED (use_type)
> || TYPE_PRECISION (new_type) != TYPE_PRECISION (use_type))
> {
> + *type_out = get_vectype_for_scalar_type (use_type);
> + if (!*type_out)
> + return NULL;
> +
> /* Create NEW_TYPE->USE_TYPE conversion. */
> new_oprnd = make_ssa_name (use_type);
> pattern_stmt = gimple_build_assign (new_oprnd, NOP_EXPR, var);
> STMT_VINFO_RELATED_STMT (vinfo_for_stmt (use_stmt)) = pattern_stmt;
>
> - *type_in = get_vectype_for_scalar_type (new_type);
> - *type_out = get_vectype_for_scalar_type (use_type);
> -
> /* We created a pattern statement for the last statement in the
> sequence, so we don't need to associate it with the pattern
> statement created for PREV_STMT. Therefore, we add PREV_STMT
> @@ -1583,8 +1556,7 @@ vect_recog_over_widening_pattern (vec<gi
> STMT_VINFO_PATTERN_DEF_SEQ (vinfo_for_stmt (use_stmt))
> = STMT_VINFO_PATTERN_DEF_SEQ (vinfo_for_stmt (prev_stmt));
>
> - *type_in = vectype;
> - *type_out = NULL_TREE;
> + *type_out = vectype;
> }
>
> stmts->safe_push (use_stmt);
> @@ -1648,8 +1620,6 @@ vect_recog_over_widening_pattern (vec<gi
>
> Output:
>
> - * TYPE_IN: The type of the input arguments to the pattern.
> -
> * TYPE_OUT: The type of the output of this pattern.
>
> * Return value: A new stmt that will be used to replace the sequence of
> @@ -1657,8 +1627,7 @@ vect_recog_over_widening_pattern (vec<gi
> WIDEN_LSHIFT_EXPR <a_t, CONST>. */
>
> static gimple *
> -vect_recog_widen_shift_pattern (vec<gimple *> *stmts,
> - tree *type_in, tree *type_out)
> +vect_recog_widen_shift_pattern (vec<gimple *> *stmts, tree *type_out)
> {
> gimple *last_stmt = stmts->pop ();
> gimple *def_stmt0;
> @@ -1740,7 +1709,6 @@ vect_recog_widen_shift_pattern (vec<gimp
> &dummy_int, &dummy_vec))
> return NULL;
>
> - *type_in = vectype;
> *type_out = vectype_out;
>
> /* Pattern supported. Create a stmt to be used to replace the pattern. */
> @@ -1783,15 +1751,13 @@ vect_recog_widen_shift_pattern (vec<gimp
>
> Output:
>
> - * TYPE_IN: The type of the input arguments to the pattern.
> -
> * TYPE_OUT: The type of the output of this pattern.
>
> * Return value: A new stmt that will be used to replace the rotate
> S0 stmt. */
>
> static gimple *
> -vect_recog_rotate_pattern (vec<gimple *> *stmts, tree *type_in, tree
> *type_out)
> +vect_recog_rotate_pattern (vec<gimple *> *stmts, tree *type_out)
> {
> gimple *last_stmt = stmts->pop ();
> tree oprnd0, oprnd1, lhs, var, var1, var2, vectype, type, stype, def, def2;
> @@ -1876,10 +1842,7 @@ vect_recog_rotate_pattern (vec<gimple *>
> return NULL;
> }
>
> - *type_in = vectype;
> *type_out = vectype;
> - if (*type_in == NULL_TREE)
> - return NULL;
>
> if (dt == vect_external_def
> && TREE_CODE (oprnd1) == SSA_NAME
> @@ -2034,16 +1997,13 @@ vect_recog_rotate_pattern (vec<gimple *>
>
> Output:
>
> - * TYPE_IN: The type of the input arguments to the pattern.
> -
> * TYPE_OUT: The type of the output of this pattern.
>
> * Return value: A new stmt that will be used to replace the shift/rotate
> S3 stmt. */
>
> static gimple *
> -vect_recog_vector_vector_shift_pattern (vec<gimple *> *stmts,
> - tree *type_in, tree *type_out)
> +vect_recog_vector_vector_shift_pattern (vec<gimple *> *stmts, tree *type_out)
> {
> gimple *last_stmt = stmts->pop ();
> tree oprnd0, oprnd1, lhs, var;
> @@ -2085,9 +2045,8 @@ vect_recog_vector_vector_shift_pattern (
> if (!def_vinfo)
> return NULL;
>
> - *type_in = get_vectype_for_scalar_type (TREE_TYPE (oprnd0));
> - *type_out = *type_in;
> - if (*type_in == NULL_TREE)
> + *type_out = get_vectype_for_scalar_type (TREE_TYPE (oprnd0));
> + if (*type_out == NULL_TREE)
> return NULL;
>
> tree def = NULL_TREE;
> @@ -2449,16 +2408,13 @@ vect_synth_mult_by_constant (tree op, tr
>
> Output:
>
> - * TYPE_IN: The type of the input arguments to the pattern.
> -
> * TYPE_OUT: The type of the output of this pattern.
>
> * Return value: A new stmt that will be used to replace
> the multiplication. */
>
> static gimple *
> -vect_recog_mult_pattern (vec<gimple *> *stmts,
> - tree *type_in, tree *type_out)
> +vect_recog_mult_pattern (vec<gimple *> *stmts, tree *type_out)
> {
> gimple *last_stmt = stmts->pop ();
> tree oprnd0, oprnd1, vectype, itype;
> @@ -2504,7 +2460,6 @@ vect_recog_mult_pattern (vec<gimple *> *
> vect_pattern_detected ("vect_recog_mult_pattern", last_stmt);
>
> stmts->safe_push (last_stmt);
> - *type_in = vectype;
> *type_out = vectype;
>
> return pattern_stmt;
> @@ -2543,16 +2498,13 @@ vect_recog_mult_pattern (vec<gimple *> *
>
> Output:
>
> - * TYPE_IN: The type of the input arguments to the pattern.
> -
> * TYPE_OUT: The type of the output of this pattern.
>
> * Return value: A new stmt that will be used to replace the division
> S1 or modulo S4 stmt. */
>
> static gimple *
> -vect_recog_divmod_pattern (vec<gimple *> *stmts,
> - tree *type_in, tree *type_out)
> +vect_recog_divmod_pattern (vec<gimple *> *stmts, tree *type_out)
> {
> gimple *last_stmt = stmts->pop ();
> tree oprnd0, oprnd1, vectype, itype, cond;
> @@ -2703,7 +2655,6 @@ vect_recog_divmod_pattern (vec<gimple *>
>
> stmts->safe_push (last_stmt);
>
> - *type_in = vectype;
> *type_out = vectype;
> return pattern_stmt;
> }
> @@ -2955,7 +2906,6 @@ vect_recog_divmod_pattern (vec<gimple *>
>
> stmts->safe_push (last_stmt);
>
> - *type_in = vectype;
> *type_out = vectype;
> return pattern_stmt;
> }
> @@ -2980,8 +2930,6 @@ vect_recog_divmod_pattern (vec<gimple *>
>
> Output:
>
> - * TYPE_IN: The type of the input arguments to the pattern.
> -
> * TYPE_OUT: The type of the output of this pattern.
>
> * Return value: A new stmt that will be used to replace the pattern.
> @@ -2991,8 +2939,7 @@ vect_recog_divmod_pattern (vec<gimple *>
> a_T = (TYPE) a_it; */
>
> static gimple *
> -vect_recog_mixed_size_cond_pattern (vec<gimple *> *stmts, tree *type_in,
> - tree *type_out)
> +vect_recog_mixed_size_cond_pattern (vec<gimple *> *stmts, tree *type_out)
> {
> gimple *last_stmt = (*stmts)[0];
> tree cond_expr, then_clause, else_clause;
> @@ -3108,7 +3055,6 @@ vect_recog_mixed_size_cond_pattern (vec<
> def_stmt_info = new_stmt_vec_info (def_stmt, vinfo);
> set_vinfo_for_stmt (def_stmt, def_stmt_info);
> STMT_VINFO_VECTYPE (def_stmt_info) = vecitype;
> - *type_in = vecitype;
> *type_out = vectype;
>
> vect_pattern_detected ("vect_recog_mixed_size_cond_pattern", last_stmt);
> @@ -3581,8 +3527,6 @@ search_type_for_mask (tree var, vec_info
>
> Output:
>
> - * TYPE_IN: The type of the input arguments to the pattern.
> -
> * TYPE_OUT: The type of the output of this pattern.
>
> * Return value: A new stmt that will be used to replace the pattern.
> @@ -3602,8 +3546,7 @@ search_type_for_mask (tree var, vec_info
> but the above is more efficient. */
>
> static gimple *
> -vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in,
> - tree *type_out)
> +vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_out)
> {
> gimple *last_stmt = stmts->pop ();
> enum tree_code rhs_code;
> @@ -3679,7 +3622,6 @@ vect_recog_bool_pattern (vec<gimple *> *
> }
>
> *type_out = vectype;
> - *type_in = vectype;
> stmts->safe_push (last_stmt);
> vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
>
> @@ -3718,7 +3660,6 @@ vect_recog_bool_pattern (vec<gimple *> *
> gimple_assign_rhs2 (last_stmt),
> gimple_assign_rhs3 (last_stmt));
> *type_out = vectype;
> - *type_in = vectype;
> stmts->safe_push (last_stmt);
> vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
>
> @@ -3775,7 +3716,6 @@ vect_recog_bool_pattern (vec<gimple *> *
> STMT_VINFO_DR_WRT_VEC_LOOP (pattern_stmt_info)
> = STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo);
> *type_out = vectype;
> - *type_in = vectype;
> stmts->safe_push (last_stmt);
> vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
>
> @@ -3839,8 +3779,7 @@ build_mask_conversion (tree mask, tree v
> S4' c_1' = m_3'' ? c_2 : c_3; */
>
> static gimple *
> -vect_recog_mask_conversion_pattern (vec<gimple *> *stmts, tree *type_in,
> - tree *type_out)
> +vect_recog_mask_conversion_pattern (vec<gimple *> *stmts, tree *type_out)
> {
> gimple *last_stmt = stmts->pop ();
> enum tree_code rhs_code;
> @@ -3911,7 +3850,6 @@ vect_recog_mask_conversion_pattern (vec<
> = STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo);
>
> *type_out = vectype1;
> - *type_in = vectype1;
> stmts->safe_push (last_stmt);
> vect_pattern_detected ("vect_recog_mask_conversion_pattern",
> last_stmt);
>
> @@ -4035,7 +3973,6 @@ vect_recog_mask_conversion_pattern (vec<
> gimple_assign_rhs3 (last_stmt));
>
> *type_out = vectype1;
> - *type_in = vectype1;
> stmts->safe_push (last_stmt);
> vect_pattern_detected ("vect_recog_mask_conversion_pattern",
> last_stmt);
>
> @@ -4081,7 +4018,6 @@ vect_recog_mask_conversion_pattern (vec<
> pattern_stmt = gimple_build_assign (lhs, rhs_code, rhs1, rhs2);
>
> *type_out = vectype1;
> - *type_in = vectype1;
> stmts->safe_push (last_stmt);
> vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
>
> @@ -4180,14 +4116,14 @@ vect_add_conversion_to_patterm (tree typ
>
> /* Try to convert STMT into a call to a gather load or scatter store
> internal function. Return the final statement on success and set
> - *TYPE_IN and *TYPE_OUT to the vector type being loaded or stored.
> + *TYPE_OUT to the vector type being loaded or stored.
>
> This function only handles gathers and scatters that were recognized
> as such from the outset (indicated by STMT_VINFO_GATHER_SCATTER_P). */
>
> static gimple *
> vect_try_gather_scatter_pattern (gimple *stmt, stmt_vec_info last_stmt_info,
> - tree *type_in, tree *type_out)
> + tree *type_out)
> {
> /* Currently we only support this for loop vectorization. */
> stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
> @@ -4264,7 +4200,6 @@ vect_try_gather_scatter_pattern (gimple
>
> tree vectype = STMT_VINFO_VECTYPE (stmt_info);
> *type_out = vectype;
> - *type_in = vectype;
> vect_pattern_detected ("gather/scatter pattern", stmt);
>
> return pattern_stmt;
> @@ -4273,19 +4208,52 @@ vect_try_gather_scatter_pattern (gimple
> /* Pattern wrapper around vect_try_gather_scatter_pattern. */
>
> static gimple *
> -vect_recog_gather_scatter_pattern (vec<gimple *> *stmts, tree *type_in,
> - tree *type_out)
> +vect_recog_gather_scatter_pattern (vec<gimple *> *stmts, tree *type_out)
> {
> gimple *last_stmt = stmts->pop ();
> stmt_vec_info last_stmt_info = vinfo_for_stmt (last_stmt);
> gimple *pattern_stmt = vect_try_gather_scatter_pattern (last_stmt,
> last_stmt_info,
> - type_in, type_out);
> + type_out);
> if (pattern_stmt)
> stmts->safe_push (last_stmt);
> return pattern_stmt;
> }
>
> +typedef gimple *(*vect_recog_func_ptr) (vec<gimple *> *, tree *);
> +
> +struct vect_recog_func
> +{
> + vect_recog_func_ptr fn;
> + const char *name;
> +};
> +
> +/* Note that ordering matters - the first pattern matching on a stmt is
> + taken which means usually the more complex one needs to preceed the
> + less comples onex (widen_sum only after dot_prod or sad for example). */
> +static vect_recog_func vect_vect_recog_func_ptrs[] = {
> + { vect_recog_widen_mult_pattern, "widen_mult" },
> + { vect_recog_dot_prod_pattern, "dot_prod" },
> + { vect_recog_sad_pattern, "sad" },
> + { vect_recog_widen_sum_pattern, "widen_sum" },
> + { vect_recog_pow_pattern, "pow" },
> + { vect_recog_widen_shift_pattern, "widen_shift" },
> + { vect_recog_over_widening_pattern, "over_widening" },
> + { vect_recog_rotate_pattern, "rotate" },
> + { vect_recog_vector_vector_shift_pattern, "vector_vector_shift" },
> + { vect_recog_divmod_pattern, "divmod" },
> + { vect_recog_mult_pattern, "mult" },
> + { vect_recog_mixed_size_cond_pattern, "mixed_size_cond" },
> + { vect_recog_bool_pattern, "bool" },
> + /* This must come before mask conversion, and includes the parts
> + of mask conversion that are needed for gather and scatter
> + internal functions. */
> + { vect_recog_gather_scatter_pattern, "gather_scatter" },
> + { vect_recog_mask_conversion_pattern, "mask_conversion" }
> +};
> +
> +const unsigned int NUM_PATTERNS = ARRAY_SIZE (vect_vect_recog_func_ptrs);
> +
> /* Mark statements that are involved in a pattern. */
>
> static inline void
> @@ -4337,17 +4305,13 @@ vect_mark_pattern_stmts (gimple *orig_st
> computation pattern.
> STMT: A stmt from which the pattern search should start.
>
> - If PATTERN_RECOG_FUNC successfully detected the pattern, it creates an
> - expression that computes the same functionality and can be used to
> - replace the sequence of stmts that are involved in the pattern.
> -
> - Output:
> - This function checks if the expression returned by PATTERN_RECOG_FUNC is
> - supported in vector form by the target. We use 'TYPE_IN' to obtain the
> - relevant vector type. If 'TYPE_IN' is already a vector type, then this
> - indicates that target support had already been checked by
> PATTERN_RECOG_FUNC.
> - If 'TYPE_OUT' is also returned by PATTERN_RECOG_FUNC, we check that it
> fits
> - to the available target pattern.
> + If PATTERN_RECOG_FUNC successfully detected the pattern, it creates
> + a sequence of statements that has the same functionality and can be
> + used to replace STMT. It returns the last statement in the sequence
> + and adds any earlier statements to STMT's STMT_VINFO_PATTERN_DEF_SEQ.
> + PATTERN_RECOG_FUNC also sets *TYPE_OUT to the vector type of the final
> + statement, having first checked that the target supports the new operation
> + in that type.
>
> This function also does some bookkeeping, as explained in the
> documentation
> for vect_recog_pattern. */
> @@ -4361,56 +4325,19 @@ vect_pattern_recog_1 (vect_recog_func *r
> stmt_vec_info stmt_info;
> loop_vec_info loop_vinfo;
> tree pattern_vectype;
> - tree type_in, type_out;
> - enum tree_code code;
> int i;
>
> stmts_to_replace->truncate (0);
> stmts_to_replace->quick_push (stmt);
> - pattern_stmt = recog_func->fn (stmts_to_replace, &type_in, &type_out);
> + pattern_stmt = recog_func->fn (stmts_to_replace, &pattern_vectype);
> if (!pattern_stmt)
> return false;
>
> stmt = stmts_to_replace->last ();
> stmt_info = vinfo_for_stmt (stmt);
> loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
> + gcc_assert (pattern_vectype);
>
> - if (VECTOR_BOOLEAN_TYPE_P (type_in)
> - || VECTOR_TYPE_P (type_in))
> - {
> - /* No need to check target support (already checked by the pattern
> - recognition function). */
> - pattern_vectype = type_out ? type_out : type_in;
> - }
> - else
> - {
> - /* Check target support */
> - type_in = get_vectype_for_scalar_type (type_in);
> - if (!type_in)
> - return false;
> - if (type_out)
> - type_out = get_vectype_for_scalar_type (type_out);
> - else
> - type_out = type_in;
> - if (!type_out)
> - return false;
> - pattern_vectype = type_out;
> -
> - if (is_gimple_assign (pattern_stmt))
> - {
> - enum insn_code icode;
> - code = gimple_assign_rhs_code (pattern_stmt);
> - optab optab = optab_for_tree_code (code, type_in, optab_default);
> - machine_mode vec_mode = TYPE_MODE (type_in);
> - if (!optab
> - || (icode = optab_handler (optab, vec_mode)) == CODE_FOR_nothing
> - || (insn_data[icode].operand[0].mode != TYPE_MODE (type_out)))
> - return false;
> - }
> - else
> - gcc_assert (is_gimple_call (pattern_stmt));
> - }
> -
> /* Found a vectorizable pattern. */
> if (dump_enabled_p ())
> {
