Richard Biener <rguent...@suse.de> writes:
> I've been sitting on this for a few days since I'm not 100% happy
> with how the code looks like. There's possibly still holes in it
> (chains with mixed signed/unsigned adds for example might pick
> up signed adds in the epilogue), but the wrong-code cases should
> work fine now. I'm probably going to followup with some
> mass renaming of variable/parameter names to make it more clear
> which stmt / type we are actually looking at ...
>
> Bootstrapped and tested on x86_64-unknown-linux-gnu, applied to trunk.
Does this look like the right way of updating neutral_op_for_slp_reduction?
It now needs to know whether the caller is using STMT_VINFO_REDUC_VECTYPE
(for an epilogue value) or STMT_VINFO_REDUC_VECTYPE_IN (for a PHI argument).
Fixes various gcc.target/aarch64/sve/slp_* tests, will give it a
full test on aarch64-linux-gnu.
Thanks,
Richard
2019-11-08 Richard Sandiford <richard.sandif...@arm.com>
gcc/
* tree-vect-loop.c (neutral_op_for_slp_reduction): Take the
vector type as an argument rather than reading it from the
stmt_vec_info.
(vect_create_epilog_for_reduction): Update accordingly,
passing the STMT_VINFO_REDUC_VECTYPE.
(vectorizable_reduction): Likewise.
(vect_transform_cycle_phi): Likewise, but passing the
STMT_VINFO_REDUC_VECTYPE_IN.
Index: gcc/tree-vect-loop.c
===================================================================
--- gcc/tree-vect-loop.c 2019-11-08 09:06:29.654896085 +0000
+++ gcc/tree-vect-loop.c 2019-11-08 10:41:54.498861004 +0000
@@ -2586,17 +2586,17 @@ reduction_fn_for_scalar_code (enum tree_
/* If there is a neutral value X such that SLP reduction NODE would not
be affected by the introduction of additional X elements, return that X,
- otherwise return null. CODE is the code of the reduction. REDUC_CHAIN
- is true if the SLP statements perform a single reduction, false if each
- statement performs an independent reduction. */
+ otherwise return null. CODE is the code of the reduction and VECTOR_TYPE
+ is the vector type that would hold element X. REDUC_CHAIN is true if
+ the SLP statements perform a single reduction, false if each statement
+ performs an independent reduction. */
static tree
-neutral_op_for_slp_reduction (slp_tree slp_node, tree_code code,
- bool reduc_chain)
+neutral_op_for_slp_reduction (slp_tree slp_node, tree vector_type,
+ tree_code code, bool reduc_chain)
{
vec<stmt_vec_info> stmts = SLP_TREE_SCALAR_STMTS (slp_node);
stmt_vec_info stmt_vinfo = stmts[0];
- tree vector_type = STMT_VINFO_VECTYPE (stmt_vinfo);
tree scalar_type = TREE_TYPE (vector_type);
class loop *loop = gimple_bb (stmt_vinfo->stmt)->loop_father;
gcc_assert (loop);
@@ -4216,11 +4216,6 @@ vect_create_epilog_for_reduction (stmt_v
= as_a <gphi *> (STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info))->stmt);
enum tree_code code = STMT_VINFO_REDUC_CODE (reduc_info);
internal_fn reduc_fn = STMT_VINFO_REDUC_FN (reduc_info);
- tree neutral_op = NULL_TREE;
- if (slp_node)
- neutral_op
- = neutral_op_for_slp_reduction (slp_node_instance->reduc_phis, code,
- REDUC_GROUP_FIRST_ELEMENT (stmt_info));
stmt_vec_info prev_phi_info;
tree vectype;
machine_mode mode;
@@ -4267,11 +4262,15 @@ vect_create_epilog_for_reduction (stmt_v
gcc_assert (vectype);
mode = TYPE_MODE (vectype);
+ tree neutral_op = NULL_TREE;
tree initial_def = NULL;
tree induc_val = NULL_TREE;
tree adjustment_def = NULL;
if (slp_node)
- ;
+ neutral_op
+ = neutral_op_for_slp_reduction (slp_node_instance->reduc_phis,
+ vectype, code,
+ REDUC_GROUP_FIRST_ELEMENT (stmt_info));
else
{
/* Get at the scalar def before the loop, that defines the initial value
@@ -6210,7 +6209,7 @@ vectorizable_reduction (stmt_vec_info st
tree neutral_op = NULL_TREE;
if (slp_node)
neutral_op = neutral_op_for_slp_reduction
- (slp_node_instance->reduc_phis, orig_code,
+ (slp_node_instance->reduc_phis, vectype_out, orig_code,
REDUC_GROUP_FIRST_ELEMENT (stmt_info) != NULL);
if (double_reduc && reduction_type == FOLD_LEFT_REDUCTION)
@@ -6793,7 +6792,7 @@ vect_transform_cycle_phi (stmt_vec_info
gcc_assert (slp_node == slp_node_instance->reduc_phis);
stmt_vec_info first = REDUC_GROUP_FIRST_ELEMENT (reduc_stmt_info);
tree neutral_op
- = neutral_op_for_slp_reduction (slp_node,
+ = neutral_op_for_slp_reduction (slp_node, vectype_in,
STMT_VINFO_REDUC_CODE (reduc_info),
first != NULL);
get_initial_defs_for_reduction (slp_node_instance->reduc_phis,
