This patch adds a vectoriser helper routine to calculate how
many copies of a vector statement we need. At present this
is always:
LOOP_VINFO_VECT_FACTOR (loop_vinfo) / TYPE_VECTOR_SUBPARTS (vectype)
but later patches add other cases. Another benefit of using
a helper routine is that it can assert that the division is
exact (which it must be).
Tested on aarch64-linux-gnu, x86_64-linux-gnu and powerpc64le-linux-gnu.
OK to install?
Richard
2017-09-14 Richard Sandiford <richard.sandif...@linaro.org>
Alan Hayward <alan.hayw...@arm.com>
David Sherwood <david.sherw...@arm.com>
gcc/
* tree-vectorizer.h (vect_get_num_copies): New function.
* tree-vect-data-refs.c (vect_get_data_access_cost): Use it.
* tree-vect-loop.c (vectorizable_reduction): Likewise.
(vectorizable_induction): Likewise.
(vectorizable_live_operation): Likewise.
* tree-vect-stmts.c (vectorizable_mask_load_store): Likewise.
(vectorizable_bswap): Likewise.
(vectorizable_call): Likewise.
(vectorizable_conversion): Likewise.
(vectorizable_assignment): Likewise.
(vectorizable_shift): Likewise.
(vectorizable_operation): Likewise.
(vectorizable_store): Likewise.
(vectorizable_load): Likewise.
(vectorizable_condition): Likewise.
(vectorizable_comparison): Likewise.
(vect_analyze_stmt): Pass the slp node to vectorizable_live_operation.
Index: gcc/tree-vectorizer.h
===================================================================
--- gcc/tree-vectorizer.h 2017-09-14 11:25:32.166167193 +0100
+++ gcc/tree-vectorizer.h 2017-09-14 11:27:50.352072753 +0100
@@ -1076,6 +1076,20 @@ unlimited_cost_model (loop_p loop)
return (flag_vect_cost_model == VECT_COST_MODEL_UNLIMITED);
}
+/* Return the number of copies needed for loop vectorization when
+ a statement operates on vectors of type VECTYPE. This is the
+ vectorization factor divided by the number of elements in
+ VECTYPE and is always known at compile time. */
+
+static inline unsigned int
+vect_get_num_copies (loop_vec_info loop_vinfo, tree vectype)
+{
+ gcc_checking_assert (LOOP_VINFO_VECT_FACTOR (loop_vinfo)
+ % TYPE_VECTOR_SUBPARTS (vectype) == 0);
+ return (LOOP_VINFO_VECT_FACTOR (loop_vinfo)
+ / TYPE_VECTOR_SUBPARTS (vectype));
+}
+
/* Source location */
extern source_location vect_location;
Index: gcc/tree-vect-data-refs.c
===================================================================
--- gcc/tree-vect-data-refs.c 2017-09-14 11:25:32.163167193 +0100
+++ gcc/tree-vect-data-refs.c 2017-09-14 11:27:50.350257085 +0100
@@ -1181,10 +1181,13 @@ vect_get_data_access_cost (struct data_r
{
gimple *stmt = DR_STMT (dr);
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
- int nunits = TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info));
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
- int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
- int ncopies = MAX (1, vf / nunits); /* TODO: Handle SLP properly */
+ int ncopies;
+
+ if (PURE_SLP_STMT (stmt_info))
+ ncopies = 1;
+ else
+ ncopies = vect_get_num_copies (loop_vinfo, STMT_VINFO_VECTYPE (stmt_info));
if (DR_IS_READ (dr))
vect_get_load_cost (dr, ncopies, true, inside_cost, outside_cost,
Index: gcc/tree-vect-loop.c
===================================================================
--- gcc/tree-vect-loop.c 2017-09-14 11:27:16.962234838 +0100
+++ gcc/tree-vect-loop.c 2017-09-14 11:27:50.351164919 +0100
@@ -5683,8 +5683,7 @@ vectorizable_reduction (gimple *stmt, gi
if (slp_node)
ncopies = 1;
else
- ncopies = (LOOP_VINFO_VECT_FACTOR (loop_vinfo)
- / TYPE_VECTOR_SUBPARTS (vectype_in));
+ ncopies = vect_get_num_copies (loop_vinfo, vectype_in);
use_operand_p use_p;
gimple *use_stmt;
@@ -5980,8 +5979,7 @@ vectorizable_reduction (gimple *stmt, gi
if (slp_node)
ncopies = 1;
else
- ncopies = (LOOP_VINFO_VECT_FACTOR (loop_vinfo)
- / TYPE_VECTOR_SUBPARTS (vectype_in));
+ ncopies = vect_get_num_copies (loop_vinfo, vectype_in);
gcc_assert (ncopies >= 1);
@@ -6550,7 +6548,7 @@ vectorizable_induction (gimple *phi,
if (slp_node)
ncopies = 1;
else
- ncopies = vf / nunits;
+ ncopies = vect_get_num_copies (loop_vinfo, vectype);
gcc_assert (ncopies >= 1);
/* FORNOW. These restrictions should be relaxed. */
@@ -7013,12 +7011,17 @@ vectorizable_live_operation (gimple *stm
tree lhs, lhs_type, bitsize, vec_bitsize;
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
int nunits = TYPE_VECTOR_SUBPARTS (vectype);
- int ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
+ int ncopies;
gimple *use_stmt;
auto_vec<tree> vec_oprnds;
gcc_assert (STMT_VINFO_LIVE_P (stmt_info));
+ if (slp_node)
+ ncopies = 1;
+ else
+ ncopies = vect_get_num_copies (loop_vinfo, vectype);
+
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def)
return false;
Index: gcc/tree-vect-stmts.c
===================================================================
--- gcc/tree-vect-stmts.c 2017-09-14 11:25:32.166167193 +0100
+++ gcc/tree-vect-stmts.c 2017-09-14 11:27:50.352072753 +0100
@@ -2038,7 +2038,7 @@ vectorizable_mask_load_store (gimple *st
if (slp_node != NULL)
return false;
- ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
+ ncopies = vect_get_num_copies (loop_vinfo, vectype);
gcc_assert (ncopies >= 1);
mask = gimple_call_arg (stmt, 2);
@@ -2472,7 +2472,7 @@ vectorizable_bswap (gimple *stmt, gimple
if (slp_node)
ncopies = 1;
else
- ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
+ ncopies = vect_get_num_copies (loop_vinfo, vectype);
gcc_assert (ncopies >= 1);
@@ -2805,9 +2805,9 @@ vectorizable_call (gimple *gs, gimple_st
if (slp_node)
ncopies = 1;
else if (modifier == NARROW && ifn == IFN_LAST)
- ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_out;
+ ncopies = vect_get_num_copies (loop_vinfo, vectype_out);
else
- ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
+ ncopies = vect_get_num_copies (loop_vinfo, vectype_in);
/* Sanity check: make sure that at least one copy of the vectorized stmt
needs to be generated. */
@@ -4204,9 +4204,9 @@ vectorizable_conversion (gimple *stmt, g
if (slp_node)
ncopies = 1;
else if (modifier == NARROW)
- ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_out;
+ ncopies = vect_get_num_copies (loop_vinfo, vectype_out);
else
- ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
+ ncopies = vect_get_num_copies (loop_vinfo, vectype_in);
/* Sanity check: make sure that at least one copy of the vectorized stmt
needs to be generated. */
@@ -4690,7 +4690,7 @@ vectorizable_assignment (gimple *stmt, g
if (slp_node)
ncopies = 1;
else
- ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
+ ncopies = vect_get_num_copies (loop_vinfo, vectype);
gcc_assert (ncopies >= 1);
@@ -4948,7 +4948,7 @@ vectorizable_shift (gimple *stmt, gimple
if (slp_node)
ncopies = 1;
else
- ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
+ ncopies = vect_get_num_copies (loop_vinfo, vectype);
gcc_assert (ncopies >= 1);
@@ -5379,7 +5379,7 @@ vectorizable_operation (gimple *stmt, gi
if (slp_node)
ncopies = 1;
else
- ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
+ ncopies = vect_get_num_copies (loop_vinfo, vectype);
gcc_assert (ncopies >= 1);
@@ -5718,7 +5718,7 @@ vectorizable_store (gimple *stmt, gimple
if (slp)
ncopies = 1;
else
- ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
+ ncopies = vect_get_num_copies (loop_vinfo, vectype);
gcc_assert (ncopies >= 1);
@@ -6713,7 +6713,7 @@ vectorizable_load (gimple *stmt, gimple_
if (slp)
ncopies = 1;
else
- ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
+ ncopies = vect_get_num_copies (loop_vinfo, vectype);
gcc_assert (ncopies >= 1);
@@ -7926,13 +7926,12 @@ vectorizable_condition (gimple *stmt, gi
return false;
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
- int nunits = TYPE_VECTOR_SUBPARTS (vectype);
tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
if (slp_node)
ncopies = 1;
else
- ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
+ ncopies = vect_get_num_copies (loop_vinfo, vectype);
gcc_assert (ncopies >= 1);
if (reduc_index && ncopies > 1)
@@ -8269,7 +8268,7 @@ vectorizable_comparison (gimple *stmt, g
if (slp_node)
ncopies = 1;
else
- ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
+ ncopies = vect_get_num_copies (loop_vinfo, vectype);
gcc_assert (ncopies >= 1);
if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
@@ -8700,7 +8699,7 @@ vect_analyze_stmt (gimple *stmt, bool *n
need extra handling, except for vectorizable reductions. */
if (STMT_VINFO_LIVE_P (stmt_info)
&& STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type)
- ok = vectorizable_live_operation (stmt, NULL, NULL, -1, NULL);
+ ok = vectorizable_live_operation (stmt, NULL, node, -1, NULL);
if (!ok)
{
