Introduce LOOP_VINFO_IV_INCREMENT which stores the number of scalar
iterations an iteration of the vectorized loop has processed. Then
updates IV updates and reductions to use this value instead of VF.
Update IV update logic so the IV_INCREMENT can be loop-variant.
Simplify/remove SELECT_VL IV increment special cases by instead using
IV_INCREMENT.
As part of this change, the grouped load/store data pointer logic changes
from:
ADDRESS_1 = PHI <... , ADDRESS_3>
V1 = LOAD ADDRESS_1
ADDRESS_2 = ADDRESS_1 + BUMP
V2 = LOAD ADDRESS_2
ADDRESS_3 = ADDRESS_2 + BUMP
to:
ADDRESS_1 = PHI <... , ADDRESS_3>
V1 = LOAD ADDRESS_1
ADDRESS_2 = ADDRESS_1 + BUMP
V2 = LOAD ADDRESS_2
ADDRESS_3 = ADDRESS_1 + STEP * IV_INCREMENT
gcc/ChangeLog:
* tree-ssa-loop-manip.cc (create_iv): Change step to not
necessarily be loop invariant.
* tree-vect-data-refs.cc (bump_vector_ptr): Remove logic for
updating use-def chain.
* tree-vect-loop-manip.cc (vect_set_loop_controls_directly):
Change to use LOOP_VINFO_IV_INCREMENT instead of VF.
(vect_set_loop_condition_partial_vectors_avx512):
Change to use LOOP_VINFO_IV_INCREMENT instead of VF.
(vect_gen_vector_loop_niters): Update to handle
LOOP_VINFO_IV_INCREMENT.
(vect_get_loop_iv_increment): New function.
* tree-vect-loop.cc (vectorizable_induction): Update to use
LOOP_VINFO_IV_INCREMENT instead of VF, and remove SELECT_VL
logic.
(vect_update_ivs_after_vectorizer_for_early_breaks):
Update to use LOOP_VINFO_IV_INCREMENT over VF and move IV
update to vect_iv_increment_position.
(vect_transform_loop): Add initialization of
LOOP_VINFO_IV_INCREMENT.
* tree-vect-stmts.cc (vect_get_strided_load_store_ops):
Update to use dr_increment and dr_bump, remove SELECT_VL logic.
(vect_get_data_ptr_increment): Change to return the increment
needed to advance the pointer to the next iteration.
(vect_get_data_ptr_bump): New function.
(vectorizable_scan_store): Update to use vect_get_data_ptr_bump
and to remove SELECT_VL logic.
(vectorizable_store): Update to use vect_get_data_ptr_bump and
vect_get_data_ptr_increment.
(vectorizable_load): Update to use vect_get_data_ptr_bump and
vect_get_data_ptr_increment.
* tree-vectorizer.h (loop_vec_info): Add iv_increment field.
(LOOP_VINFO_IV_INCREMENT): New macro.
(bump_vector_ptr): Remove incr_ptr argument.
(vect_get_loop_iv_increment): New function.
---
gcc/tree-ssa-loop-manip.cc | 7 +-
gcc/tree-vect-data-refs.cc | 40 +-----
gcc/tree-vect-loop-manip.cc | 69 +++++++---
gcc/tree-vect-loop.cc | 76 +++--------
gcc/tree-vect-stmts.cc | 248 ++++++++++++++++++------------------
gcc/tree-vectorizer.h | 8 +-
6 files changed, 206 insertions(+), 242 deletions(-)
diff --git a/gcc/tree-ssa-loop-manip.cc b/gcc/tree-ssa-loop-manip.cc
index 03e255281b7..f48379438fc 100644
--- a/gcc/tree-ssa-loop-manip.cc
+++ b/gcc/tree-ssa-loop-manip.cc
@@ -152,13 +152,10 @@ create_iv (tree base, tree_code incr_op, tree step, tree
var, class loop *loop,
step = fold_build1 (NEGATE_EXPR, TREE_TYPE (step), step);
incr_op = POINTER_PLUS_EXPR;
}
- /* Gimplify the step if necessary. We put the computations in front of the
- loop (i.e. the step should be loop invariant). */
- step = force_gimple_operand (step, &stmts, true, NULL_TREE);
- if (stmts)
- gsi_insert_seq_on_edge_immediate (pe, stmts);
gimple_seq incr_stmts = nullptr;
+ /* Gimplify the step if necessary. */
+ step = force_gimple_operand (step, &incr_stmts, true, NULL_TREE);
gimple_seq_add_stmt (&incr_stmts,
gimple_build_assign (va, incr_op, vb, step));
insert_iv_increment (incr_pos, after, incr_stmts);
diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc
index 9ec21429686..f1c23b28516 100644
--- a/gcc/tree-vect-data-refs.cc
+++ b/gcc/tree-vect-data-refs.cc
@@ -6022,30 +6022,12 @@ vect_create_data_ref_ptr (vec_info *vinfo,
stmt_vec_info stmt_info,
/* Function bump_vector_ptr
- Increment a pointer (to a vector type) by vector-size. If requested,
- i.e. if PTR-INCR is given, then also connect the new increment stmt
- to the existing def-use update-chain of the pointer, by modifying
- the PTR_INCR as illustrated below:
-
- The pointer def-use update-chain before this function:
- DATAREF_PTR = phi (p_0, p_2)
- ....
- PTR_INCR: p_2 = DATAREF_PTR + step
-
- The pointer def-use update-chain after this function:
- DATAREF_PTR = phi (p_0, p_2)
- ....
- NEW_DATAREF_PTR = DATAREF_PTR + BUMP
- ....
- PTR_INCR: p_2 = NEW_DATAREF_PTR + step
+ Increment a pointer (to a vector type) by update.
Input:
DATAREF_PTR - ssa_name of a pointer (to vector type) that is being updated
in the loop.
- PTR_INCR - optional. The stmt that updates the pointer in each iteration of
- the loop. The increment amount across iterations is expected
- to be vector_size.
- BSI - location where the new update stmt is to be placed.
+ GSI - location where the new update stmt is to be placed.
STMT_INFO - the original scalar memory-access stmt that is being vectorized.
UPDATE - The offset by which to bump the pointer.
@@ -6055,13 +6037,11 @@ vect_create_data_ref_ptr (vec_info *vinfo,
stmt_vec_info stmt_info,
tree
bump_vector_ptr (vec_info *vinfo,
- tree dataref_ptr, gimple *ptr_incr, gimple_stmt_iterator *gsi,
+ tree dataref_ptr, gimple_stmt_iterator *gsi,
stmt_vec_info stmt_info, tree update)
{
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
gimple *incr_stmt;
- ssa_op_iter iter;
- use_operand_p use_p;
tree new_dataref_ptr;
if (TREE_CODE (dataref_ptr) == SSA_NAME)
@@ -6092,20 +6072,6 @@ bump_vector_ptr (vec_info *vinfo,
/* Copy the points-to information if it exists. */
duplicate_ssa_name_ptr_info (new_dataref_ptr, DR_PTR_INFO (dr));
- if (!ptr_incr)
- return new_dataref_ptr;
-
- /* Update the vector-pointer's cross-iteration increment. */
- FOR_EACH_SSA_USE_OPERAND (use_p, ptr_incr, iter, SSA_OP_USE)
- {
- tree use = USE_FROM_PTR (use_p);
-
- if (use == dataref_ptr)
- SET_USE (use_p, new_dataref_ptr);
- else
- gcc_assert (operand_equal_p (use, update, 0));
- }
-
return new_dataref_ptr;
}
diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
index cd1ea746ae4..6264e977bec 100644
--- a/gcc/tree-vect-loop-manip.cc
+++ b/gcc/tree-vect-loop-manip.cc
@@ -514,7 +514,6 @@ vect_set_loop_controls_directly (class loop *loop,
loop_vec_info loop_vinfo,
tree ctrl_type = rgc->type;
unsigned int nitems_per_iter = rgc->max_nscalars_per_iter * rgc->factor;
poly_uint64 nitems_per_ctrl = TYPE_VECTOR_SUBPARTS (ctrl_type) * rgc->factor;
- poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
tree length_limit = NULL_TREE;
/* For length, we need length_limit to ensure length in range. */
if (!use_masks_p)
@@ -525,7 +524,11 @@ vect_set_loop_controls_directly (class loop *loop,
loop_vec_info loop_vinfo,
of the vector loop, and the number that it should skip during the
first iteration of the vector loop. */
tree nitems_total = niters;
- tree nitems_step = build_int_cst (iv_type, vf);
+ tree nitems_vf
+ = build_int_cst (iv_type, LOOP_VINFO_VECT_FACTOR (loop_vinfo));
+ tree nitems_step
+ = gimple_convert (&loop_cond_gsi, true, GSI_SAME_STMT, UNKNOWN_LOCATION,
+ iv_type, LOOP_VINFO_IV_INCREMENT (loop_vinfo));
tree nitems_skip = niters_skip;
if (nitems_per_iter != 1)
{
@@ -535,8 +538,11 @@ vect_set_loop_controls_directly (class loop *loop,
loop_vec_info loop_vinfo,
tree iv_factor = build_int_cst (iv_type, nitems_per_iter);
nitems_total = gimple_build (preheader_seq, MULT_EXPR, compare_type,
nitems_total, compare_factor);
- nitems_step = gimple_build (preheader_seq, MULT_EXPR, iv_type,
- nitems_step, iv_factor);
+ nitems_vf = gimple_build (preheader_seq, MULT_EXPR, iv_type,
+ nitems_vf, iv_factor);
+ nitems_step
+ = gimple_build (&loop_cond_gsi, true, GSI_SAME_STMT, UNKNOWN_LOCATION,
+ MULT_EXPR, iv_type, nitems_step, iv_factor);
if (nitems_skip)
nitems_skip = gimple_build (preheader_seq, MULT_EXPR, compare_type,
nitems_skip, compare_factor);
@@ -578,22 +584,27 @@ vect_set_loop_controls_directly (class loop *loop,
loop_vec_info loop_vinfo,
insert_after, &index_before_incr, &index_after_incr);
tree vectype = build_zero_cst (rgc->type);
tree len = gimple_build (header_seq, IFN_SELECT_VL, iv_type,
- index_before_incr, nitems_step,
+ index_before_incr, nitems_vf,
vectype);
gimple_seq_add_stmt (header_seq, gimple_build_assign (step, len));
+ /* Also set the LOOP_VINFO_IV_INCREMENT. */
+ gassign* assign_iv_increment
+ = gimple_build_assign (LOOP_VINFO_IV_INCREMENT (loop_vinfo), len);
+ gimple_seq_add_stmt (header_seq, assign_iv_increment);
}
else
{
+ gsi_insert_before (&incr_gsi,
+ gimple_build_assign (step, MIN_EXPR,
+ index_before_incr,
+ nitems_step),
+ GSI_SAME_STMT);
create_iv (nitems_total, MINUS_EXPR, nitems_step, NULL_TREE, loop,
&incr_gsi, insert_after, &index_before_incr,
&index_after_incr);
- gimple_seq_add_stmt (header_seq,
- gimple_build_assign (step, MIN_EXPR,
- index_before_incr,
- nitems_step));
}
*iv_step = step;
- *compare_step = nitems_step;
+ *compare_step = nitems_vf;
return LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo) ? index_after_incr
: index_before_incr;
}
@@ -636,7 +647,7 @@ vect_set_loop_controls_directly (class loop *loop,
loop_vec_info loop_vinfo,
COMPARE_TYPE. */
test_index = index_before_incr;
tree adjust = gimple_convert (preheader_seq, compare_type,
- nitems_step);
+ nitems_vf);
if (nitems_skip)
adjust = gimple_build (preheader_seq, MINUS_EXPR, compare_type,
adjust, nitems_skip);
@@ -1040,14 +1051,18 @@ vect_set_loop_condition_partial_vectors_avx512 (class
loop *loop,
iv_type, niters, skip);
}
- /* The iteration step is the vectorization factor. */
- tree iv_step = build_int_cst (iv_type, vf);
-
- /* Create the decrement IV. */
- tree index_before_incr, index_after_incr;
gimple_stmt_iterator incr_gsi;
+ tree index_before_incr, index_after_incr;
bool insert_after;
vect_iv_increment_position (exit_edge, &incr_gsi, &insert_after);
+
+ /* The iteration step is the vectorization factor. */
+ tree iv_step = gimple_convert (&incr_gsi, insert_after,
+ insert_after ? GSI_NEW_STMT : GSI_SAME_STMT,
+ UNKNOWN_LOCATION, iv_type,
+ LOOP_VINFO_IV_INCREMENT (loop_vinfo));
+
+ /* Create the decrement IV. */
create_iv (niters_adj, MINUS_EXPR, iv_step, NULL_TREE, loop,
&incr_gsi, insert_after, &index_before_incr,
&index_after_incr);
@@ -2911,7 +2926,8 @@ vect_gen_vector_loop_niters (loop_vec_info loop_vinfo,
tree niters,
/* To silence some unexpected warnings, simply initialize to 0. */
unsigned HOST_WIDE_INT const_vf = 0;
if (vf.is_constant (&const_vf)
- && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
+ && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
+ && is_gimple_constant (LOOP_VINFO_IV_INCREMENT (loop_vinfo)))
{
/* Create: niters / vf, which is equivalent to niters >> log2(vf) when
vf is a power of two, and when not we approximate using a
@@ -2938,7 +2954,7 @@ vect_gen_vector_loop_niters (loop_vec_info loop_vinfo,
tree niters,
else
{
niters_vector = ni_minus_gap;
- step_vector = build_int_cst (type, vf);
+ step_vector = fold_convert (type, LOOP_VINFO_IV_INCREMENT (loop_vinfo));
}
if (!is_gimple_val (niters_vector))
@@ -2986,6 +3002,23 @@ vect_gen_vector_loop_niters (loop_vec_info loop_vinfo,
tree niters,
return;
}
+/* Finds the amount IV's should be incremented by each iteration.
+ Stored in LOOP_VINFO_IV_INCREMENT. */
+
+tree
+vect_get_loop_iv_increment (loop_vec_info loop_vinfo)
+{
+ if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
+ {
+ /* Fill this in later when building the loop controls. */
+ tree iv_increment = make_temp_ssa_name (sizetype, NULL, "iv_increment");
+ SSA_NAME_DEF_STMT (iv_increment) = gimple_build_nop ();
+ return iv_increment;
+ }
+ else
+ return build_int_cst (sizetype, LOOP_VINFO_VECT_FACTOR (loop_vinfo));
+}
+
/* Given NITERS_VECTOR which is the number of iterations for vectorized
loop specified by LOOP_VINFO after vectorization, compute the number
of iterations before vectorization (niters_vector * vf) and store it
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index ac7e08cf205..787f1d45c0b 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -9516,9 +9516,7 @@ vectorizable_induction (loop_vec_info loop_vinfo,
tree new_name;
gphi *induction_phi;
tree induc_def, vec_dest;
- poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
unsigned i;
- tree expr;
tree index_vectype = NULL_TREE;
gimple_stmt_iterator si;
enum vect_induction_op_type induction_type
@@ -9760,11 +9758,13 @@ vectorizable_induction (loop_vec_info loop_vinfo,
{
if (SCALAR_FLOAT_TYPE_P (stept))
{
- tree tem = build_int_cst (integer_type_node, vf);
+ tree tem = gimple_convert (&init_stmts, integer_type_node,
+ LOOP_VINFO_IV_INCREMENT (loop_vinfo));
lupdate_mul = gimple_build (&init_stmts, FLOAT_EXPR, stept, tem);
}
else
- lupdate_mul = build_int_cst (stept, vf);
+ lupdate_mul = gimple_convert (&init_stmts, stept,
+ LOOP_VINFO_IV_INCREMENT (loop_vinfo));
lupdate_mul = gimple_build_vector_from_val (&init_stmts, step_vectype,
lupdate_mul);
}
@@ -9896,36 +9896,8 @@ vectorizable_induction (loop_vec_info loop_vinfo,
/* Create the iv update inside the loop */
tree up = vec_step;
if (lupdate_mul)
- {
- if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
- {
- /* When we're using loop_len produced by SELEC_VL, the
- non-final iterations are not always processing VF
- elements. So vectorize induction variable instead of
-
- _21 = vect_vec_iv_.6_22 + { VF, ... };
-
- We should generate:
-
- _35 = .SELECT_VL (ivtmp_33, VF);
- vect_cst__22 = [vec_duplicate_expr] _35;
- _21 = vect_vec_iv_.6_22 + vect_cst__22; */
- vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
- tree len = vect_get_loop_len (loop_vinfo, NULL, lens, 1,
- vectype, 0, 0, false);
- if (SCALAR_FLOAT_TYPE_P (stept))
- expr = gimple_build (&stmts, FLOAT_EXPR, stept, len);
- else
- expr = gimple_convert (&stmts, stept, len);
- lupdate_mul = gimple_build_vector_from_val (&stmts, step_vectype,
- expr);
- up = gimple_build (&stmts, MULT_EXPR,
- step_vectype, vec_step, lupdate_mul);
- }
- else
- up = gimple_build (&init_stmts, MULT_EXPR, step_vectype,
- vec_step, lupdate_mul);
- }
+ up = gimple_build (&init_stmts, MULT_EXPR, step_vectype, vec_step,
+ lupdate_mul);
vec_def = gimple_convert (&stmts, step_vectype, induc_def);
vec_def = gimple_build (&stmts, PLUS_EXPR, step_vectype, vec_def, up);
vec_def = gimple_convert (&stmts, vectype, vec_def);
@@ -11065,7 +11037,6 @@ vect_update_ivs_after_vectorizer_for_early_breaks
(loop_vec_info loop_vinfo)
tree phi_var = LOOP_VINFO_EARLY_BRK_NITERS_VAR (loop_vinfo);
tree niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
- poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
tree ty_var = TREE_TYPE (phi_var);
auto loop = LOOP_VINFO_LOOP (loop_vinfo);
tree induc_var = niters_skip ? copy_ssa_name (phi_var) : phi_var;
@@ -11077,27 +11048,15 @@ vect_update_ivs_after_vectorizer_for_early_breaks
(loop_vec_info loop_vinfo)
gimple_seq init_stmts = NULL;
gimple_seq stmts = NULL;
gimple_seq iv_stmts = NULL;
- tree tree_vf = build_int_cst (ty_var, vf);
-
- /* For loop len targets we have to use .SELECT_VL (ivtmp_33, VF); instead of
- just += VF as the VF can change in between two loop iterations. */
- if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
- {
- vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
- tree_vf = vect_get_loop_len (loop_vinfo, NULL, lens, 1,
- NULL_TREE, 0, 0, true);
- }
+ tree tree_iv_incr = LOOP_VINFO_IV_INCREMENT (loop_vinfo);
tree iter_var;
if (POINTER_TYPE_P (ty_var))
- {
- tree offset = gimple_convert (&stmts, sizetype, tree_vf);
- iter_var = gimple_build (&stmts, POINTER_PLUS_EXPR, ty_var, induc_def,
- gimple_convert (&stmts, sizetype, offset));
- }
+ iter_var = gimple_build (&stmts, POINTER_PLUS_EXPR, ty_var, induc_def,
+ tree_iv_incr);
else
{
- tree offset = gimple_convert (&stmts, ty_var, tree_vf);
+ tree offset = gimple_convert (&stmts, ty_var, tree_iv_incr);
iter_var = gimple_build (&stmts, PLUS_EXPR, ty_var, induc_def, offset);
}
@@ -11144,10 +11103,14 @@ vect_update_ivs_after_vectorizer_for_early_breaks
(loop_vec_info loop_vinfo)
/* Write the init_stmts in the loop-preheader block. */
auto psi = gsi_last_nondebug_bb (pe->src);
gsi_insert_seq_after (&psi, init_stmts, GSI_LAST_NEW_STMT);
- /* Wite the adjustments in the header block. */
- basic_block bb = loop->header;
- auto si = gsi_after_labels (bb);
- gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT);
+
+ /* Write the adjustments at the end of the iv increment. */
+ bool insert_after;
+ gimple_stmt_iterator incr_gsi;
+ vect_iv_increment_position (LOOP_VINFO_MAIN_EXIT (loop_vinfo), &incr_gsi,
+ &insert_after);
+
+ gsi_insert_seq_before (&incr_gsi, stmts, GSI_SAME_STMT);
}
/* Function vect_transform_loop.
@@ -11241,6 +11204,9 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple
*loop_vectorized_call)
bool niters_no_overflow = uncounted_p ? false /* Not known. */
: loop_niters_no_overflow (loop_vinfo);
+ LOOP_VINFO_IV_INCREMENT (loop_vinfo)
+ = vect_get_loop_iv_increment (loop_vinfo);
+
epilogue = vect_do_peeling (loop_vinfo, niters, nitersm1, &niters_vector,
&step_vector, &niters_vector_mult_vf, th,
check_profitability, niters_no_overflow,
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index da87b329715..7f1f2c4da5c 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -3169,44 +3169,32 @@ vect_get_gather_scatter_ops (class loop *loop, slp_tree
slp_node,
the gather load or scatter store operation described by GS_INFO.
STMT_INFO is the load or store statement.
- Set *DATAREF_BUMP to the amount that should be added to the base
- address after each copy of the vectorized statement. Set *VEC_OFFSET
- to an invariant offset vector in which element I has the value
- I * DR_STEP / SCALE. */
+ Set *DR_INCREMENT to the amount that should be added to pointer base address
+ to get to the next iteration's base address.
+ Set *DR_BUMP to the amount that should be added to the base
+ address after each copy of the vectorized statement in a grouped read.
+ Set *VEC_OFFSET to an invariant offset vector in which element I has the
+ value I * DR_STEP / SCALE. */
static void
vect_get_strided_load_store_ops (stmt_vec_info stmt_info, slp_tree node,
tree vectype, tree offset_vectype,
loop_vec_info loop_vinfo,
gimple_stmt_iterator *gsi,
- tree *dataref_bump, tree *vec_offset,
- vec_loop_lens *loop_lens)
+ tree *dr_increment, tree *dr_bump,
+ tree *vec_offset)
{
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
- if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
- {
- /* _31 = .SELECT_VL (ivtmp_29, POLY_INT_CST [4, 4]);
- ivtmp_8 = _31 * 16 (step in bytes);
- .MASK_LEN_SCATTER_STORE (vectp_a.9_7, ... );
- vectp_a.9_26 = vectp_a.9_7 + ivtmp_8; */
- tree loop_len
- = vect_get_loop_len (loop_vinfo, gsi, loop_lens, 1, vectype, 0, 0,
true);
- tree tmp
- = fold_build2 (MULT_EXPR, sizetype,
- fold_convert (sizetype, unshare_expr (DR_STEP (dr))),
- loop_len);
- *dataref_bump = force_gimple_operand_gsi (gsi, tmp, true, NULL_TREE,
true,
- GSI_SAME_STMT);
- }
- else
- {
- tree bump
- = size_binop (MULT_EXPR,
- fold_convert (sizetype, unshare_expr (DR_STEP (dr))),
- size_int (TYPE_VECTOR_SUBPARTS (vectype)));
- *dataref_bump = cse_and_gimplify_to_preheader (loop_vinfo, bump);
- }
+ tree increment = size_binop (MULT_EXPR,
+ fold_convert (sizetype, unshare_expr (DR_STEP (dr))),
+ LOOP_VINFO_IV_INCREMENT (loop_vinfo));
+ *dr_increment = force_gimple_operand_gsi (gsi, increment, false, NULL_TREE,
+ true, GSI_SAME_STMT);
+ tree bump = size_binop (MULT_EXPR,
+ fold_convert (sizetype, unshare_expr (DR_STEP (dr))),
+ size_int (TYPE_VECTOR_SUBPARTS (vectype)));
+ *dr_bump = cse_and_gimplify_to_preheader (loop_vinfo, bump);
internal_fn ifn
= DR_IS_READ (dr) ? IFN_MASK_LEN_STRIDED_LOAD : IFN_MASK_LEN_STRIDED_STORE;
@@ -3232,47 +3220,33 @@ vect_get_strided_load_store_ops (stmt_vec_info
stmt_info, slp_tree node,
*vec_offset = cse_and_gimplify_to_preheader (loop_vinfo, offset);
}
-/* Prepare the pointer IVs which needs to be updated by a variable amount.
- Such variable amount is the outcome of .SELECT_VL. In this case, we can
- allow each iteration process the flexible number of elements as long as
- the number <= vf elments.
-
- Return data reference according to SELECT_VL.
- If new statements are needed, insert them before GSI. */
+/* Return the amount that should be added to a vector pointer to increment to
+ the next loop iteration. */
static tree
-vect_get_loop_variant_data_ptr_increment (
- vec_info *vinfo, tree aggr_type, gimple_stmt_iterator *gsi,
- vec_loop_lens *loop_lens, dr_vec_info *dr_info,
- vect_memory_access_type memory_access_type)
+vect_get_data_ptr_increment (vec_info *vinfo, gimple_stmt_iterator *gsi,
+ dr_vec_info *dr_info, tree aggr_type,
+ vect_memory_access_type memory_access_type)
{
+ if (memory_access_type == VMAT_INVARIANT)
+ return size_zero_node;
+
loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
- tree step = vect_dr_behavior (vinfo, dr_info)->step;
+ if (!loop_vinfo)
+ {
+ /* For slp there is no next iteration? */
+ tree iv_step = TYPE_SIZE_UNIT (aggr_type);
+ return build_zero_cst (TREE_TYPE (iv_step));
+ }
+ tree step = vect_dr_behavior (loop_vinfo, dr_info)->step;
/* gather/scatter never reach here. */
gcc_assert (!mat_gather_scatter_p (memory_access_type));
- /* When we support SELECT_VL pattern, we dynamic adjust
- the memory address by .SELECT_VL result.
+ tree iv_increment = LOOP_VINFO_IV_INCREMENT (loop_vinfo);
- The result of .SELECT_VL is the number of elements to
- be processed of each iteration. So the memory address
- adjustment operation should be:
-
- addr = addr + .SELECT_VL (ARG..) * step;
- */
- tree loop_len
- = vect_get_loop_len (loop_vinfo, gsi, loop_lens, 1, aggr_type, 0, 0, true);
- tree len_type = TREE_TYPE (loop_len);
- /* Since the outcome of .SELECT_VL is element size, we should adjust
- it into bytesize so that it can be used in address pointer variable
- amount IVs adjustment. */
- tree tmp = fold_build2 (MULT_EXPR, len_type, loop_len,
- wide_int_to_tree (len_type, wi::to_widest (step)));
- tree bump = make_temp_ssa_name (len_type, NULL, "ivtmp");
- gassign *assign = gimple_build_assign (bump, tmp);
- gsi_insert_before (gsi, assign, GSI_SAME_STMT);
- return bump;
+ return gimple_build (gsi, true, GSI_SAME_STMT, UNKNOWN_LOCATION,
+ MULT_EXPR, sizetype, iv_increment, step);
}
/* Return the amount that should be added to a vector pointer to move
@@ -3281,19 +3255,18 @@ vect_get_loop_variant_data_ptr_increment (
vectorization. */
static tree
-vect_get_data_ptr_increment (vec_info *vinfo, gimple_stmt_iterator *gsi,
- dr_vec_info *dr_info, tree aggr_type,
- vect_memory_access_type memory_access_type,
- vec_loop_lens *loop_lens)
+vect_get_data_ptr_bump (vec_info *vinfo,
+ dr_vec_info *dr_info, tree aggr_type,
+ vect_memory_access_type memory_access_type)
{
if (memory_access_type == VMAT_INVARIANT)
return size_zero_node;
+ /* We do not support num_vec != 1 for SELECT_VL so this value should never be
+ used. */
loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
if (loop_vinfo && LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
- return vect_get_loop_variant_data_ptr_increment (vinfo, aggr_type, gsi,
- loop_lens, dr_info,
- memory_access_type);
+ return NULL_TREE;
tree iv_step = TYPE_SIZE_UNIT (aggr_type);
tree step = vect_dr_behavior (vinfo, dr_info)->step;
@@ -7944,19 +7917,12 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info
stmt_info,
perms[i] = vect_gen_perm_mask_checked (vectype, indices);
}
- vec_loop_lens *loop_lens
- = (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)
- ? &LOOP_VINFO_LENS (loop_vinfo)
- : NULL);
-
tree vec_oprnd1 = NULL_TREE;
tree vec_oprnd2 = NULL_TREE;
tree vec_oprnd3 = NULL_TREE;
tree dataref_ptr = DR_BASE_ADDRESS (dr_info->dr);
tree dataref_offset = build_int_cst (ref_type, 0);
- tree bump = vect_get_data_ptr_increment (vinfo, gsi, dr_info,
- vectype, VMAT_CONTIGUOUS,
- loop_lens);
+ tree bump = vect_get_data_ptr_bump (vinfo, dr_info, vectype,
VMAT_CONTIGUOUS);
tree ldataref_ptr = NULL_TREE;
tree orig = NULL_TREE;
if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && !inscan_var_store)
@@ -8130,7 +8096,6 @@ vectorizable_store (vec_info *vinfo,
enum vect_def_type mask_dt = vect_unknown_def_type;
tree dataref_ptr = NULL_TREE;
tree dataref_offset = NULL_TREE;
- gimple *ptr_incr = NULL;
int j;
stmt_vec_info first_stmt_info;
bool grouped_store;
@@ -8554,13 +8519,21 @@ vectorizable_store (vec_info *vinfo,
if (!costing_p)
{
ivstep = stride_step;
+
+ tree increment = gimple_convert (&incr_gsi, false, GSI_NEW_STMT,
+ UNKNOWN_LOCATION, TREE_TYPE (ivstep),
+ LOOP_VINFO_IV_INCREMENT
+ (loop_vinfo));
+
ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (ivstep), ivstep,
- build_int_cst (TREE_TYPE (ivstep), vf));
+ increment);
standard_iv_increment_position (loop, &incr_gsi, &insert_after);
stride_base = cse_and_gimplify_to_preheader (loop_vinfo, stride_base);
- ivstep = cse_and_gimplify_to_preheader (loop_vinfo, ivstep);
+ ivstep = force_gimple_operand_gsi (&incr_gsi, unshare_expr (ivstep),
+ true, NULL_TREE, true,
+ GSI_SAME_STMT);
create_iv (stride_base, PLUS_EXPR, ivstep, NULL, loop, &incr_gsi,
insert_after, &offvar, NULL);
@@ -8701,12 +8674,15 @@ vectorizable_store (vec_info *vinfo,
if (!known_eq (poffset, 0))
offset = size_int (poffset);
- tree bump;
+ tree dr_increment;
+ tree dr_bump;
+
tree vec_offset = NULL_TREE;
if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
{
aggr_type = NULL_TREE;
- bump = NULL_TREE;
+ dr_increment = NULL_TREE;
+ dr_bump = NULL_TREE;
}
else if (mat_gather_scatter_p (memory_access_type))
{
@@ -8717,7 +8693,8 @@ vectorizable_store (vec_info *vinfo,
vect_get_strided_load_store_ops (stmt_info, slp_node, vtype,
ls.strided_offset_vectype,
loop_vinfo, gsi,
- &bump, &vec_offset, loop_lens);
+ &dr_increment, &dr_bump,
+ &vec_offset);
}
}
else
@@ -8727,8 +8704,13 @@ vectorizable_store (vec_info *vinfo,
else
aggr_type = vectype;
if (!costing_p)
- bump = vect_get_data_ptr_increment (vinfo, gsi, dr_info, aggr_type,
- memory_access_type, loop_lens);
+ {
+ dr_increment = vect_get_data_ptr_increment (vinfo, gsi, dr_info,
+ aggr_type,
+ memory_access_type);
+ dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
+ memory_access_type);
+ }
}
if (loop_vinfo && mask_node && !costing_p)
@@ -8781,7 +8763,7 @@ vectorizable_store (vec_info *vinfo,
dataref_ptr
= vect_create_data_ref_ptr (vinfo, first_stmt_info,
aggr_type, NULL, offset, &dummy,
- gsi, &ptr_incr, false, bump);
+ gsi, NULL, false, dr_increment);
}
}
else if (!costing_p)
@@ -8789,8 +8771,8 @@ vectorizable_store (vec_info *vinfo,
gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
if (mask_node)
vec_mask = vec_masks[j];
- dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
- stmt_info, bump);
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
+ stmt_info, dr_bump);
}
if (costing_p)
@@ -8937,16 +8919,16 @@ vectorizable_store (vec_info *vinfo,
dataref_ptr
= vect_create_data_ref_ptr (vinfo, first_stmt_info,
aggr_type, NULL, offset,
- &dummy, gsi, &ptr_incr, false,
- bump);
+ &dummy, gsi, NULL, false,
+ dr_increment);
}
}
else if (!costing_p)
{
gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
if (!STMT_VINFO_GATHER_SCATTER_P (stmt_info))
- dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr,
- gsi, stmt_info, bump);
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr,
+ gsi, stmt_info, dr_bump);
}
new_stmt = NULL;
@@ -9341,8 +9323,8 @@ vectorizable_store (vec_info *vinfo,
else if (!costing_p)
dataref_ptr = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
simd_lane_access_p ? loop : NULL,
- offset, &dummy, gsi, &ptr_incr,
- simd_lane_access_p, bump);
+ offset, &dummy, gsi, NULL,
+ simd_lane_access_p, dr_increment);
new_stmt = NULL;
gcc_assert (!grouped_store);
@@ -9392,8 +9374,8 @@ vectorizable_store (vec_info *vinfo,
if (i > 0)
/* Bump the vector pointer. */
- dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
- stmt_info, bump);
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
+ dr_bump);
unsigned misalign;
unsigned HOST_WIDE_INT align;
@@ -9716,7 +9698,6 @@ vectorizable_load (vec_info *vinfo,
tree dummy;
tree dataref_ptr = NULL_TREE;
tree dataref_offset = NULL_TREE;
- gimple *ptr_incr = NULL;
int i, j;
unsigned int group_size;
poly_uint64 group_gap_adj;
@@ -10225,8 +10206,12 @@ vectorizable_load (vec_info *vinfo,
vectemp = {tmp1, tmp2, ...}
*/
+ tree vf_tree
+ = gimple_convert (&incr_gsi, false, GSI_NEW_STMT, UNKNOWN_LOCATION,
+ TREE_TYPE (stride_step),
+ LOOP_VINFO_IV_INCREMENT (loop_vinfo));
ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (stride_step), stride_step,
- build_int_cst (TREE_TYPE (stride_step), vf));
+ vf_tree);
standard_iv_increment_position (loop, &incr_gsi, &insert_after);
@@ -10701,7 +10686,8 @@ vectorizable_load (vec_info *vinfo,
? size_binop (PLUS_EXPR, offset, size_int (poffset))
: size_int (poffset));
- tree bump;
+ tree dr_increment;
+ tree dr_bump;
tree vec_offset = NULL_TREE;
auto_vec<tree> vec_offsets;
@@ -10721,8 +10707,13 @@ vectorizable_load (vec_info *vinfo,
aggr_type = build_array_type_nelts (elem_type, group_size * nunits);
if (!costing_p)
- bump = vect_get_data_ptr_increment (vinfo, gsi, dr_info, aggr_type,
- memory_access_type, loop_lens);
+ {
+ dr_increment = vect_get_data_ptr_increment (vinfo, gsi, dr_info,
+ aggr_type,
+ memory_access_type);
+ dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
+ memory_access_type);
+ }
unsigned int inside_cost = 0, prologue_cost = 0;
/* For costing some adjacent vector loads, we'd like to cost with
@@ -10769,12 +10760,12 @@ vectorizable_load (vec_info *vinfo,
dataref_ptr
= vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
at_loop, offset, &dummy, gsi,
- &ptr_incr, false, bump);
+ NULL, false, dr_increment);
else
{
gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
- dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
- stmt_info, bump);
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
+ stmt_info, dr_bump);
}
if (mask_node)
vec_mask = vec_masks[j];
@@ -10900,7 +10891,7 @@ vectorizable_load (vec_info *vinfo,
if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
{
aggr_type = NULL_TREE;
- bump = NULL_TREE;
+ dr_increment = NULL_TREE;
if (!costing_p)
vect_get_gather_scatter_ops (loop, slp_node, &dataref_ptr,
&vec_offsets);
@@ -10913,11 +10904,12 @@ vectorizable_load (vec_info *vinfo,
vect_get_strided_load_store_ops (stmt_info, slp_node, vectype,
ls.strided_offset_vectype,
loop_vinfo, gsi,
- &bump, &vec_offset, loop_lens);
+ &dr_increment, &dr_bump,
+ &vec_offset);
dataref_ptr
= vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
at_loop, offset, &dummy, gsi,
- &ptr_incr, false, bump);
+ NULL, false, dr_increment);
}
}
@@ -10941,8 +10933,8 @@ vectorizable_load (vec_info *vinfo,
final_mask, vec_mask, gsi);
if (i > 0 && !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
- dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr,
- gsi, stmt_info, bump);
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
+ stmt_info, dr_bump);
}
/* 2. Create the vector-load in the loop. */
@@ -11326,8 +11318,13 @@ vectorizable_load (vec_info *vinfo,
aggr_type = vectype;
if (!costing_p)
- bump = vect_get_data_ptr_increment (vinfo, gsi, dr_info, aggr_type,
- memory_access_type, loop_lens);
+ {
+ dr_increment
+ = vect_get_data_ptr_increment (vinfo, gsi, dr_info, aggr_type,
+ memory_access_type);
+ dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
+ memory_access_type);
+ }
poly_uint64 group_elt = 0;
unsigned int inside_cost = 0, prologue_cost = 0;
@@ -11358,8 +11355,8 @@ vectorizable_load (vec_info *vinfo,
dataref_ptr
= vect_create_data_ref_ptr (vinfo, first_stmt_info_for_drptr,
aggr_type, at_loop, offset, &dummy,
- gsi, &ptr_incr, simd_lane_access_p,
- bump);
+ gsi, NULL, simd_lane_access_p,
+ dr_increment);
/* Adjust the pointer by the difference to first_stmt. */
data_reference_p ptrdr
= STMT_VINFO_DATA_REF (first_stmt_info_for_drptr);
@@ -11367,7 +11364,7 @@ vectorizable_load (vec_info *vinfo,
size_binop (MINUS_EXPR,
DR_INIT (first_dr_info->dr),
DR_INIT (ptrdr)));
- dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
stmt_info, diff);
if (alignment_support_scheme == dr_explicit_realign)
{
@@ -11383,17 +11380,17 @@ vectorizable_load (vec_info *vinfo,
dataref_ptr
= vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
at_loop,
- offset, &dummy, gsi, &ptr_incr,
- simd_lane_access_p, bump);
+ offset, &dummy, gsi, NULL,
+ simd_lane_access_p, dr_increment);
}
else if (!costing_p)
{
gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
if (dataref_offset)
- dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset, bump);
+ dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset, dr_bump);
else
- dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
- stmt_info, bump);
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
+ dr_bump);
}
auto_vec<tree> dr_chain;
@@ -11419,8 +11416,8 @@ vectorizable_load (vec_info *vinfo,
final_mask, vec_mask, gsi);
if (i > 0)
- dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr,
- gsi, stmt_info, bump);
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
+ dr_bump);
}
/* 2. Create the vector-load in the loop. */
@@ -11766,8 +11763,8 @@ vectorizable_load (vec_info *vinfo,
bump = size_binop (MULT_EXPR, vs, TYPE_SIZE_UNIT (elem_type));
bump = size_binop (MINUS_EXPR, bump, size_one_node);
- ptr = bump_vector_ptr (vinfo, dataref_ptr, NULL, gsi, stmt_info,
- bump);
+ /* !!! MIGHT BE BROKEN? */
+ ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info, bump);
new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR, ptr,
build_int_cst (TREE_TYPE (ptr),
-(HOST_WIDE_INT)
align));
@@ -11924,8 +11921,8 @@ vectorizable_load (vec_info *vinfo,
if (tree_int_cst_sgn (vect_dr_behavior (vinfo, dr_info)->step) == -1)
bump_val = -bump_val;
tree bump = wide_int_to_tree (sizetype, bump_val);
- dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
- stmt_info, bump);
+ dataref_ptr
+ = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info, bump);
group_elt = 0;
}
}
@@ -11940,8 +11937,7 @@ vectorizable_load (vec_info *vinfo,
if (tree_int_cst_sgn (vect_dr_behavior (vinfo, dr_info)->step) == -1)
bump_val = -bump_val;
tree bump = wide_int_to_tree (sizetype, bump_val);
- dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
- stmt_info, bump);
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info, bump);
}
if (ls.slp_perm)
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 3a01e1be0f1..64df5757f6f 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -975,6 +975,10 @@ public:
it can be that no unrolling is needed, and thus this is 1. */
poly_uint64 vectorization_factor;
+ /* Gimple operand for the number of scalar iteration handed per loop
+ iteration, and therefore how much to increment each IV by. */
+ tree iv_increment;
+
/* If this loop is an epilogue loop whose main loop can be skipped,
MAIN_LOOP_EDGE is the edge from the main loop to this loop's
preheader. SKIP_MAIN_LOOP_EDGE is then the edge that skips the
@@ -1295,6 +1299,7 @@ public:
#define LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT(L) (L)->allow_mutual_alignment
#define LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS(L) (L)->partial_load_store_bias
#define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor
+#define LOOP_VINFO_IV_INCREMENT(L) (L)->iv_increment
#define LOOP_VINFO_MAX_VECT_FACTOR(L) (L)->max_vectorization_factor
#define LOOP_VINFO_MASKS(L) (L)->masks
#define LOOP_VINFO_LENS(L) (L)->lens
@@ -2641,7 +2646,7 @@ extern tree vect_create_data_ref_ptr (vec_info *,
tree *, gimple_stmt_iterator *,
gimple **, bool,
tree = NULL_TREE);
-extern tree bump_vector_ptr (vec_info *, tree, gimple *, gimple_stmt_iterator
*,
+extern tree bump_vector_ptr (vec_info *, tree, gimple_stmt_iterator *,
stmt_vec_info, tree);
extern void vect_copy_ref_info (tree, tree);
extern tree vect_create_destination_var (tree, tree);
@@ -2675,6 +2680,7 @@ extern opt_loop_vec_info vect_analyze_loop (class loop *,
gimple *,
extern tree vect_build_loop_niters (loop_vec_info, bool * = NULL);
extern void vect_gen_vector_loop_niters (loop_vec_info, tree, tree *,
tree *, bool);
+extern tree vect_get_loop_iv_increment (loop_vec_info);
extern tree vect_halve_mask_nunits (tree, machine_mode);
extern tree vect_double_mask_nunits (tree, machine_mode);
extern void vect_record_loop_mask (loop_vec_info, vec_loop_masks *,
--
2.34.1
