https://gcc.gnu.org/g:4e0964279b73d3af5b085f731c87c200b7564db4
commit r17-2050-g4e0964279b73d3af5b085f731c87c200b7564db4 Author: Alfie Richards <[email protected]> Date: Wed May 20 14:50:25 2026 +0000 vect: Introduce LOOP_VINFO_IV_INCREMENT Introduce LOOP_VINFO_IV_INCREMENT which stores the number of scalar iterations an iteration of the vectorized loop has processed. Then updates IV updates and reductions to use this value instead of VF. Update IV update logic so the IV_INCREMENT can be loop-variant. Simplify/remove SELECT_VL IV increment special cases by instead using IV_INCREMENT. As part of this change, the grouped load/store data pointer logic changes from: ADDRESS_1 = PHI <... , ADDRESS_3> V1 = LOAD ADDRESS_1 ADDRESS_2 = ADDRESS_1 + BUMP V2 = LOAD ADDRESS_2 ADDRESS_3 = ADDRESS_2 + BUMP to: ADDRESS_1 = PHI <... , ADDRESS_3> V1 = LOAD ADDRESS_1 ADDRESS_2 = ADDRESS_1 + BUMP V2 = LOAD ADDRESS_2 ADDRESS_3 = ADDRESS_1 + STEP * IV_INCREMENT gcc/ChangeLog: * tree-ssa-loop-manip.h: Add argument to create_iv. * tree-ssa-loop-manip.cc (create_iv): Change step to not necessarily be loop invariant. * tree-vect-data-refs.cc (bump_vector_ptr): Remove logic for updating use-def chain. * tree-vect-loop-manip.cc (vect_set_loop_controls_directly): Change to use LOOP_VINFO_IV_INCREMENT instead of VF. (vect_set_loop_condition_partial_vectors_avx512): Change to use LOOP_VINFO_IV_INCREMENT instead of VF. (vect_gen_vector_loop_niters): Update to handle LOOP_VINFO_IV_INCREMENT. (vect_get_loop_iv_increment): New function. * tree-vect-loop.cc (vectorizable_induction): Update to use LOOP_VINFO_IV_INCREMENT instead of VF, and remove SELECT_VL logic. (vect_update_ivs_after_vectorizer_for_early_breaks): Update to use LOOP_VINFO_IV_INCREMENT over VF and move IV update to vect_iv_increment_position. (vect_transform_loop): Add initialization of LOOP_VINFO_IV_INCREMENT. * tree-vect-stmts.cc (vect_get_strided_load_store_ops): Update to use dr_increment and dr_bump, remove SELECT_VL logic. (vect_get_data_ptr_increment): Change to return the increment needed to advance the pointer to the next iteration. (vect_get_data_ptr_bump): New function. (vectorizable_scan_store): Update to use vect_get_data_ptr_bump and to remove SELECT_VL logic. (vectorizable_store): Update to use vect_get_data_ptr_bump and vect_get_data_ptr_increment. (vectorizable_load): Update to use vect_get_data_ptr_bump and vect_get_data_ptr_increment. * tree-vectorizer.h (loop_vec_info): Add iv_increment field. (LOOP_VINFO_IV_INCREMENT): New macro. (LOOP_VINFO_IV_INCREMENT_INVARIANT_P): New macro. (bump_vector_ptr): Remove incr_ptr argument. (vect_get_loop_iv_increment): New function. gcc/testsuite/ChangeLog: * gcc.target/riscv/rvv/autovec/pr112399.c: Update for different scheduling. Diff: --- .../gcc.target/riscv/rvv/autovec/pr112399.c | 2 +- gcc/tree-ssa-loop-manip.cc | 21 +- gcc/tree-ssa-loop-manip.h | 3 +- gcc/tree-vect-data-refs.cc | 44 +--- gcc/tree-vect-loop-manip.cc | 88 +++++-- gcc/tree-vect-loop.cc | 113 ++++----- gcc/tree-vect-stmts.cc | 256 ++++++++++----------- gcc/tree-vectorizer.h | 10 +- 8 files changed, 281 insertions(+), 256 deletions(-) diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/pr112399.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/pr112399.c index 948e12b84745..3e5b9457d68e 100644 --- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/pr112399.c +++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/pr112399.c @@ -7,12 +7,12 @@ ** foo: ** ... ** vsetvli\s*[a-x0-9]+,\s*[a-x0-9]+,\s*e64,\s*m1,\s*tu,\s*m[au] -** slli\s*[a-x0-9]+,\s*[a-x0-9]+,\s*3 ** vle64\.v\s*v[0-9]+,\s*0\([a-x0-9]+\) ** vle64\.v\s*v[0-9]+,\s*0\([a-x0-9]+\) ** vfmul\.vv\s*v[0-9]+,\s*v[0-9]+,\s*v[0-9]+ ** vle64\.v\s*\s*v[0-9]+,\s*0\([a-x0-9]+\) ** vfmacc\.vv\s*\s*v[0-9]+,\s*v[0-9]+,\s*v[0-9]+ +** slli\s*[a-x0-9]+,\s*[a-x0-9]+,\s*3 ** add\s*[a-x0-9]+,\s*[a-x0-9]+,\s*[a-x0-9]+ ** add\s*[a-x0-9]+,\s*[a-x0-9]+,\s*[a-x0-9]+ ** add\s*[a-x0-9]+,\s*[a-x0-9]+,\s*[a-x0-9]+ diff --git a/gcc/tree-ssa-loop-manip.cc b/gcc/tree-ssa-loop-manip.cc index df6cd8629bcf..486667bd937a 100644 --- a/gcc/tree-ssa-loop-manip.cc +++ b/gcc/tree-ssa-loop-manip.cc @@ -89,12 +89,14 @@ insert_iv_increment (gimple_stmt_iterator *incr_pos, bool after, INCR_POS (after it if AFTER is true, before it otherwise). INCR_POS and AFTER can be computed using standard_iv_increment_position. The ssa versions of the variable before and after increment will be stored in VAR_BEFORE and - VAR_AFTER (unless they are NULL). */ + VAR_AFTER (unless they are NULL). If LOOP_INVARIANT is true then the + calculation step will be added to the pre-header. Otherwise, the step + calculation is added to incr_pos. */ void create_iv (tree base, tree_code incr_op, tree step, tree var, class loop *loop, gimple_stmt_iterator *incr_pos, bool after, tree *var_before, - tree *var_after) + tree *var_after, bool loop_invariant) { gphi *phi; tree initial, step1; @@ -150,13 +152,18 @@ create_iv (tree base, tree_code incr_op, tree step, tree var, class loop *loop, step = fold_build1 (NEGATE_EXPR, TREE_TYPE (step), step); incr_op = POINTER_PLUS_EXPR; } - /* Gimplify the step if necessary. We put the computations in front of the - loop (i.e. the step should be loop invariant). */ - step = force_gimple_operand (step, &stmts, true, NULL_TREE); - if (stmts) - gsi_insert_seq_on_edge_immediate (pe, stmts); gimple_seq incr_stmts = nullptr; + /* Gimplify the step if necessary. */ + if (loop_invariant) + { + step = force_gimple_operand (step, &stmts, true, NULL_TREE); + if (stmts) + gsi_insert_seq_on_edge_immediate (pe, stmts); + } + else + step = force_gimple_operand (step, &incr_stmts, true, NULL_TREE); + gimple_seq_add_stmt (&incr_stmts, gimple_build_assign (va, incr_op, vb, step)); insert_iv_increment (incr_pos, after, incr_stmts); diff --git a/gcc/tree-ssa-loop-manip.h b/gcc/tree-ssa-loop-manip.h index e5289ea0cd79..521aac2dbac1 100644 --- a/gcc/tree-ssa-loop-manip.h +++ b/gcc/tree-ssa-loop-manip.h @@ -24,7 +24,8 @@ typedef void (*transform_callback)(class loop *, void *); extern void insert_iv_increment (gimple_stmt_iterator *, bool, gimple_seq); extern void create_iv (tree, tree_code, tree, tree, class loop *, - gimple_stmt_iterator *, bool, tree *, tree *); + gimple_stmt_iterator *, bool, tree *, tree *, + bool = true); extern void rewrite_into_loop_closed_ssa (bitmap, unsigned); extern void verify_loop_closed_ssa (bool, class loop * = NULL); diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc index 6311e0d03df2..ab850de024bf 100644 --- a/gcc/tree-vect-data-refs.cc +++ b/gcc/tree-vect-data-refs.cc @@ -6015,7 +6015,9 @@ vect_create_data_ref_ptr (vec_info *vinfo, stmt_vec_info stmt_info, create_iv (aggr_ptr_init, PLUS_EXPR, iv_step, aggr_ptr, loop, &incr_gsi, insert_after, - &indx_before_incr, &indx_after_incr); + &indx_before_incr, &indx_after_incr, + !loop_vinfo + || LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)); incr = gsi_stmt (incr_gsi); /* Copy the points-to information if it exists. */ @@ -6059,30 +6061,12 @@ vect_create_data_ref_ptr (vec_info *vinfo, stmt_vec_info stmt_info, /* Function bump_vector_ptr - Increment a pointer (to a vector type) by vector-size. If requested, - i.e. if PTR-INCR is given, then also connect the new increment stmt - to the existing def-use update-chain of the pointer, by modifying - the PTR_INCR as illustrated below: - - The pointer def-use update-chain before this function: - DATAREF_PTR = phi (p_0, p_2) - .... - PTR_INCR: p_2 = DATAREF_PTR + step - - The pointer def-use update-chain after this function: - DATAREF_PTR = phi (p_0, p_2) - .... - NEW_DATAREF_PTR = DATAREF_PTR + BUMP - .... - PTR_INCR: p_2 = NEW_DATAREF_PTR + step + Increment DATAREF_PTR by UPDATE. Input: DATAREF_PTR - ssa_name of a pointer (to vector type) that is being updated in the loop. - PTR_INCR - optional. The stmt that updates the pointer in each iteration of - the loop. The increment amount across iterations is expected - to be vector_size. - BSI - location where the new update stmt is to be placed. + GSI - location where the new update stmt is to be placed. STMT_INFO - the original scalar memory-access stmt that is being vectorized. UPDATE - The offset by which to bump the pointer. @@ -6092,13 +6076,11 @@ vect_create_data_ref_ptr (vec_info *vinfo, stmt_vec_info stmt_info, tree bump_vector_ptr (vec_info *vinfo, - tree dataref_ptr, gimple *ptr_incr, gimple_stmt_iterator *gsi, + tree dataref_ptr, gimple_stmt_iterator *gsi, stmt_vec_info stmt_info, tree update) { struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); gimple *incr_stmt; - ssa_op_iter iter; - use_operand_p use_p; tree new_dataref_ptr; if (TREE_CODE (dataref_ptr) == SSA_NAME) @@ -6129,20 +6111,6 @@ bump_vector_ptr (vec_info *vinfo, /* Copy the points-to information if it exists. */ duplicate_ssa_name_ptr_info (new_dataref_ptr, DR_PTR_INFO (dr)); - if (!ptr_incr) - return new_dataref_ptr; - - /* Update the vector-pointer's cross-iteration increment. */ - FOR_EACH_SSA_USE_OPERAND (use_p, ptr_incr, iter, SSA_OP_USE) - { - tree use = USE_FROM_PTR (use_p); - - if (use == dataref_ptr) - SET_USE (use_p, new_dataref_ptr); - else - gcc_assert (operand_equal_p (use, update, 0)); - } - return new_dataref_ptr; } diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc index a64a79d3299f..ff90a21c8dd2 100644 --- a/gcc/tree-vect-loop-manip.cc +++ b/gcc/tree-vect-loop-manip.cc @@ -514,7 +514,6 @@ vect_set_loop_controls_directly (class loop *loop, loop_vec_info loop_vinfo, tree ctrl_type = rgc->type; unsigned int nitems_per_iter = rgc->max_nscalars_per_iter * rgc->factor; poly_uint64 nitems_per_ctrl = TYPE_VECTOR_SUBPARTS (ctrl_type) * rgc->factor; - poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); tree length_limit = NULL_TREE; /* For length, we need length_limit to ensure length in range. */ if (!use_masks_p) @@ -525,7 +524,15 @@ vect_set_loop_controls_directly (class loop *loop, loop_vec_info loop_vinfo, of the vector loop, and the number that it should skip during the first iteration of the vector loop. */ tree nitems_total = niters; - tree nitems_step = build_int_cst (iv_type, vf); + tree nitems_vf + = build_int_cst (iv_type, LOOP_VINFO_VECT_FACTOR (loop_vinfo)); + tree nitems_step + = LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo) + ? gimple_convert (preheader_seq, iv_type, + LOOP_VINFO_IV_INCREMENT (loop_vinfo)) + : gimple_convert (&loop_cond_gsi, true, GSI_SAME_STMT, UNKNOWN_LOCATION, + iv_type, LOOP_VINFO_IV_INCREMENT (loop_vinfo)); + tree nitems_skip = niters_skip; if (nitems_per_iter != 1) { @@ -535,8 +542,14 @@ vect_set_loop_controls_directly (class loop *loop, loop_vec_info loop_vinfo, tree iv_factor = build_int_cst (iv_type, nitems_per_iter); nitems_total = gimple_build (preheader_seq, MULT_EXPR, compare_type, nitems_total, compare_factor); - nitems_step = gimple_build (preheader_seq, MULT_EXPR, iv_type, - nitems_step, iv_factor); + nitems_vf = gimple_build (preheader_seq, MULT_EXPR, iv_type, + nitems_vf, iv_factor); + nitems_step = LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo) + ? gimple_build (preheader_seq, MULT_EXPR, iv_type, + nitems_step, iv_factor) + : gimple_build (&loop_cond_gsi, true, GSI_SAME_STMT, + UNKNOWN_LOCATION, MULT_EXPR, iv_type, + nitems_step, iv_factor); if (nitems_skip) nitems_skip = gimple_build (preheader_seq, MULT_EXPR, compare_type, nitems_skip, compare_factor); @@ -578,9 +591,21 @@ vect_set_loop_controls_directly (class loop *loop, loop_vec_info loop_vinfo, insert_after, &index_before_incr, &index_after_incr); tree vectype = build_zero_cst (rgc->type); tree len = gimple_build (header_seq, IFN_SELECT_VL, iv_type, - index_before_incr, nitems_step, + index_before_incr, nitems_vf, vectype); gimple_seq_add_stmt (header_seq, gimple_build_assign (step, len)); + len = gimple_convert (header_seq, sizetype, len); + + /* Remove the previous initialization of IV_INCREMENT to VARYING. */ + gimple *varying_def + = SSA_NAME_DEF_STMT (LOOP_VINFO_IV_INCREMENT (loop_vinfo)); + auto def_gsi = gsi_for_stmt (varying_def); + gsi_remove (&def_gsi, true); + + /* Set the LOOP_VINFO_IV_INCREMENT to be len. */ + gassign* assign_iv_increment + = gimple_build_assign (LOOP_VINFO_IV_INCREMENT (loop_vinfo), len); + gimple_seq_add_stmt (header_seq, assign_iv_increment); } else { @@ -593,7 +618,7 @@ vect_set_loop_controls_directly (class loop *loop, loop_vec_info loop_vinfo, nitems_step)); } *iv_step = step; - *compare_step = nitems_step; + *compare_step = nitems_vf; return LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo) ? index_after_incr : index_before_incr; } @@ -601,7 +626,8 @@ vect_set_loop_controls_directly (class loop *loop, loop_vec_info loop_vinfo, /* Create increment IV. */ create_iv (build_int_cst (iv_type, 0), PLUS_EXPR, nitems_step, NULL_TREE, loop, &incr_gsi, insert_after, &index_before_incr, - &index_after_incr); + &index_after_incr, + LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)); tree zero_index = build_int_cst (compare_type, 0); tree test_index, test_limit, first_limit; @@ -636,7 +662,7 @@ vect_set_loop_controls_directly (class loop *loop, loop_vec_info loop_vinfo, COMPARE_TYPE. */ test_index = index_before_incr; tree adjust = gimple_convert (preheader_seq, compare_type, - nitems_step); + nitems_vf); if (nitems_skip) adjust = gimple_build (preheader_seq, MINUS_EXPR, compare_type, adjust, nitems_skip); @@ -1040,14 +1066,16 @@ vect_set_loop_condition_partial_vectors_avx512 (class loop *loop, iv_type, niters, skip); } - /* The iteration step is the vectorization factor. */ - tree iv_step = build_int_cst (iv_type, vf); - - /* Create the decrement IV. */ - tree index_before_incr, index_after_incr; gimple_stmt_iterator incr_gsi; + tree index_before_incr, index_after_incr; bool insert_after; vect_iv_increment_position (exit_edge, &incr_gsi, &insert_after); + + /* The iteration step is the vectorization factor. */ + tree iv_step = gimple_convert (&preheader_seq, iv_type, + LOOP_VINFO_IV_INCREMENT (loop_vinfo)); + + /* Create the decrement IV. */ create_iv (niters_adj, MINUS_EXPR, iv_step, NULL_TREE, loop, &incr_gsi, insert_after, &index_before_incr, &index_after_incr); @@ -1230,7 +1258,7 @@ vect_set_loop_condition_partial_vectors_avx512 (class loop *loop, loop handles exactly VF scalars per iteration. */ static gcond * -vect_set_loop_condition_normal (loop_vec_info /* loop_vinfo */, edge exit_edge, +vect_set_loop_condition_normal (loop_vec_info loop_vinfo, edge exit_edge, class loop *loop, tree niters, tree step, tree final_iv, bool niters_maybe_zero, gimple_stmt_iterator loop_cond_gsi) @@ -1325,7 +1353,10 @@ vect_set_loop_condition_normal (loop_vec_info /* loop_vinfo */, edge exit_edge, vect_iv_increment_position (exit_edge, &incr_gsi, &insert_after); create_iv (init, PLUS_EXPR, step, NULL_TREE, loop, - &incr_gsi, insert_after, &indx_before_incr, &indx_after_incr); + &incr_gsi, insert_after, + &indx_before_incr, &indx_after_incr, + LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)); + indx_after_incr = force_gimple_operand_gsi (&loop_cond_gsi, indx_after_incr, true, NULL_TREE, true, GSI_SAME_STMT); @@ -2979,7 +3010,8 @@ vect_gen_vector_loop_niters (loop_vec_info loop_vinfo, tree niters, /* To silence some unexpected warnings, simply initialize to 0. */ unsigned HOST_WIDE_INT const_vf = 0; if (vf.is_constant (&const_vf) - && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)) + && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) + && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)) { /* Create: niters / vf, which is equivalent to niters >> log2(vf) when vf is a power of two, and when not we approximate using a @@ -3054,6 +3086,30 @@ vect_gen_vector_loop_niters (loop_vec_info loop_vinfo, tree niters, return; } +/* Finds the amount IV's should be incremented by each iteration. + Stored in LOOP_VINFO_IV_INCREMENT. */ + +tree +vect_get_loop_iv_increment (loop_vec_info loop_vinfo) +{ + if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo)) + { + /* For now, set this as varying. + Fill this in later when building the loop controls. */ + tree iv_increment = make_temp_ssa_name (sizetype, NULL, "iv_increment"); + + gcall *varying_call = gimple_build_call_internal (IFN_VARYING, 0); + gimple_call_set_lhs (varying_call, iv_increment); + + gimple_stmt_iterator gsi = gsi_after_labels (loop_vinfo->loop->header); + gsi_insert_before (&gsi, varying_call, GSI_NEW_STMT); + + return iv_increment; + } + else + return build_int_cst (sizetype, LOOP_VINFO_VECT_FACTOR (loop_vinfo)); +} + /* Given NITERS_VECTOR which is the number of iterations for vectorized loop specified by LOOP_VINFO after vectorization, compute the number of iterations before vectorization (niters_vector * vf) and store it diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc index 0167d52b28b2..931b942d7cb9 100644 --- a/gcc/tree-vect-loop.cc +++ b/gcc/tree-vect-loop.cc @@ -9524,9 +9524,7 @@ vectorizable_induction (loop_vec_info loop_vinfo, tree new_name; gphi *induction_phi; tree induc_def, vec_dest; - poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); unsigned i; - tree expr; tree index_vectype = NULL_TREE; gimple_stmt_iterator si; enum vect_induction_op_type induction_type @@ -9633,7 +9631,8 @@ vectorizable_induction (loop_vec_info loop_vinfo, if (!target_supports_op_p (step_vectype, PLUS_EXPR, optab_default) || !target_supports_op_p (step_vectype, MINUS_EXPR, optab_default)) return false; - if (!nunits.is_constant ()) + if (!nunits.is_constant () + || !LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)) { if (!target_supports_op_p (step_vectype, MULT_EXPR, optab_default)) return false; @@ -9734,7 +9733,8 @@ vectorizable_induction (loop_vec_info loop_vinfo, unsigned HOST_WIDE_INT const_nunits; if (nested_in_vect_loop) nivs = nvects; - else if (nunits.is_constant (&const_nunits)) + else if (nunits.is_constant (&const_nunits) + && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)) { /* Compute the number of distinct IVs we need. First reduce group_size if it is a multiple of const_nunits so we get @@ -9750,10 +9750,12 @@ vectorizable_induction (loop_vec_info loop_vinfo, nivs = 1; } gimple_seq init_stmts = NULL; + gimple_seq lupdate_mul_stmts = NULL; tree lupdate_mul = NULL_TREE; if (!nested_in_vect_loop) { - if (nunits.is_constant (&const_nunits)) + if (nunits.is_constant (&const_nunits) + && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)) { /* The number of iterations covered in one vector iteration. */ unsigned lup_mul = (nvects * const_nunits) / group_size; @@ -9766,14 +9768,23 @@ vectorizable_induction (loop_vec_info loop_vinfo, } else { + gimple_seq *update_stmts + = LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo) + ? &init_stmts + : &lupdate_mul_stmts; if (SCALAR_FLOAT_TYPE_P (stept)) { - tree tem = build_int_cst (integer_type_node, vf); - lupdate_mul = gimple_build (&init_stmts, FLOAT_EXPR, stept, tem); + tree increment + = gimple_convert (update_stmts, integer_type_node, + LOOP_VINFO_IV_INCREMENT (loop_vinfo)); + lupdate_mul = gimple_build (update_stmts, FLOAT_EXPR, stept, + increment); } else - lupdate_mul = build_int_cst (stept, vf); - lupdate_mul = gimple_build_vector_from_val (&init_stmts, step_vectype, + lupdate_mul = gimple_convert (update_stmts, stept, + LOOP_VINFO_IV_INCREMENT (loop_vinfo)); + lupdate_mul = gimple_build_vector_from_val (update_stmts, + step_vectype, lupdate_mul); } } @@ -9796,7 +9807,8 @@ vectorizable_induction (loop_vec_info loop_vinfo, { gimple_seq stmts = NULL; bool invariant = true; - if (nunits.is_constant (&const_nunits)) + if (nunits.is_constant (&const_nunits) + && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)) { tree_vector_builder step_elts (step_vectype, const_nunits, 1); tree_vector_builder init_elts (vectype, const_nunits, 1); @@ -9904,36 +9916,12 @@ vectorizable_induction (loop_vec_info loop_vinfo, /* Create the iv update inside the loop */ tree up = vec_step; if (lupdate_mul) - { - if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo)) - { - /* When we're using loop_len produced by SELEC_VL, the - non-final iterations are not always processing VF - elements. So vectorize induction variable instead of - - _21 = vect_vec_iv_.6_22 + { VF, ... }; - - We should generate: - - _35 = .SELECT_VL (ivtmp_33, VF); - vect_cst__22 = [vec_duplicate_expr] _35; - _21 = vect_vec_iv_.6_22 + vect_cst__22; */ - vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo); - tree len = vect_get_loop_len (loop_vinfo, NULL, lens, 1, - vectype, 0, 0, false); - if (SCALAR_FLOAT_TYPE_P (stept)) - expr = gimple_build (&stmts, FLOAT_EXPR, stept, len); - else - expr = gimple_convert (&stmts, stept, len); - lupdate_mul = gimple_build_vector_from_val (&stmts, step_vectype, - expr); - up = gimple_build (&stmts, MULT_EXPR, - step_vectype, vec_step, lupdate_mul); - } - else - up = gimple_build (&init_stmts, MULT_EXPR, step_vectype, - vec_step, lupdate_mul); - } + { + if (lupdate_mul_stmts) + gimple_seq_add_seq (&stmts, lupdate_mul_stmts); + up = gimple_build (&stmts, MULT_EXPR, step_vectype, vec_step, + lupdate_mul); + } vec_def = gimple_convert (&stmts, step_vectype, induc_def); vec_def = gimple_build (&stmts, PLUS_EXPR, step_vectype, vec_def, up); vec_def = gimple_convert (&stmts, vectype, vec_def); @@ -9964,7 +9952,8 @@ vectorizable_induction (loop_vec_info loop_vinfo, if (!nested_in_vect_loop) { /* Fill up to the number of vectors we need for the whole group. */ - if (nunits.is_constant (&const_nunits)) + if (nunits.is_constant (&const_nunits) + && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)) nivs = least_common_multiple (group_size, const_nunits) / const_nunits; else nivs = 1; @@ -9980,7 +9969,8 @@ vectorizable_induction (loop_vec_info loop_vinfo, stmts by adding VF' * stride to the IVs generated above. */ if (ivn < nvects) { - if (nunits.is_constant (&const_nunits)) + if (nunits.is_constant (&const_nunits) + && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)) { unsigned vfp = (least_common_multiple (group_size, const_nunits) / group_size); @@ -11073,7 +11063,6 @@ vect_update_ivs_after_vectorizer_for_early_breaks (loop_vec_info loop_vinfo) tree phi_var = LOOP_VINFO_EARLY_BRK_NITERS_VAR (loop_vinfo); tree niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo); - poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); tree ty_var = TREE_TYPE (phi_var); auto loop = LOOP_VINFO_LOOP (loop_vinfo); tree induc_var = niters_skip ? copy_ssa_name (phi_var) : phi_var; @@ -11090,27 +11079,15 @@ vect_update_ivs_after_vectorizer_for_early_breaks (loop_vec_info loop_vinfo) gimple_seq init_stmts = NULL; gimple_seq stmts = NULL; gimple_seq iv_stmts = NULL; - tree tree_vf = build_int_cst (ty_var, vf); - - /* For loop len targets we have to use .SELECT_VL (ivtmp_33, VF); instead of - just += VF as the VF can change in between two loop iterations. */ - if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo)) - { - vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo); - tree_vf = vect_get_loop_len (loop_vinfo, NULL, lens, 1, - NULL_TREE, 0, 0, true); - } + tree tree_iv_incr = LOOP_VINFO_IV_INCREMENT (loop_vinfo); tree iter_var; if (POINTER_TYPE_P (ty_var)) - { - tree offset = gimple_convert (&stmts, sizetype, tree_vf); - iter_var = gimple_build (&stmts, POINTER_PLUS_EXPR, ty_var, induc_def, - gimple_convert (&stmts, sizetype, offset)); - } + iter_var = gimple_build (&stmts, POINTER_PLUS_EXPR, ty_var, induc_def, + tree_iv_incr); else { - tree offset = gimple_convert (&stmts, ty_var, tree_vf); + tree offset = gimple_convert (&stmts, ty_var, tree_iv_incr); iter_var = gimple_build (&stmts, PLUS_EXPR, ty_var, induc_def, offset); } @@ -11157,10 +11134,17 @@ vect_update_ivs_after_vectorizer_for_early_breaks (loop_vec_info loop_vinfo) /* Write the init_stmts in the loop-preheader block. */ auto psi = gsi_last_nondebug_bb (pe->src); gsi_insert_seq_after (&psi, init_stmts, GSI_LAST_NEW_STMT); - /* Write the adjustments in the header block. */ - basic_block bb = loop->header; - auto si = gsi_after_labels (bb); - gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT); + + /* Write the adjustments at the end of the iv increment. */ + bool insert_after; + gimple_stmt_iterator incr_gsi; + vect_iv_increment_position (LOOP_VINFO_MAIN_EXIT (loop_vinfo), &incr_gsi, + &insert_after); + + if (insert_after) + gsi_insert_seq_after (&incr_gsi, stmts, GSI_NEW_STMT); + else + gsi_insert_seq_before (&incr_gsi, stmts, GSI_NEW_STMT); } /* Function vect_transform_loop. @@ -11259,6 +11243,9 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call) check_profitability, niters_no_overflow, &advance); + LOOP_VINFO_IV_INCREMENT (loop_vinfo) + = vect_get_loop_iv_increment (loop_vinfo); + /* Assign hierarchical discriminators to the vectorized loop. */ poly_uint64 vf_val = LOOP_VINFO_VECT_FACTOR (loop_vinfo); unsigned int vf_int = constant_lower_bound (vf_val); diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc index c7d91be895dc..76e24b2c1699 100644 --- a/gcc/tree-vect-stmts.cc +++ b/gcc/tree-vect-stmts.cc @@ -3170,44 +3170,35 @@ vect_get_gather_scatter_ops (class loop *loop, slp_tree slp_node, the gather load or scatter store operation described by GS_INFO. STMT_INFO is the load or store statement. - Set *DATAREF_BUMP to the amount that should be added to the base - address after each copy of the vectorized statement. Set *VEC_OFFSET - to an invariant offset vector in which element I has the value - I * DR_STEP / SCALE. */ + Set *DR_STEP to the amount that should be added to pointer base address + to get to the next iteration's base address. + Set *DR_BUMP to the amount that should be added to the base + address after each copy of the vectorized statement in a grouped read. + Set *VEC_OFFSET to an invariant offset vector in which element I has the + value I * DR_STEP / SCALE. */ static void vect_get_strided_load_store_ops (stmt_vec_info stmt_info, slp_tree node, tree vectype, tree offset_vectype, loop_vec_info loop_vinfo, gimple_stmt_iterator *gsi, - tree *dataref_bump, tree *vec_offset, - vec_loop_lens *loop_lens) + tree *dr_step, tree *dr_bump, + tree *vec_offset) { struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); - if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo)) - { - /* _31 = .SELECT_VL (ivtmp_29, POLY_INT_CST [4, 4]); - ivtmp_8 = _31 * 16 (step in bytes); - .MASK_LEN_SCATTER_STORE (vectp_a.9_7, ... ); - vectp_a.9_26 = vectp_a.9_7 + ivtmp_8; */ - tree loop_len - = vect_get_loop_len (loop_vinfo, gsi, loop_lens, 1, vectype, 0, 0, true); - tree tmp - = fold_build2 (MULT_EXPR, sizetype, - fold_convert (sizetype, unshare_expr (DR_STEP (dr))), - loop_len); - *dataref_bump = force_gimple_operand_gsi (gsi, tmp, true, NULL_TREE, true, - GSI_SAME_STMT); - } - else - { - tree bump - = size_binop (MULT_EXPR, - fold_convert (sizetype, unshare_expr (DR_STEP (dr))), - size_int (TYPE_VECTOR_SUBPARTS (vectype))); - *dataref_bump = cse_and_gimplify_to_preheader (loop_vinfo, bump); - } + tree dr_step_temp + = size_binop (MULT_EXPR, + fold_convert (sizetype, unshare_expr (DR_STEP (dr))), + LOOP_VINFO_IV_INCREMENT (loop_vinfo)); + *dr_step = LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo) + ? cse_and_gimplify_to_preheader (loop_vinfo, dr_step_temp) + : force_gimple_operand_gsi (gsi, dr_step_temp, false, NULL_TREE, + true, GSI_SAME_STMT); + tree bump = size_binop (MULT_EXPR, + fold_convert (sizetype, unshare_expr (DR_STEP (dr))), + size_int (TYPE_VECTOR_SUBPARTS (vectype))); + *dr_bump = cse_and_gimplify_to_preheader (loop_vinfo, bump); internal_fn ifn = DR_IS_READ (dr) ? IFN_MASK_LEN_STRIDED_LOAD : IFN_MASK_LEN_STRIDED_STORE; @@ -3233,47 +3224,31 @@ vect_get_strided_load_store_ops (stmt_vec_info stmt_info, slp_tree node, *vec_offset = cse_and_gimplify_to_preheader (loop_vinfo, offset); } -/* Prepare the pointer IVs which needs to be updated by a variable amount. - Such variable amount is the outcome of .SELECT_VL. In this case, we can - allow each iteration process the flexible number of elements as long as - the number <= vf elements. - - Return data reference according to SELECT_VL. - If new statements are needed, insert them before GSI. */ +/* Return the amount that should be added to a vector pointer, represented by + DR_INFO, to increment to the next vectorized iteration. */ static tree -vect_get_loop_variant_data_ptr_increment ( - vec_info *vinfo, tree aggr_type, gimple_stmt_iterator *gsi, - vec_loop_lens *loop_lens, dr_vec_info *dr_info, - vect_memory_access_type memory_access_type) +vect_get_data_ptr_step (vec_info *vinfo, dr_vec_info *dr_info, + vect_memory_access_type memory_access_type) { + if (memory_access_type == VMAT_INVARIANT) + return size_zero_node; + loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo); - tree step = vect_dr_behavior (vinfo, dr_info)->step; + + /* For BB SLP there is no next iteration. */ + if (!loop_vinfo) + return build_zero_cst (sizetype); + + tree step = vect_dr_behavior (loop_vinfo, dr_info)->step; /* gather/scatter never reach here. */ gcc_assert (!mat_gather_scatter_p (memory_access_type)); - /* When we support SELECT_VL pattern, we dynamic adjust - the memory address by .SELECT_VL result. + tree iv_increment = LOOP_VINFO_IV_INCREMENT (loop_vinfo); - The result of .SELECT_VL is the number of elements to - be processed of each iteration. So the memory address - adjustment operation should be: - - addr = addr + .SELECT_VL (ARG..) * step; - */ - tree loop_len - = vect_get_loop_len (loop_vinfo, gsi, loop_lens, 1, aggr_type, 0, 0, true); - tree len_type = TREE_TYPE (loop_len); - /* Since the outcome of .SELECT_VL is element size, we should adjust - it into bytesize so that it can be used in address pointer variable - amount IVs adjustment. */ - tree tmp = fold_build2 (MULT_EXPR, len_type, loop_len, - wide_int_to_tree (len_type, wi::to_widest (step))); - tree bump = make_temp_ssa_name (len_type, NULL, "ivtmp"); - gassign *assign = gimple_build_assign (bump, tmp); - gsi_insert_before (gsi, assign, GSI_SAME_STMT); - return bump; + return fold_build2 (MULT_EXPR, sizetype, iv_increment, + fold_convert (sizetype, step)); } /* Return the amount that should be added to a vector pointer to move @@ -3282,19 +3257,18 @@ vect_get_loop_variant_data_ptr_increment ( vectorization. */ static tree -vect_get_data_ptr_increment (vec_info *vinfo, gimple_stmt_iterator *gsi, - dr_vec_info *dr_info, tree aggr_type, - vect_memory_access_type memory_access_type, - vec_loop_lens *loop_lens) +vect_get_data_ptr_bump (vec_info *vinfo, + dr_vec_info *dr_info, tree aggr_type, + vect_memory_access_type memory_access_type) { if (memory_access_type == VMAT_INVARIANT) return size_zero_node; loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo); + /* We do not support SLP loads where num_vec != 1 with SELECT_VL so this value + should never be needed. */ if (loop_vinfo && LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo)) - return vect_get_loop_variant_data_ptr_increment (vinfo, aggr_type, gsi, - loop_lens, dr_info, - memory_access_type); + return NULL_TREE; tree iv_step = TYPE_SIZE_UNIT (aggr_type); tree step = vect_dr_behavior (vinfo, dr_info)->step; @@ -7945,19 +7919,12 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, perms[i] = vect_gen_perm_mask_checked (vectype, indices); } - vec_loop_lens *loop_lens - = (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo) - ? &LOOP_VINFO_LENS (loop_vinfo) - : NULL); - tree vec_oprnd1 = NULL_TREE; tree vec_oprnd2 = NULL_TREE; tree vec_oprnd3 = NULL_TREE; tree dataref_ptr = DR_BASE_ADDRESS (dr_info->dr); tree dataref_offset = build_int_cst (ref_type, 0); - tree bump = vect_get_data_ptr_increment (vinfo, gsi, dr_info, - vectype, VMAT_CONTIGUOUS, - loop_lens); + tree bump = vect_get_data_ptr_bump (vinfo, dr_info, vectype, VMAT_CONTIGUOUS); tree ldataref_ptr = NULL_TREE; tree orig = NULL_TREE; if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && !inscan_var_store) @@ -8131,7 +8098,6 @@ vectorizable_store (vec_info *vinfo, enum vect_def_type mask_dt = vect_unknown_def_type; tree dataref_ptr = NULL_TREE; tree dataref_offset = NULL_TREE; - gimple *ptr_incr = NULL; int j; stmt_vec_info first_stmt_info; bool grouped_store; @@ -8558,15 +8524,26 @@ vectorizable_store (vec_info *vinfo, if (!costing_p) { ivstep = stride_step; + + tree increment = fold_convert (TREE_TYPE (ivstep), + LOOP_VINFO_IV_INCREMENT (loop_vinfo)); + ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (ivstep), ivstep, - build_int_cst (TREE_TYPE (ivstep), vf)); + increment); standard_iv_increment_position (loop, &incr_gsi, &insert_after); stride_base = cse_and_gimplify_to_preheader (loop_vinfo, stride_base); - ivstep = cse_and_gimplify_to_preheader (loop_vinfo, ivstep); + if (LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)) + ivstep = cse_and_gimplify_to_preheader (loop_vinfo, ivstep); + else + ivstep = force_gimple_operand_gsi (&incr_gsi, unshare_expr (ivstep), + true, NULL_TREE, true, + GSI_SAME_STMT); + create_iv (stride_base, PLUS_EXPR, ivstep, NULL, loop, &incr_gsi, - insert_after, &offvar, NULL); + insert_after, &offvar, NULL, + LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)); stride_step = cse_and_gimplify_to_preheader (loop_vinfo, stride_step); } @@ -8705,12 +8682,15 @@ vectorizable_store (vec_info *vinfo, if (!known_eq (poffset, 0)) offset = size_int (poffset); - tree bump; + tree dr_increment; + tree dr_bump; + tree vec_offset = NULL_TREE; if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) { aggr_type = NULL_TREE; - bump = NULL_TREE; + dr_increment = NULL_TREE; + dr_bump = NULL_TREE; } else if (mat_gather_scatter_p (memory_access_type)) { @@ -8721,7 +8701,8 @@ vectorizable_store (vec_info *vinfo, vect_get_strided_load_store_ops (stmt_info, slp_node, vtype, ls.strided_offset_vectype, loop_vinfo, gsi, - &bump, &vec_offset, loop_lens); + &dr_increment, &dr_bump, + &vec_offset); } } else @@ -8731,8 +8712,12 @@ vectorizable_store (vec_info *vinfo, else aggr_type = vectype; if (!costing_p) - bump = vect_get_data_ptr_increment (vinfo, gsi, dr_info, aggr_type, - memory_access_type, loop_lens); + { + dr_increment = vect_get_data_ptr_step (vinfo, dr_info, + memory_access_type); + dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type, + memory_access_type); + } } if (loop_vinfo && mask_node && !costing_p) @@ -8785,7 +8770,7 @@ vectorizable_store (vec_info *vinfo, dataref_ptr = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type, NULL, offset, &dummy, - gsi, &ptr_incr, false, bump); + gsi, NULL, false, dr_increment); } } else if (!costing_p) @@ -8793,8 +8778,8 @@ vectorizable_store (vec_info *vinfo, gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo)); if (mask_node) vec_mask = vec_masks[j]; - dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi, - stmt_info, bump); + dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, + stmt_info, dr_bump); } if (costing_p) @@ -8941,16 +8926,16 @@ vectorizable_store (vec_info *vinfo, dataref_ptr = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type, NULL, offset, - &dummy, gsi, &ptr_incr, false, - bump); + &dummy, gsi, NULL, false, + dr_increment); } } else if (!costing_p) { gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo)); if (!STMT_VINFO_GATHER_SCATTER_P (stmt_info)) - dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, - gsi, stmt_info, bump); + dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, + gsi, stmt_info, dr_bump); } new_stmt = NULL; @@ -9347,8 +9332,8 @@ vectorizable_store (vec_info *vinfo, else if (!costing_p) dataref_ptr = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type, simd_lane_access_p ? loop : NULL, - offset, &dummy, gsi, &ptr_incr, - simd_lane_access_p, bump); + offset, &dummy, gsi, NULL, + simd_lane_access_p, dr_increment); new_stmt = NULL; gcc_assert (!grouped_store); @@ -9398,8 +9383,8 @@ vectorizable_store (vec_info *vinfo, if (i > 0) /* Bump the vector pointer. */ - dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi, - stmt_info, bump); + dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info, + dr_bump); unsigned misalign; unsigned HOST_WIDE_INT align; @@ -9722,7 +9707,6 @@ vectorizable_load (vec_info *vinfo, tree dummy; tree dataref_ptr = NULL_TREE; tree dataref_offset = NULL_TREE; - gimple *ptr_incr = NULL; int i, j; unsigned int group_size; poly_uint64 group_gap_adj; @@ -10231,16 +10215,22 @@ vectorizable_load (vec_info *vinfo, vectemp = {tmp1, tmp2, ...} */ - ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (stride_step), stride_step, - build_int_cst (TREE_TYPE (stride_step), vf)); + tree increment = fold_convert (TREE_TYPE (stride_step), + LOOP_VINFO_IV_INCREMENT (loop_vinfo)); + ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (stride_step), + stride_step, increment); standard_iv_increment_position (loop, &incr_gsi, &insert_after); stride_base = cse_and_gimplify_to_preheader (loop_vinfo, stride_base); - ivstep = cse_and_gimplify_to_preheader (loop_vinfo, ivstep); - create_iv (stride_base, PLUS_EXPR, ivstep, NULL, - loop, &incr_gsi, insert_after, - &offvar, NULL); + if (LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)) + ivstep = cse_and_gimplify_to_preheader (loop_vinfo, ivstep); + else + ivstep = force_gimple_operand_gsi (&incr_gsi, unshare_expr (ivstep), + true, NULL_TREE, true, + GSI_SAME_STMT); + create_iv (stride_base, PLUS_EXPR, ivstep, NULL, loop, &incr_gsi, + insert_after, &offvar, NULL, true); stride_step = cse_and_gimplify_to_preheader (loop_vinfo, stride_step); } @@ -10710,7 +10700,8 @@ vectorizable_load (vec_info *vinfo, ? size_binop (PLUS_EXPR, offset, size_int (poffset)) : size_int (poffset)); - tree bump; + tree dr_increment; + tree dr_bump; tree vec_offset = NULL_TREE; auto_vec<tree> vec_offsets; @@ -10730,8 +10721,12 @@ vectorizable_load (vec_info *vinfo, aggr_type = build_array_type_nelts (elem_type, group_size * nunits); if (!costing_p) - bump = vect_get_data_ptr_increment (vinfo, gsi, dr_info, aggr_type, - memory_access_type, loop_lens); + { + dr_increment = vect_get_data_ptr_step (vinfo, dr_info, + memory_access_type); + dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type, + memory_access_type); + } unsigned int inside_cost = 0, prologue_cost = 0; /* For costing some adjacent vector loads, we'd like to cost with @@ -10778,12 +10773,12 @@ vectorizable_load (vec_info *vinfo, dataref_ptr = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type, at_loop, offset, &dummy, gsi, - &ptr_incr, false, bump); + NULL, false, dr_increment); else { gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo)); - dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi, - stmt_info, bump); + dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, + stmt_info, dr_bump); } if (mask_node) vec_mask = vec_masks[j]; @@ -10909,7 +10904,7 @@ vectorizable_load (vec_info *vinfo, if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) { aggr_type = NULL_TREE; - bump = NULL_TREE; + dr_increment = NULL_TREE; if (!costing_p) vect_get_gather_scatter_ops (loop, slp_node, &dataref_ptr, &vec_offsets); @@ -10922,11 +10917,12 @@ vectorizable_load (vec_info *vinfo, vect_get_strided_load_store_ops (stmt_info, slp_node, vectype, ls.strided_offset_vectype, loop_vinfo, gsi, - &bump, &vec_offset, loop_lens); + &dr_increment, &dr_bump, + &vec_offset); dataref_ptr = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type, at_loop, offset, &dummy, gsi, - &ptr_incr, false, bump); + NULL, false, dr_increment); } } @@ -10950,8 +10946,8 @@ vectorizable_load (vec_info *vinfo, final_mask, vec_mask, gsi); if (i > 0 && !STMT_VINFO_GATHER_SCATTER_P (stmt_info)) - dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, - gsi, stmt_info, bump); + dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, + stmt_info, dr_bump); } /* 2. Create the vector-load in the loop. */ @@ -11336,8 +11332,12 @@ vectorizable_load (vec_info *vinfo, aggr_type = vectype; if (!costing_p) - bump = vect_get_data_ptr_increment (vinfo, gsi, dr_info, aggr_type, - memory_access_type, loop_lens); + { + dr_increment = vect_get_data_ptr_step (vinfo, dr_info, + memory_access_type); + dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type, + memory_access_type); + } poly_uint64 group_elt = 0; unsigned int inside_cost = 0, prologue_cost = 0; @@ -11368,8 +11368,8 @@ vectorizable_load (vec_info *vinfo, dataref_ptr = vect_create_data_ref_ptr (vinfo, first_stmt_info_for_drptr, aggr_type, at_loop, offset, &dummy, - gsi, &ptr_incr, simd_lane_access_p, - bump); + gsi, NULL, simd_lane_access_p, + dr_increment); /* Adjust the pointer by the difference to first_stmt. */ data_reference_p ptrdr = STMT_VINFO_DATA_REF (first_stmt_info_for_drptr); @@ -11377,7 +11377,7 @@ vectorizable_load (vec_info *vinfo, size_binop (MINUS_EXPR, DR_INIT (first_dr_info->dr), DR_INIT (ptrdr))); - dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi, + dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info, diff); if (alignment_support_scheme == dr_explicit_realign) { @@ -11393,8 +11393,8 @@ vectorizable_load (vec_info *vinfo, dataref_ptr = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type, at_loop, - offset, &dummy, gsi, &ptr_incr, - simd_lane_access_p, bump); + offset, &dummy, gsi, NULL, + simd_lane_access_p, dr_increment); } auto_vec<tree> dr_chain; @@ -11420,8 +11420,8 @@ vectorizable_load (vec_info *vinfo, final_mask, vec_mask, gsi); if (i > 0) - dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, - gsi, stmt_info, bump); + dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info, + dr_bump); } /* 2. Create the vector-load in the loop. */ @@ -11767,8 +11767,7 @@ vectorizable_load (vec_info *vinfo, bump = size_binop (MULT_EXPR, vs, TYPE_SIZE_UNIT (elem_type)); bump = size_binop (MINUS_EXPR, bump, size_one_node); - ptr = bump_vector_ptr (vinfo, dataref_ptr, NULL, gsi, stmt_info, - bump); + ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info, bump); new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR, ptr, build_int_cst (TREE_TYPE (ptr), -(HOST_WIDE_INT) align)); @@ -11925,8 +11924,8 @@ vectorizable_load (vec_info *vinfo, if (tree_int_cst_sgn (vect_dr_behavior (vinfo, dr_info)->step) == -1) bump_val = -bump_val; tree bump = wide_int_to_tree (sizetype, bump_val); - dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi, - stmt_info, bump); + dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info, + bump); group_elt = 0; } } @@ -11941,8 +11940,7 @@ vectorizable_load (vec_info *vinfo, if (tree_int_cst_sgn (vect_dr_behavior (vinfo, dr_info)->step) == -1) bump_val = -bump_val; tree bump = wide_int_to_tree (sizetype, bump_val); - dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi, - stmt_info, bump); + dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info, bump); } if (ls.slp_perm) diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 0b646d6b29e7..2e4756926f2a 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -978,6 +978,10 @@ public: it can be that no unrolling is needed, and thus this is 1. */ poly_uint64 vectorization_factor; + /* Gimple operand for the number of scalar iteration handed per loop + iteration, and therefore how much to increment each IV by. */ + tree iv_increment; + /* If this loop is an epilogue loop whose main loop can be skipped, MAIN_LOOP_EDGE is the edge from the main loop to this loop's preheader. SKIP_MAIN_LOOP_EDGE is then the edge that skips the @@ -1302,6 +1306,9 @@ public: #define LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT(L) (L)->allow_mutual_alignment #define LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS(L) (L)->partial_load_store_bias #define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor +#define LOOP_VINFO_IV_INCREMENT(L) (L)->iv_increment +#define LOOP_VINFO_IV_INCREMENT_INVARIANT_P(L) \ + (!LOOP_VINFO_USING_SELECT_VL_P (L)) #define LOOP_VINFO_MAX_VECT_FACTOR(L) (L)->max_vectorization_factor #define LOOP_VINFO_MASKS(L) (L)->masks #define LOOP_VINFO_LENS(L) (L)->lens @@ -2650,7 +2657,7 @@ extern tree vect_create_data_ref_ptr (vec_info *, tree *, gimple_stmt_iterator *, gimple **, bool, tree = NULL_TREE); -extern tree bump_vector_ptr (vec_info *, tree, gimple *, gimple_stmt_iterator *, +extern tree bump_vector_ptr (vec_info *, tree, gimple_stmt_iterator *, stmt_vec_info, tree); extern void vect_copy_ref_info (tree, tree); extern tree vect_create_destination_var (tree, tree); @@ -2684,6 +2691,7 @@ extern opt_loop_vec_info vect_analyze_loop (class loop *, gimple *, extern tree vect_build_loop_niters (loop_vec_info, bool * = NULL); extern void vect_gen_vector_loop_niters (loop_vec_info, tree, tree *, tree *, bool); +extern tree vect_get_loop_iv_increment (loop_vec_info); extern tree vect_halve_mask_nunits (tree, machine_mode); extern tree vect_double_mask_nunits (tree, machine_mode); extern void vect_record_loop_mask (loop_vec_info, vec_loop_masks *,
