On Thu, Sep 14, 2023 at 5:12 AM Kewen Lin <[email protected]> wrote:
>
> This patch is an initial patch to move costing next to the
> transform, it still adopts vect_model_store_cost for costing
> but moves and duplicates it down according to the handlings
> of different vect_memory_access_types or some special
> handling need, hope it can make the subsequent patches easy
> to review. This patch should not have any functional
> changes.
OK
> gcc/ChangeLog:
>
> * tree-vect-stmts.cc (vectorizable_store): Move and duplicate the call
> to vect_model_store_cost down to some different transform paths
> according to the handlings of different vect_memory_access_types
> or some special handling need.
> ---
> gcc/tree-vect-stmts.cc | 79 ++++++++++++++++++++++++++++++++----------
> 1 file changed, 60 insertions(+), 19 deletions(-)
>
> diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
> index a5caaf0bca2..36f7c5b9f4b 100644
> --- a/gcc/tree-vect-stmts.cc
> +++ b/gcc/tree-vect-stmts.cc
> @@ -8372,7 +8372,8 @@ vectorizable_store (vec_info *vinfo,
> return false;
> }
>
> - if (!vec_stmt) /* transformation not required. */
> + bool costing_p = !vec_stmt;
> + if (costing_p) /* transformation not required. */
> {
> STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) = memory_access_type;
>
> @@ -8401,11 +8402,6 @@ vectorizable_store (vec_info *vinfo,
> "Vectorizing an unaligned access.\n");
>
> STMT_VINFO_TYPE (stmt_info) = store_vec_info_type;
> - vect_model_store_cost (vinfo, stmt_info, ncopies,
> - memory_access_type, &gs_info,
> - alignment_support_scheme,
> - misalignment, vls_type, slp_node, cost_vec);
> - return true;
> }
> gcc_assert (memory_access_type == STMT_VINFO_MEMORY_ACCESS_TYPE
> (stmt_info));
>
> @@ -8415,12 +8411,27 @@ vectorizable_store (vec_info *vinfo,
>
> if (memory_access_type == VMAT_GATHER_SCATTER && gs_info.decl)
> {
> - vect_build_scatter_store_calls (vinfo, stmt_info, gsi, vec_stmt,
> - &gs_info, mask);
> + if (costing_p)
> + vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
> + &gs_info, alignment_support_scheme,
> misalignment,
> + vls_type, slp_node, cost_vec);
> + else
> + vect_build_scatter_store_calls (vinfo, stmt_info, gsi, vec_stmt,
> + &gs_info, mask);
> return true;
> }
> else if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) >= 3)
> - return vectorizable_scan_store (vinfo, stmt_info, gsi, vec_stmt,
> ncopies);
> + {
> + gcc_assert (memory_access_type == VMAT_CONTIGUOUS);
> + if (costing_p)
> + {
> + vect_model_store_cost (vinfo, stmt_info, ncopies,
> memory_access_type,
> + &gs_info, alignment_support_scheme,
> + misalignment, vls_type, slp_node, cost_vec);
> + return true;
> + }
> + return vectorizable_scan_store (vinfo, stmt_info, gsi, vec_stmt,
> ncopies);
> + }
>
> if (grouped_store)
> {
> @@ -8449,13 +8460,21 @@ vectorizable_store (vec_info *vinfo,
> else
> ref_type = reference_alias_ptr_type (DR_REF (first_dr_info->dr));
>
> - if (dump_enabled_p ())
> - dump_printf_loc (MSG_NOTE, vect_location,
> - "transform store. ncopies = %d\n", ncopies);
> + if (!costing_p && dump_enabled_p ())
> + dump_printf_loc (MSG_NOTE, vect_location, "transform store. ncopies =
> %d\n",
> + ncopies);
>
> if (memory_access_type == VMAT_ELEMENTWISE
> || memory_access_type == VMAT_STRIDED_SLP)
> {
> + if (costing_p)
> + {
> + vect_model_store_cost (vinfo, stmt_info, ncopies,
> memory_access_type,
> + &gs_info, alignment_support_scheme,
> + misalignment, vls_type, slp_node, cost_vec);
> + return true;
> + }
> +
> gimple_stmt_iterator incr_gsi;
> bool insert_after;
> gimple *incr;
> @@ -8718,8 +8737,9 @@ vectorizable_store (vec_info *vinfo,
> else if (memory_access_type == VMAT_GATHER_SCATTER)
> {
> aggr_type = elem_type;
> - vect_get_strided_load_store_ops (stmt_info, loop_vinfo, gsi, &gs_info,
> - &bump, &vec_offset, loop_lens);
> + if (!costing_p)
> + vect_get_strided_load_store_ops (stmt_info, loop_vinfo, gsi, &gs_info,
> + &bump, &vec_offset, loop_lens);
> }
> else
> {
> @@ -8731,7 +8751,7 @@ vectorizable_store (vec_info *vinfo,
> memory_access_type, loop_lens);
> }
>
> - if (mask)
> + if (mask && !costing_p)
> LOOP_VINFO_HAS_MASK_STORE (loop_vinfo) = true;
>
> /* In case the vectorization factor (VF) is bigger than the number
> @@ -8782,6 +8802,13 @@ vectorizable_store (vec_info *vinfo,
> if (memory_access_type == VMAT_LOAD_STORE_LANES)
> {
> gcc_assert (!slp && grouped_store);
> + if (costing_p)
> + {
> + vect_model_store_cost (vinfo, stmt_info, ncopies,
> memory_access_type,
> + &gs_info, alignment_support_scheme,
> + misalignment, vls_type, slp_node, cost_vec);
> + return true;
> + }
> for (j = 0; j < ncopies; j++)
> {
> gimple *new_stmt;
> @@ -8927,6 +8954,13 @@ vectorizable_store (vec_info *vinfo,
> if (memory_access_type == VMAT_GATHER_SCATTER)
> {
> gcc_assert (!slp && !grouped_store);
> + if (costing_p)
> + {
> + vect_model_store_cost (vinfo, stmt_info, ncopies,
> memory_access_type,
> + &gs_info, alignment_support_scheme,
> + misalignment, vls_type, slp_node, cost_vec);
> + return true;
> + }
> auto_vec<tree> vec_offsets;
> for (j = 0; j < ncopies; j++)
> {
> @@ -9091,7 +9125,7 @@ vectorizable_store (vec_info *vinfo,
> for (j = 0; j < ncopies; j++)
> {
> gimple *new_stmt;
> - if (j == 0)
> + if (j == 0 && !costing_p)
> {
> if (slp)
> {
> @@ -9158,7 +9192,7 @@ vectorizable_store (vec_info *vinfo,
> offset, &dummy, gsi, &ptr_incr,
> simd_lane_access_p, bump);
> }
> - else
> + else if (!costing_p)
> {
> gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
> /* DR_CHAIN is then used as an input to vect_permute_store_chain().
> @@ -9179,7 +9213,7 @@ vectorizable_store (vec_info *vinfo,
> }
>
> new_stmt = NULL;
> - if (grouped_store)
> + if (!costing_p && grouped_store)
> /* Permute. */
> vect_permute_store_chain (vinfo, dr_chain, group_size, stmt_info, gsi,
> &result_chain);
> @@ -9187,6 +9221,8 @@ vectorizable_store (vec_info *vinfo,
> stmt_vec_info next_stmt_info = first_stmt_info;
> for (i = 0; i < vec_num; i++)
> {
> + if (costing_p)
> + continue;
> unsigned misalign;
> unsigned HOST_WIDE_INT align;
>
> @@ -9361,7 +9397,7 @@ vectorizable_store (vec_info *vinfo,
> if (!next_stmt_info)
> break;
> }
> - if (!slp)
> + if (!slp && !costing_p)
> {
> if (j == 0)
> *vec_stmt = new_stmt;
> @@ -9369,6 +9405,11 @@ vectorizable_store (vec_info *vinfo,
> }
> }
>
> + if (costing_p)
> + vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
> + &gs_info, alignment_support_scheme, misalignment,
> + vls_type, slp_node, cost_vec);
> +
> return true;
> }
>
> --
> 2.31.1
>
