On Tue, Aug 22, 2023 at 10:49 AM Kewen.Lin <li...@linux.ibm.com> wrote:
>
> Hi,
>
> Like commit r14-3214 which moves the handlings on memory
> access type VMAT_LOAD_STORE_LANES in vectorizable_load
> final loop nest, this one is to deal with the function
> vectorizable_store.
>
> Bootstrapped and regtested on x86_64-redhat-linux,
> aarch64-linux-gnu and powerpc64{,le}-linux-gnu.
>
> Is it ok for trunk?
OK.
> BR,
> Kewen
> -----
>
> gcc/ChangeLog:
>
> * tree-vect-stmts.cc (vectorizable_store): Move the handlings on
> VMAT_LOAD_STORE_LANES in the final loop nest to its own loop,
> and update the final nest accordingly.
> ---
> gcc/tree-vect-stmts.cc | 732 ++++++++++++++++++++++-------------------
> 1 file changed, 387 insertions(+), 345 deletions(-)
>
> diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
> index fcaa4127e52..18f5ebcc09c 100644
> --- a/gcc/tree-vect-stmts.cc
> +++ b/gcc/tree-vect-stmts.cc
> @@ -8779,42 +8779,29 @@ vectorizable_store (vec_info *vinfo,
> */
>
> auto_vec<tree> dr_chain (group_size);
> - auto_vec<tree> result_chain (group_size);
> auto_vec<tree> vec_masks;
> tree vec_mask = NULL;
> - auto_vec<tree> vec_offsets;
> auto_delete_vec<auto_vec<tree>> gvec_oprnds (group_size);
> for (i = 0; i < group_size; i++)
> gvec_oprnds.quick_push (new auto_vec<tree> (ncopies));
> - auto_vec<tree, 1> vec_oprnds;
> - for (j = 0; j < ncopies; j++)
> +
> + if (memory_access_type == VMAT_LOAD_STORE_LANES)
> {
> - gimple *new_stmt;
> - if (j == 0)
> + gcc_assert (!slp && grouped_store);
> + for (j = 0; j < ncopies; j++)
> {
> - if (slp)
> - {
> - /* Get vectorized arguments for SLP_NODE. */
> - vect_get_vec_defs (vinfo, stmt_info, slp_node, 1,
> - op, &vec_oprnds);
> - vec_oprnd = vec_oprnds[0];
> - }
> - else
> - {
> - /* For interleaved stores we collect vectorized defs for all the
> - stores in the group in DR_CHAIN. DR_CHAIN is then used as an
> - input to vect_permute_store_chain().
> -
> - If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN
> - is of size 1. */
> + gimple *new_stmt;
> + if (j == 0)
> + {
> + /* For interleaved stores we collect vectorized defs for all
> + the stores in the group in DR_CHAIN. DR_CHAIN is then used
> + as an input to vect_permute_store_chain(). */
> stmt_vec_info next_stmt_info = first_stmt_info;
> for (i = 0; i < group_size; i++)
> {
> /* Since gaps are not supported for interleaved stores,
> - DR_GROUP_SIZE is the exact number of stmts in the chain.
> - Therefore, NEXT_STMT_INFO can't be NULL_TREE. In case
> - that there is no interleaving, DR_GROUP_SIZE is 1,
> - and only one iteration of the loop will be executed. */
> + DR_GROUP_SIZE is the exact number of stmts in the
> + chain. Therefore, NEXT_STMT_INFO can't be NULL_TREE. */
> op = vect_get_store_rhs (next_stmt_info);
> vect_get_vec_defs_for_operand (vinfo, next_stmt_info,
> ncopies,
> op, gvec_oprnds[i]);
> @@ -8825,66 +8812,37 @@ vectorizable_store (vec_info *vinfo,
> if (mask)
> {
> vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
> - mask, &vec_masks,
> mask_vectype);
> + mask, &vec_masks,
> + mask_vectype);
> vec_mask = vec_masks[0];
> }
> - }
>
> - /* We should have catched mismatched types earlier. */
> - gcc_assert (useless_type_conversion_p (vectype,
> - TREE_TYPE (vec_oprnd)));
> - bool simd_lane_access_p
> - = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0;
> - if (simd_lane_access_p
> - && !loop_masks
> - && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
> - && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
> - && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info))
> - && integer_zerop (DR_INIT (first_dr_info->dr))
> - && alias_sets_conflict_p (get_alias_set (aggr_type),
> - get_alias_set (TREE_TYPE (ref_type))))
> - {
> - dataref_ptr = unshare_expr (DR_BASE_ADDRESS
> (first_dr_info->dr));
> - dataref_offset = build_int_cst (ref_type, 0);
> + /* We should have catched mismatched types earlier. */
> + gcc_assert (
> + useless_type_conversion_p (vectype, TREE_TYPE (vec_oprnd)));
> + dataref_ptr
> + = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
> + NULL, offset, &dummy, gsi,
> + &ptr_incr, false, bump);
> }
> - else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> - vect_get_gather_scatter_ops (loop_vinfo, loop, stmt_info,
> - slp_node, &gs_info, &dataref_ptr,
> - &vec_offsets);
> else
> - dataref_ptr
> - = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
> - simd_lane_access_p ? loop : NULL,
> - offset, &dummy, gsi, &ptr_incr,
> - simd_lane_access_p, bump);
> - }
> - else
> - {
> - gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
> - /* DR_CHAIN is then used as an input to vect_permute_store_chain().
> - If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN is
> - of size 1. */
> - for (i = 0; i < group_size; i++)
> {
> - vec_oprnd = (*gvec_oprnds[i])[j];
> - dr_chain[i] = vec_oprnd;
> + gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
> + /* DR_CHAIN is then used as an input to
> + vect_permute_store_chain(). */
> + for (i = 0; i < group_size; i++)
> + {
> + vec_oprnd = (*gvec_oprnds[i])[j];
> + dr_chain[i] = vec_oprnd;
> + }
> + if (mask)
> + vec_mask = vec_masks[j];
> + dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr,
> gsi,
> + stmt_info, bump);
> }
> - if (mask)
> - vec_mask = vec_masks[j];
> - if (dataref_offset)
> - dataref_offset
> - = int_const_binop (PLUS_EXPR, dataref_offset, bump);
> - else if (!STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> - dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
> - stmt_info, bump);
> - }
> -
> - if (memory_access_type == VMAT_LOAD_STORE_LANES)
> - {
> - tree vec_array;
>
> /* Get an array into which we can store the individual vectors. */
> - vec_array = create_vector_array (vectype, vec_num);
> + tree vec_array = create_vector_array (vectype, vec_num);
>
> /* Invalidate the current contents of VEC_ARRAY. This should
> become an RTL clobber too, which prevents the vector registers
> @@ -8895,8 +8853,8 @@ vectorizable_store (vec_info *vinfo,
> for (i = 0; i < vec_num; i++)
> {
> vec_oprnd = dr_chain[i];
> - write_vector_array (vinfo, stmt_info,
> - gsi, vec_oprnd, vec_array, i);
> + write_vector_array (vinfo, stmt_info, gsi, vec_oprnd, vec_array,
> + i);
> }
>
> tree final_mask = NULL;
> @@ -8906,8 +8864,8 @@ vectorizable_store (vec_info *vinfo,
> final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
> ncopies, vectype, j);
> if (vec_mask)
> - final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
> - final_mask, vec_mask, gsi);
> + final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
> final_mask,
> + vec_mask, gsi);
>
> if (lanes_ifn == IFN_MASK_LEN_STORE_LANES)
> {
> @@ -8955,8 +8913,7 @@ vectorizable_store (vec_info *vinfo,
> /* Emit:
> MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
> data_ref = create_array_ref (aggr_type, dataref_ptr, ref_type);
> - call = gimple_build_call_internal (IFN_STORE_LANES, 1,
> - vec_array);
> + call = gimple_build_call_internal (IFN_STORE_LANES, 1,
> vec_array);
> gimple_call_set_lhs (call, data_ref);
> }
> gimple_call_set_nothrow (call, true);
> @@ -8965,301 +8922,386 @@ vectorizable_store (vec_info *vinfo,
>
> /* Record that VEC_ARRAY is now dead. */
> vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
> + if (j == 0)
> + *vec_stmt = new_stmt;
> + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
> }
> - else
> - {
> - new_stmt = NULL;
> - if (grouped_store)
> - /* Permute. */
> - vect_permute_store_chain (vinfo, dr_chain, group_size, stmt_info,
> - gsi, &result_chain);
>
> - stmt_vec_info next_stmt_info = first_stmt_info;
> - for (i = 0; i < vec_num; i++)
> - {
> - unsigned misalign;
> - unsigned HOST_WIDE_INT align;
> + return true;
> + }
>
> - tree final_mask = NULL_TREE;
> - tree final_len = NULL_TREE;
> - tree bias = NULL_TREE;
> - if (loop_masks)
> - final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
> - vec_num * ncopies,
> - vectype, vec_num * j + i);
> - if (vec_mask)
> - final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
> - final_mask, vec_mask, gsi);
> + auto_vec<tree> result_chain (group_size);
> + auto_vec<tree> vec_offsets;
> + auto_vec<tree, 1> vec_oprnds;
> + for (j = 0; j < ncopies; j++)
> + {
> + gimple *new_stmt;
> + if (j == 0)
> + {
> + if (slp)
> + {
> + /* Get vectorized arguments for SLP_NODE. */
> + vect_get_vec_defs (vinfo, stmt_info, slp_node, 1, op,
> + &vec_oprnds);
> + vec_oprnd = vec_oprnds[0];
> + }
> + else
> + {
> + /* For interleaved stores we collect vectorized defs for all the
> + stores in the group in DR_CHAIN. DR_CHAIN is then used as an
> + input to vect_permute_store_chain().
>
> - if (memory_access_type == VMAT_GATHER_SCATTER
> - && gs_info.ifn != IFN_LAST)
> + If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN
> + is of size 1. */
> + stmt_vec_info next_stmt_info = first_stmt_info;
> + for (i = 0; i < group_size; i++)
> {
> - if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> - vec_offset = vec_offsets[vec_num * j + i];
> - tree scale = size_int (gs_info.scale);
> -
> - if (gs_info.ifn == IFN_MASK_LEN_SCATTER_STORE)
> - {
> - if (loop_lens)
> - final_len
> - = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
> - vec_num * ncopies, vectype,
> - vec_num * j + i, 1);
> - else
> - final_len
> - = build_int_cst (sizetype,
> - TYPE_VECTOR_SUBPARTS (vectype));
> - signed char biasval
> - = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
> - bias = build_int_cst (intQI_type_node, biasval);
> - if (!final_mask)
> - {
> - mask_vectype = truth_type_for (vectype);
> - final_mask = build_minus_one_cst (mask_vectype);
> - }
> - }
> -
> - gcall *call;
> - if (final_len && final_mask)
> - call
> - = gimple_build_call_internal
> (IFN_MASK_LEN_SCATTER_STORE,
> - 7, dataref_ptr,
> vec_offset,
> - scale, vec_oprnd,
> final_mask,
> - final_len, bias);
> - else if (final_mask)
> - call = gimple_build_call_internal
> - (IFN_MASK_SCATTER_STORE, 5, dataref_ptr, vec_offset,
> - scale, vec_oprnd, final_mask);
> - else
> - call = gimple_build_call_internal
> - (IFN_SCATTER_STORE, 4, dataref_ptr, vec_offset,
> - scale, vec_oprnd);
> - gimple_call_set_nothrow (call, true);
> - vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
> - new_stmt = call;
> - break;
> + /* Since gaps are not supported for interleaved stores,
> + DR_GROUP_SIZE is the exact number of stmts in the chain.
> + Therefore, NEXT_STMT_INFO can't be NULL_TREE. In case
> + that there is no interleaving, DR_GROUP_SIZE is 1,
> + and only one iteration of the loop will be executed. */
> + op = vect_get_store_rhs (next_stmt_info);
> + vect_get_vec_defs_for_operand (vinfo, next_stmt_info,
> ncopies,
> + op, gvec_oprnds[i]);
> + vec_oprnd = (*gvec_oprnds[i])[0];
> + dr_chain.quick_push (vec_oprnd);
> + next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
> }
> - else if (memory_access_type == VMAT_GATHER_SCATTER)
> + if (mask)
> {
> - /* Emulated scatter. */
> - gcc_assert (!final_mask);
> - unsigned HOST_WIDE_INT const_nunits = nunits.to_constant ();
> - unsigned HOST_WIDE_INT const_offset_nunits
> - = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype)
> - .to_constant ();
> - vec<constructor_elt, va_gc> *ctor_elts;
> - vec_alloc (ctor_elts, const_nunits);
> - gimple_seq stmts = NULL;
> - tree elt_type = TREE_TYPE (vectype);
> - unsigned HOST_WIDE_INT elt_size
> - = tree_to_uhwi (TYPE_SIZE (elt_type));
> - /* We support offset vectors with more elements
> - than the data vector for now. */
> - unsigned HOST_WIDE_INT factor
> - = const_offset_nunits / const_nunits;
> - vec_offset = vec_offsets[j / factor];
> - unsigned elt_offset = (j % factor) * const_nunits;
> - tree idx_type = TREE_TYPE (TREE_TYPE (vec_offset));
> - tree scale = size_int (gs_info.scale);
> - align = get_object_alignment (DR_REF (first_dr_info->dr));
> - tree ltype = build_aligned_type (TREE_TYPE (vectype),
> align);
> - for (unsigned k = 0; k < const_nunits; ++k)
> - {
> - /* Compute the offsetted pointer. */
> - tree boff = size_binop (MULT_EXPR, TYPE_SIZE (idx_type),
> - bitsize_int (k + elt_offset));
> - tree idx = gimple_build (&stmts, BIT_FIELD_REF,
> - idx_type, vec_offset,
> - TYPE_SIZE (idx_type), boff);
> - idx = gimple_convert (&stmts, sizetype, idx);
> - idx = gimple_build (&stmts, MULT_EXPR,
> - sizetype, idx, scale);
> - tree ptr = gimple_build (&stmts, PLUS_EXPR,
> - TREE_TYPE (dataref_ptr),
> - dataref_ptr, idx);
> - ptr = gimple_convert (&stmts, ptr_type_node, ptr);
> - /* Extract the element to be stored. */
> - tree elt = gimple_build (&stmts, BIT_FIELD_REF,
> - TREE_TYPE (vectype), vec_oprnd,
> - TYPE_SIZE (elt_type),
> - bitsize_int (k * elt_size));
> - gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
> - stmts = NULL;
> - tree ref = build2 (MEM_REF, ltype, ptr,
> - build_int_cst (ref_type, 0));
> - new_stmt = gimple_build_assign (ref, elt);
> - vect_finish_stmt_generation (vinfo, stmt_info,
> - new_stmt, gsi);
> - }
> - break;
> + vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
> + mask, &vec_masks,
> + mask_vectype);
> + vec_mask = vec_masks[0];
> }
> + }
>
> - if (i > 0)
> - /* Bump the vector pointer. */
> - dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr,
> - gsi, stmt_info, bump);
> + /* We should have catched mismatched types earlier. */
> + gcc_assert (useless_type_conversion_p (vectype,
> + TREE_TYPE (vec_oprnd)));
> + bool simd_lane_access_p
> + = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0;
> + if (simd_lane_access_p
> + && !loop_masks
> + && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
> + && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
> + && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info))
> + && integer_zerop (DR_INIT (first_dr_info->dr))
> + && alias_sets_conflict_p (get_alias_set (aggr_type),
> + get_alias_set (TREE_TYPE (ref_type))))
> + {
> + dataref_ptr = unshare_expr (DR_BASE_ADDRESS
> (first_dr_info->dr));
> + dataref_offset = build_int_cst (ref_type, 0);
> + }
> + else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> + vect_get_gather_scatter_ops (loop_vinfo, loop, stmt_info,
> slp_node,
> + &gs_info, &dataref_ptr,
> &vec_offsets);
> + else
> + dataref_ptr
> + = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
> + simd_lane_access_p ? loop : NULL,
> + offset, &dummy, gsi, &ptr_incr,
> + simd_lane_access_p, bump);
> + }
> + else
> + {
> + gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
> + /* DR_CHAIN is then used as an input to vect_permute_store_chain().
> + If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN is
> + of size 1. */
> + for (i = 0; i < group_size; i++)
> + {
> + vec_oprnd = (*gvec_oprnds[i])[j];
> + dr_chain[i] = vec_oprnd;
> + }
> + if (mask)
> + vec_mask = vec_masks[j];
> + if (dataref_offset)
> + dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset,
> bump);
> + else if (!STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> + dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
> + stmt_info, bump);
> + }
>
> - if (slp)
> - vec_oprnd = vec_oprnds[i];
> - else if (grouped_store)
> - /* For grouped stores vectorized defs are interleaved in
> - vect_permute_store_chain(). */
> - vec_oprnd = result_chain[i];
> + new_stmt = NULL;
> + if (grouped_store)
> + /* Permute. */
> + vect_permute_store_chain (vinfo, dr_chain, group_size, stmt_info, gsi,
> + &result_chain);
>
> - align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
> - if (alignment_support_scheme == dr_aligned)
> - misalign = 0;
> - else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
> - {
> - align = dr_alignment (vect_dr_behavior (vinfo,
> first_dr_info));
> - misalign = 0;
> - }
> - else
> - misalign = misalignment;
> - if (dataref_offset == NULL_TREE
> - && TREE_CODE (dataref_ptr) == SSA_NAME)
> - set_ptr_info_alignment (get_ptr_info (dataref_ptr), align,
> - misalign);
> - align = least_bit_hwi (misalign | align);
> -
> - if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
> - {
> - tree perm_mask = perm_mask_for_reverse (vectype);
> - tree perm_dest = vect_create_destination_var
> - (vect_get_store_rhs (stmt_info), vectype);
> - tree new_temp = make_ssa_name (perm_dest);
> -
> - /* Generate the permute statement. */
> - gimple *perm_stmt
> - = gimple_build_assign (new_temp, VEC_PERM_EXPR, vec_oprnd,
> - vec_oprnd, perm_mask);
> - vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt,
> gsi);
> -
> - perm_stmt = SSA_NAME_DEF_STMT (new_temp);
> - vec_oprnd = new_temp;
> - }
> + stmt_vec_info next_stmt_info = first_stmt_info;
> + for (i = 0; i < vec_num; i++)
> + {
> + unsigned misalign;
> + unsigned HOST_WIDE_INT align;
>
> - /* Compute IFN when LOOP_LENS or final_mask valid. */
> - machine_mode vmode = TYPE_MODE (vectype);
> - machine_mode new_vmode = vmode;
> - internal_fn partial_ifn = IFN_LAST;
> - if (loop_lens)
> - {
> - opt_machine_mode new_ovmode
> - = get_len_load_store_mode (vmode, false, &partial_ifn);
> - new_vmode = new_ovmode.require ();
> - unsigned factor
> - = (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
> - final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
> - vec_num * ncopies, vectype,
> - vec_num * j + i, factor);
> - }
> - else if (final_mask)
> - {
> - if (!can_vec_mask_load_store_p (vmode,
> - TYPE_MODE (TREE_TYPE
> (final_mask)),
> - false, &partial_ifn))
> - gcc_unreachable ();
> - }
> + tree final_mask = NULL_TREE;
> + tree final_len = NULL_TREE;
> + tree bias = NULL_TREE;
> + if (loop_masks)
> + final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
> + vec_num * ncopies, vectype,
> + vec_num * j + i);
> + if (vec_mask)
> + final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
> final_mask,
> + vec_mask, gsi);
>
> - if (partial_ifn == IFN_MASK_LEN_STORE)
> + if (memory_access_type == VMAT_GATHER_SCATTER
> + && gs_info.ifn != IFN_LAST)
> + {
> + if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> + vec_offset = vec_offsets[vec_num * j + i];
> + tree scale = size_int (gs_info.scale);
> +
> + if (gs_info.ifn == IFN_MASK_LEN_SCATTER_STORE)
> {
> - if (!final_len)
> - {
> - /* Pass VF value to 'len' argument of
> - MASK_LEN_STORE if LOOP_LENS is invalid. */
> - final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
> - }
> + if (loop_lens)
> + final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
> + vec_num * ncopies, vectype,
> + vec_num * j + i, 1);
> + else
> + final_len = build_int_cst (sizetype,
> + TYPE_VECTOR_SUBPARTS
> (vectype));
> + signed char biasval
> + = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
> + bias = build_int_cst (intQI_type_node, biasval);
> if (!final_mask)
> {
> - /* Pass all ones value to 'mask' argument of
> - MASK_LEN_STORE if final_mask is invalid. */
> mask_vectype = truth_type_for (vectype);
> final_mask = build_minus_one_cst (mask_vectype);
> }
> }
> - if (final_len)
> - {
> - signed char biasval
> - = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
>
> - bias = build_int_cst (intQI_type_node, biasval);
> + gcall *call;
> + if (final_len && final_mask)
> + call = gimple_build_call_internal (IFN_MASK_LEN_SCATTER_STORE,
> + 7, dataref_ptr, vec_offset,
> + scale, vec_oprnd,
> final_mask,
> + final_len, bias);
> + else if (final_mask)
> + call
> + = gimple_build_call_internal (IFN_MASK_SCATTER_STORE, 5,
> + dataref_ptr, vec_offset,
> scale,
> + vec_oprnd, final_mask);
> + else
> + call = gimple_build_call_internal (IFN_SCATTER_STORE, 4,
> + dataref_ptr, vec_offset,
> + scale, vec_oprnd);
> + gimple_call_set_nothrow (call, true);
> + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
> + new_stmt = call;
> + break;
> + }
> + else if (memory_access_type == VMAT_GATHER_SCATTER)
> + {
> + /* Emulated scatter. */
> + gcc_assert (!final_mask);
> + unsigned HOST_WIDE_INT const_nunits = nunits.to_constant ();
> + unsigned HOST_WIDE_INT const_offset_nunits
> + = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype).to_constant
> ();
> + vec<constructor_elt, va_gc> *ctor_elts;
> + vec_alloc (ctor_elts, const_nunits);
> + gimple_seq stmts = NULL;
> + tree elt_type = TREE_TYPE (vectype);
> + unsigned HOST_WIDE_INT elt_size
> + = tree_to_uhwi (TYPE_SIZE (elt_type));
> + /* We support offset vectors with more elements
> + than the data vector for now. */
> + unsigned HOST_WIDE_INT factor
> + = const_offset_nunits / const_nunits;
> + vec_offset = vec_offsets[j / factor];
> + unsigned elt_offset = (j % factor) * const_nunits;
> + tree idx_type = TREE_TYPE (TREE_TYPE (vec_offset));
> + tree scale = size_int (gs_info.scale);
> + align = get_object_alignment (DR_REF (first_dr_info->dr));
> + tree ltype = build_aligned_type (TREE_TYPE (vectype), align);
> + for (unsigned k = 0; k < const_nunits; ++k)
> + {
> + /* Compute the offsetted pointer. */
> + tree boff = size_binop (MULT_EXPR, TYPE_SIZE (idx_type),
> + bitsize_int (k + elt_offset));
> + tree idx
> + = gimple_build (&stmts, BIT_FIELD_REF, idx_type,
> vec_offset,
> + TYPE_SIZE (idx_type), boff);
> + idx = gimple_convert (&stmts, sizetype, idx);
> + idx = gimple_build (&stmts, MULT_EXPR, sizetype, idx,
> scale);
> + tree ptr
> + = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE
> (dataref_ptr),
> + dataref_ptr, idx);
> + ptr = gimple_convert (&stmts, ptr_type_node, ptr);
> + /* Extract the element to be stored. */
> + tree elt
> + = gimple_build (&stmts, BIT_FIELD_REF, TREE_TYPE
> (vectype),
> + vec_oprnd, TYPE_SIZE (elt_type),
> + bitsize_int (k * elt_size));
> + gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
> + stmts = NULL;
> + tree ref
> + = build2 (MEM_REF, ltype, ptr, build_int_cst (ref_type,
> 0));
> + new_stmt = gimple_build_assign (ref, elt);
> + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
> gsi);
> }
> + break;
> + }
>
> - /* Arguments are ready. Create the new vector stmt. */
> - if (final_len)
> - {
> - gcall *call;
> - tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
> - /* Need conversion if it's wrapped with VnQI. */
> - if (vmode != new_vmode)
> - {
> - tree new_vtype
> - = build_vector_type_for_mode
> (unsigned_intQI_type_node,
> - new_vmode);
> - tree var
> - = vect_get_new_ssa_name (new_vtype, vect_simple_var);
> - vec_oprnd
> - = build1 (VIEW_CONVERT_EXPR, new_vtype, vec_oprnd);
> - gassign *new_stmt
> - = gimple_build_assign (var, VIEW_CONVERT_EXPR,
> - vec_oprnd);
> - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
> - gsi);
> - vec_oprnd = var;
> - }
> + if (i > 0)
> + /* Bump the vector pointer. */
> + dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
> + stmt_info, bump);
>
> - if (partial_ifn == IFN_MASK_LEN_STORE)
> - call = gimple_build_call_internal (IFN_MASK_LEN_STORE, 6,
> - dataref_ptr, ptr,
> - final_mask, final_len,
> - bias, vec_oprnd);
> - else
> - call
> - = gimple_build_call_internal (IFN_LEN_STORE, 5,
> - dataref_ptr, ptr,
> - final_len, bias,
> - vec_oprnd);
> - gimple_call_set_nothrow (call, true);
> - vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
> - new_stmt = call;
> + if (slp)
> + vec_oprnd = vec_oprnds[i];
> + else if (grouped_store)
> + /* For grouped stores vectorized defs are interleaved in
> + vect_permute_store_chain(). */
> + vec_oprnd = result_chain[i];
> +
> + align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
> + if (alignment_support_scheme == dr_aligned)
> + misalign = 0;
> + else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
> + {
> + align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
> + misalign = 0;
> + }
> + else
> + misalign = misalignment;
> + if (dataref_offset == NULL_TREE
> + && TREE_CODE (dataref_ptr) == SSA_NAME)
> + set_ptr_info_alignment (get_ptr_info (dataref_ptr), align,
> + misalign);
> + align = least_bit_hwi (misalign | align);
> +
> + if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
> + {
> + tree perm_mask = perm_mask_for_reverse (vectype);
> + tree perm_dest
> + = vect_create_destination_var (vect_get_store_rhs (stmt_info),
> + vectype);
> + tree new_temp = make_ssa_name (perm_dest);
> +
> + /* Generate the permute statement. */
> + gimple *perm_stmt
> + = gimple_build_assign (new_temp, VEC_PERM_EXPR, vec_oprnd,
> + vec_oprnd, perm_mask);
> + vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
> +
> + perm_stmt = SSA_NAME_DEF_STMT (new_temp);
> + vec_oprnd = new_temp;
> + }
> +
> + /* Compute IFN when LOOP_LENS or final_mask valid. */
> + machine_mode vmode = TYPE_MODE (vectype);
> + machine_mode new_vmode = vmode;
> + internal_fn partial_ifn = IFN_LAST;
> + if (loop_lens)
> + {
> + opt_machine_mode new_ovmode
> + = get_len_load_store_mode (vmode, false, &partial_ifn);
> + new_vmode = new_ovmode.require ();
> + unsigned factor
> + = (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
> + final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
> + vec_num * ncopies, vectype,
> + vec_num * j + i, factor);
> + }
> + else if (final_mask)
> + {
> + if (!can_vec_mask_load_store_p (
> + vmode, TYPE_MODE (TREE_TYPE (final_mask)), false,
> + &partial_ifn))
> + gcc_unreachable ();
> + }
> +
> + if (partial_ifn == IFN_MASK_LEN_STORE)
> + {
> + if (!final_len)
> + {
> + /* Pass VF value to 'len' argument of
> + MASK_LEN_STORE if LOOP_LENS is invalid. */
> + final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
> }
> - else if (final_mask)
> + if (!final_mask)
> {
> - tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
> - gcall *call
> - = gimple_build_call_internal (IFN_MASK_STORE, 4,
> - dataref_ptr, ptr,
> - final_mask, vec_oprnd);
> - gimple_call_set_nothrow (call, true);
> - vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
> - new_stmt = call;
> + /* Pass all ones value to 'mask' argument of
> + MASK_LEN_STORE if final_mask is invalid. */
> + mask_vectype = truth_type_for (vectype);
> + final_mask = build_minus_one_cst (mask_vectype);
> }
> - else
> + }
> + if (final_len)
> + {
> + signed char biasval
> + = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
> +
> + bias = build_int_cst (intQI_type_node, biasval);
> + }
> +
> + /* Arguments are ready. Create the new vector stmt. */
> + if (final_len)
> + {
> + gcall *call;
> + tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
> + /* Need conversion if it's wrapped with VnQI. */
> + if (vmode != new_vmode)
> {
> - data_ref = fold_build2 (MEM_REF, vectype,
> - dataref_ptr,
> - dataref_offset
> - ? dataref_offset
> - : build_int_cst (ref_type, 0));
> - if (alignment_support_scheme == dr_aligned)
> - ;
> - else
> - TREE_TYPE (data_ref)
> - = build_aligned_type (TREE_TYPE (data_ref),
> - align * BITS_PER_UNIT);
> - vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
> - new_stmt = gimple_build_assign (data_ref, vec_oprnd);
> + tree new_vtype
> + = build_vector_type_for_mode (unsigned_intQI_type_node,
> + new_vmode);
> + tree var = vect_get_new_ssa_name (new_vtype,
> vect_simple_var);
> + vec_oprnd = build1 (VIEW_CONVERT_EXPR, new_vtype,
> vec_oprnd);
> + gassign *new_stmt
> + = gimple_build_assign (var, VIEW_CONVERT_EXPR, vec_oprnd);
> vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
> gsi);
> + vec_oprnd = var;
> }
>
> - if (slp)
> - continue;
> -
> - next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
> - if (!next_stmt_info)
> - break;
> + if (partial_ifn == IFN_MASK_LEN_STORE)
> + call = gimple_build_call_internal (IFN_MASK_LEN_STORE, 6,
> + dataref_ptr, ptr,
> final_mask,
> + final_len, bias,
> vec_oprnd);
> + else
> + call = gimple_build_call_internal (IFN_LEN_STORE, 5,
> + dataref_ptr, ptr,
> final_len,
> + bias, vec_oprnd);
> + gimple_call_set_nothrow (call, true);
> + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
> + new_stmt = call;
> + }
> + else if (final_mask)
> + {
> + tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
> + gcall *call
> + = gimple_build_call_internal (IFN_MASK_STORE, 4, dataref_ptr,
> + ptr, final_mask, vec_oprnd);
> + gimple_call_set_nothrow (call, true);
> + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
> + new_stmt = call;
> + }
> + else
> + {
> + data_ref
> + = fold_build2 (MEM_REF, vectype, dataref_ptr,
> + dataref_offset ? dataref_offset
> + : build_int_cst (ref_type, 0));
> + if (alignment_support_scheme == dr_aligned)
> + ;
> + else
> + TREE_TYPE (data_ref)
> + = build_aligned_type (TREE_TYPE (data_ref),
> + align * BITS_PER_UNIT);
> + vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
> + new_stmt = gimple_build_assign (data_ref, vec_oprnd);
> + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
> }
> +
> + if (slp)
> + continue;
> +
> + next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
> + if (!next_stmt_info)
> + break;
> }
> if (!slp)
> {
> --
> 2.31.1
