On Mon, Sep 8, 2025 at 3:19 PM Robin Dapp <rdapp....@gmail.com> wrote:
>
> Hi,
>
> This patch adds gather/scatter handling for grouped access. The idea is
> to e.g. replace an access (for uint8_t elements) like
> arr[0]
> arr[1]
> arr[2]
> arr[3]
> arr[0 + step]
> arr[1 + step]
> ...
> by a gather load of uint32_t
> arr[0]
> arr[0 + step * 1]
> arr[0 + step * 2]
> ...
> where the offset vector is a simple series with step STEP.
> If supported, such a gather can be implemented as a strided load.
>
> Bootstrapped on x86 and power10. Regtested on aarch64 and riscv.
>
> Regards
> Robin
>
>
> PR target/118019
>
> gcc/ChangeLog:
>
> * internal-fn.cc (get_supported_else_vals): Exit at invalid
> index.
> (internal_strided_fn_supported_p): New function.
> * internal-fn.h (internal_strided_fn_supported_p): Declare.
> * tree-vect-data-refs.cc (vect_supportable_dr_alignment):
> Assume packed if access mode unit size unequal data ref type.
> * tree-vect-stmts.cc (vect_get_punning_vectype): New function.
> (vect_use_grouped_gather): New function.
> (get_load_store_type): Call new function.
> (vectorizable_store): Use punned vectype.
> (vectorizable_load): Ditto.
> * tree-vectorizer.h (struct vect_load_store_data): Add punned
> vectype.
>
> gcc/testsuite/ChangeLog:
>
> * gcc.target/riscv/rvv/autovec/pr118019-2.c: New test.
> ---
> gcc/internal-fn.cc | 22 +-
> gcc/internal-fn.h | 2 +
> .../gcc.target/riscv/rvv/autovec/pr118019-2.c | 50 +++++
> gcc/tree-vect-data-refs.cc | 4 +-
> gcc/tree-vect-stmts.cc | 207 ++++++++++++++++--
> gcc/tree-vectorizer.h | 1 +
> 6 files changed, 270 insertions(+), 16 deletions(-)
> create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/pr118019-2.c
>
> diff --git a/gcc/internal-fn.cc b/gcc/internal-fn.cc
> index bf2fac81807..db396c69ec5 100644
> --- a/gcc/internal-fn.cc
> +++ b/gcc/internal-fn.cc
> @@ -5234,7 +5234,7 @@ get_supported_else_vals (enum insn_code icode, unsigned
> else_index,
> vec<int> &else_vals)
> {
> const struct insn_data_d *data = &insn_data[icode];
> - if ((char)else_index >= data->n_operands)
> + if ((int)else_index >= data->n_operands || (int)else_index == -1)
> return;
>
> machine_mode else_mode = data->operand[else_index].mode;
> @@ -5309,6 +5309,26 @@ internal_gather_scatter_fn_supported_p (internal_fn
> ifn, tree vector_type,
> return ok;
> }
>
> +/* Return true if the target supports a strided load/store function IFN
> + with VECTOR_TYPE. If supported and ELSVALS is nonzero the supported else
> + values will be added to the vector ELSVALS points to. */
> +
> +bool
> +internal_strided_fn_supported_p (internal_fn ifn, tree vector_type,
> + vec<int> *elsvals)
> +{
> + machine_mode mode = TYPE_MODE (vector_type);
> + optab optab = direct_internal_fn_optab (ifn);
> + insn_code icode = direct_optab_handler (optab, mode);
> +
> + bool ok = icode != CODE_FOR_nothing;
> +
> + if (ok && elsvals)
> + get_supported_else_vals (icode, internal_fn_else_index (ifn), *elsvals);
> +
> + return ok;
> +}
> +
> /* Return true if the target supports IFN_CHECK_{RAW,WAR}_PTRS function IFN
> for pointers of type TYPE when the accesses have LENGTH bytes and their
> common byte alignment is ALIGN. */
> diff --git a/gcc/internal-fn.h b/gcc/internal-fn.h
> index fd21694dfeb..dcb707251f8 100644
> --- a/gcc/internal-fn.h
> +++ b/gcc/internal-fn.h
> @@ -246,6 +246,8 @@ extern int internal_fn_alias_ptr_index (internal_fn fn);
> extern bool internal_gather_scatter_fn_supported_p (internal_fn, tree,
> tree, tree, int,
> vec<int> * = nullptr);
> +extern bool internal_strided_fn_supported_p (internal_fn, tree,
> + vec<int> * = nullptr);
> extern bool internal_check_ptrs_fn_supported_p (internal_fn, tree,
> poly_uint64, unsigned int);
>
> diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/pr118019-2.c
> b/gcc/testsuite/gcc.target/riscv/rvv/autovec/pr118019-2.c
> new file mode 100644
> index 00000000000..d3436b78377
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/pr118019-2.c
> @@ -0,0 +1,50 @@
> +/* { dg-do compile } */
> +/* { dg-options "-O3 -march=rv64gcv_zvl512b -mabi=lp64d
> -mno-vector-strict-align" } */
> +
> +/* Ensure we use strided loads. */
> +
> +typedef unsigned char uint8_t;
> +typedef unsigned short uint16_t;
> +typedef unsigned int uint32_t;
> +
> +#define HADAMARD4(d0, d1, d2, d3, s0, s1, s2, s3) {\
> + int t0 = s0 + s1;\
> + int t1 = s0 - s1;\
> + int t2 = s2 + s3;\
> + int t3 = s2 - s3;\
> + d0 = t0 + t2;\
> + d2 = t0 - t2;\
> + d1 = t1 + t3;\
> + d3 = t1 - t3;\
> +}
> +
> +uint32_t
> +abs2 (uint32_t a)
> +{
> + uint32_t s = ((a >> 15) & 0x10001) * 0xffff;
> + return (a + s) ^ s;
> +}
> +
> +int
> +x264_pixel_satd_8x4 (uint8_t *pix1, int i_pix1, uint8_t *pix2, int i_pix2)
> +{
> + uint32_t tmp[4][4];
> + uint32_t a0, a1, a2, a3;
> + int sum = 0;
> + for (int i = 0; i < 4; i++, pix1 += i_pix1, pix2 += i_pix2)
> + {
> + a0 = (pix1[0] - pix2[0]) + ((pix1[4] - pix2[4]) << 16);
> + a1 = (pix1[1] - pix2[1]) + ((pix1[5] - pix2[5]) << 16);
> + a2 = (pix1[2] - pix2[2]) + ((pix1[6] - pix2[6]) << 16);
> + a3 = (pix1[3] - pix2[3]) + ((pix1[7] - pix2[7]) << 16);
> + HADAMARD4 (tmp[i][0], tmp[i][1], tmp[i][2], tmp[i][3], a0, a1, a2, a3);
> + }
> + for (int i = 0; i < 4; i++)
> + {
> + HADAMARD4 (a0, a1, a2, a3, tmp[0][i], tmp[1][i], tmp[2][i], tmp[3][i]);
> + sum += abs2 (a0) + abs2 (a1) + abs2 (a2) + abs2 (a3);
> + }
> + return (((uint16_t) sum) + ((uint32_t) sum >> 16)) >> 1;
> +}
> +
> +/* { dg-final { scan-assembler-times "vlse32" 4 } } */
> diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc
> index e451b72e07e..be03495c6b9 100644
> --- a/gcc/tree-vect-data-refs.cc
> +++ b/gcc/tree-vect-data-refs.cc
> @@ -6640,7 +6640,9 @@ vect_supportable_dr_alignment (vec_info *vinfo,
> dr_vec_info *dr_info,
> bool is_packed = false;
> tree type = TREE_TYPE (DR_REF (dr));
> if (misalignment == DR_MISALIGNMENT_UNKNOWN)
> - is_packed = not_size_aligned (DR_REF (dr));
> + is_packed = not_size_aligned (DR_REF (dr))
> + || tree_to_uhwi (TYPE_SIZE (type))
> + < tree_to_uhwi (TYPE_SIZE (TREE_TYPE (vectype)));
Hmm, so the existing "punning" code for VMAT_STRIDED_SLP does
tree vtype
= vector_vector_composition_type (vectype, const_nunits / n,
&ptype);
if (vtype != NULL_TREE)
{
dr_alignment_support dr_align;
int mis_align = 0;
if (VECTOR_TYPE_P (ptype))
{
mis_align = dr_misalignment (first_dr_info, ptype);
if (maybe_gt (vf, 1u)
&& !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
DR_TARGET_ALIGNMENT (first_dr_info)))
mis_align = -1;
dr_align
= vect_supportable_dr_alignment (vinfo, dr_info, ptype,
mis_align);
so the change would pessimize the handling in this case unnecessarily?
Isn't the issue that we now pass in the "wrong" element type (already for
above, but I guess it's unused when !is_gather_scatter?)? I think
clarifying the hook documentation would be good. In particular the
'misalignment' is in bytes, and IIRC is_packed specifies whether the
vector load is element aligned. That's possibly from times where we
had misalignment specified in elements, not bytes, so it somehow
feels redundant (likewise passing an element type here at all).
I think we want to handle this in a separate patch, it requires review of
the existing implementations of the hook.
> if (targetm.vectorize.support_vector_misalignment (mode, type,
> misalignment,
> is_packed,
> is_gather_scatter))
> diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
> index 5b1f291fa8d..619b99083a3 100644
> --- a/gcc/tree-vect-stmts.cc
> +++ b/gcc/tree-vect-stmts.cc
> @@ -1723,6 +1723,124 @@ vect_truncate_gather_scatter_offset (stmt_vec_info
> stmt_info, tree vectype,
> return false;
> }
>
> +/* Check if there is an integer vector type with the same size as VECTYPE but
> + NELTS units of size (TYPE_SIZE (VECTYPE) / NELTS). If so, return the
> + appropriate type. */
> +
> +static tree
> +vect_get_punning_vectype (tree vectype, int nelts)
So we have vector_vector_composition_type, can we split out the relevant
parts from there instead of duplicating it? Or maybe just add a
flag argument to the function, indicating whether a vector *ptype is OK?
Otherwise I think this looks OK.
> +{
> + gcc_assert (VECTOR_TYPE_P (vectype));
> +
> + machine_mode vmode = TYPE_MODE (vectype);
> + if (!VECTOR_MODE_P (vmode))
> + return NULL_TREE;
> +
> + poly_uint64 vbsize = GET_MODE_BITSIZE (vmode);
> + unsigned int pbsize;
> + scalar_int_mode elmode;
> + if (constant_multiple_p (vbsize, nelts, &pbsize)
> + && (int_mode_for_mode (SCALAR_TYPE_MODE
> + (TREE_TYPE (vectype))).exists (&elmode)))
> + {
> + machine_mode rmode;
> + if (int_mode_for_size (pbsize, 0).exists (&elmode)
> + && tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype))) * nelts
> + == GET_MODE_SIZE (elmode)
> + && related_vector_mode (vmode, elmode, nelts).exists (&rmode))
> + {
> + tree ptype = build_nonstandard_integer_type (pbsize, 1);
> + return build_vector_type (ptype, nelts);
> + }
> + }
> +
> + return NULL_TREE;
> +}
> +
> +/* Return true if we can use gather/scatter or strided internal functions
> + to vectorize STMT_INFO, which is a grouped or strided load or store
> + with multiple lanes and will be implemented by a type-punned access
> + of a vector with element size that matches the number of lanes.
> +
> + MASKED_P is true if load or store is conditional.
> + When returning true, fill in GS_INFO with the information required to
> + perform the operation. Also, store the punning type in PUNNED_VECTYPE.
> +
> + If successful and ELSVALS is nonzero the supported
> + else values will be stored in the vector ELSVALS points to. */
> +
> +static bool
> +vect_use_grouped_gather (stmt_vec_info stmt_info, tree vectype,
> + loop_vec_info loop_vinfo, bool masked_p,
> + unsigned int nelts,
> + gather_scatter_info *info, vec<int> *elsvals,
> + tree *pun_vectype)
> +{
> + dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
> + data_reference *dr = dr_info->dr;
> +
> + /* TODO: We can support nelts > BITS_PER_UNIT or non-power-of-two by
> + multiple gathers/scatter. */
> + if (nelts > BITS_PER_UNIT || !pow2p_hwi (nelts))
> + return false;
> +
> + /* Pun the vectype with one of the same size but an element spanning
> + NELTS elements of VECTYPE.
> + The punned type of a V16QI with NELTS = 4 would be V4SI.
> + */
> + *pun_vectype = vect_get_punning_vectype (vectype, nelts);
> +
> + if (!*pun_vectype)
> + return false;
> +
> + internal_fn ifn;
> + tree offset_vectype = *pun_vectype;
> +
> + internal_fn strided_ifn = DR_IS_READ (dr)
> + ? IFN_MASK_LEN_STRIDED_LOAD : IFN_MASK_LEN_STRIDED_STORE;
> +
> + /* Check if we have a gather/scatter with the new type. We're just trying
> + with the type itself as offset for now. If not, check if we have a
> + strided load/store. These have fewer constraints (for example no offset
> + type must exist) so it is possible that even though a gather/scatter is
> + not available we still have a strided load/store. */
> + bool ok = false;
> + if (vect_gather_scatter_fn_p
> + (loop_vinfo, DR_IS_READ (dr), masked_p, *pun_vectype,
> + TREE_TYPE (*pun_vectype), *pun_vectype, 1, &ifn,
> + &offset_vectype, elsvals))
> + ok = true;
> + else if (internal_strided_fn_supported_p (strided_ifn, *pun_vectype,
> + elsvals))
> + {
> + /* Use gather/scatter IFNs, vect_get_strided_load_store_ops
> + will switch back to the strided variants. */
> + ifn = DR_IS_READ (dr) ? IFN_MASK_LEN_GATHER_LOAD :
> + IFN_MASK_LEN_SCATTER_STORE;
> + ok = true;
> + }
> +
> + if (ok)
> + {
> + info->ifn = ifn;
> + info->decl = NULL_TREE;
> + info->base = dr->ref;
> + info->alias_ptr = build_int_cst
> + (reference_alias_ptr_type (DR_REF (dr)),
> + get_object_alignment (DR_REF (dr)));
> + info->element_type = TREE_TYPE (vectype);
> + info->offset_vectype = offset_vectype;
> + /* No need to set the offset, vect_get_strided_load_store_ops
> + will do that. */
> + info->scale = 1;
> + info->memory_type = TREE_TYPE (DR_REF (dr));
> + return true;
> + }
> +
> + return false;
> +}
> +
> +
> /* Return true if we can use gather/scatter internal functions to
> vectorize STMT_INFO, which is a grouped or strided load or store.
> MASKED_P is true if load or store is conditional. When returning
> @@ -1978,6 +2096,7 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info
> stmt_info,
> int *misalignment = &ls->misalignment;
> internal_fn *lanes_ifn = &ls->lanes_ifn;
> vec<int> *elsvals = &ls->elsvals;
> + tree *pun_vectype = &ls->pun_vectype;
> loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
> poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
> class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
> @@ -1989,6 +2108,7 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info
> stmt_info,
>
> *misalignment = DR_MISALIGNMENT_UNKNOWN;
> *poffset = 0;
> + *pun_vectype = NULL_TREE;
>
> if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
> {
> @@ -2321,13 +2441,13 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info
> stmt_info,
> if ((*memory_access_type == VMAT_ELEMENTWISE
> || *memory_access_type == VMAT_STRIDED_SLP)
> && !STMT_VINFO_GATHER_SCATTER_P (stmt_info)
> - && SLP_TREE_LANES (slp_node) == 1
> && loop_vinfo)
> {
> gather_scatter_info gs_info;
> - if (vect_use_strided_gather_scatters_p (stmt_info, vectype, loop_vinfo,
> - masked_p, &gs_info, elsvals,
> - group_size, single_element_p))
> + if (SLP_TREE_LANES (slp_node) == 1
> + && vect_use_strided_gather_scatters_p (stmt_info, vectype,
> loop_vinfo,
> + masked_p, &gs_info, elsvals,
> + group_size,
> single_element_p))
> {
> SLP_TREE_GS_SCALE (slp_node) = gs_info.scale;
> SLP_TREE_GS_BASE (slp_node) = error_mark_node;
> @@ -2335,6 +2455,28 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info
> stmt_info,
> ls->strided_offset_vectype = gs_info.offset_vectype;
> *memory_access_type = VMAT_GATHER_SCATTER_IFN;
> }
> + else if (SLP_TREE_LANES (slp_node) > 1
> + && vect_use_grouped_gather (stmt_info, vectype, loop_vinfo,
> + masked_p, SLP_TREE_LANES
> (slp_node),
> + &gs_info, elsvals, pun_vectype))
> + {
> + int puntype_misalignment = dr_misalignment
> + (first_dr_info, *pun_vectype, *poffset);
> + dr_alignment_support puntype_alignment_scheme
> + = vect_supportable_dr_alignment
> + (vinfo, first_dr_info, *pun_vectype, puntype_misalignment,
> + true);
> +
> + if (puntype_alignment_scheme == dr_aligned
> + || puntype_alignment_scheme == dr_unaligned_supported)
> + {
> + SLP_TREE_GS_SCALE (slp_node) = gs_info.scale;
> + SLP_TREE_GS_BASE (slp_node) = error_mark_node;
> + ls->gs.ifn = gs_info.ifn;
> + ls->strided_offset_vectype = gs_info.offset_vectype;
> + *memory_access_type = VMAT_GATHER_SCATTER_IFN;
> + }
> + }
> }
>
> if (*memory_access_type == VMAT_CONTIGUOUS_DOWN
> @@ -2351,14 +2493,15 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info
> stmt_info,
> }
> else
> {
> + tree vtype = ls->pun_vectype ? ls->pun_vectype : vectype;
> if (mat_gather_scatter_p (*memory_access_type)
> && !first_dr_info)
> *misalignment = DR_MISALIGNMENT_UNKNOWN;
> else
> - *misalignment = dr_misalignment (first_dr_info, vectype, *poffset);
> + *misalignment = dr_misalignment (first_dr_info, vtype, *poffset);
> *alignment_support_scheme
> = vect_supportable_dr_alignment
> - (vinfo, first_dr_info, vectype, *misalignment,
> + (vinfo, first_dr_info, vtype, *misalignment,
> mat_gather_scatter_p (*memory_access_type));
> }
>
> @@ -8364,10 +8507,13 @@ vectorizable_store (vec_info *vinfo,
> {
> aggr_type = elem_type;
> if (!costing_p)
> - vect_get_strided_load_store_ops (stmt_info, slp_node, vectype,
> - ls.strided_offset_vectype,
> - loop_vinfo, gsi,
> - &bump, &vec_offset, loop_lens);
> + {
> + tree vtype = ls.pun_vectype ? ls.pun_vectype : vectype;
> + vect_get_strided_load_store_ops (stmt_info, slp_node, vtype,
> + ls.strided_offset_vectype,
> + loop_vinfo, gsi,
> + &bump, &vec_offset, loop_lens);
> + }
> }
> else
> {
> @@ -8553,7 +8699,9 @@ vectorizable_store (vec_info *vinfo,
>
> if (mat_gather_scatter_p (memory_access_type))
> {
> - gcc_assert (!grouped_store);
> + gcc_assert (!grouped_store || ls.pun_vectype);
> + if (ls.pun_vectype)
> + vectype = ls.pun_vectype;
> auto_vec<tree> vec_offsets;
> unsigned int inside_cost = 0, prologue_cost = 0;
> int num_stmts = vec_num;
> @@ -8600,8 +8748,9 @@ vectorizable_store (vec_info *vinfo,
> if (mask_node)
> vec_mask = vec_masks[j];
> /* We should have catched mismatched types earlier. */
> - gcc_assert (useless_type_conversion_p (vectype,
> - TREE_TYPE (vec_oprnd)));
> + gcc_assert (ls.pun_vectype
> + || useless_type_conversion_p
> + (vectype, TREE_TYPE (vec_oprnd)));
> }
> tree final_mask = NULL_TREE;
> tree final_len = NULL_TREE;
> @@ -8654,6 +8803,18 @@ vectorizable_store (vec_info *vinfo,
> }
> }
>
> + if (ls.pun_vectype)
> + {
> + gimple *conv_stmt
> + = gimple_build_assign (make_ssa_name (vectype),
> + VIEW_CONVERT_EXPR,
> + build1 (VIEW_CONVERT_EXPR, vectype,
> + vec_oprnd));
> + vect_finish_stmt_generation (vinfo, stmt_info, conv_stmt,
> + gsi);
> + vec_oprnd = gimple_get_lhs (conv_stmt);
> + }
> +
> gcall *call;
> if (final_len && final_mask)
> {
> @@ -10413,7 +10574,14 @@ vectorizable_load (vec_info *vinfo,
>
> if (mat_gather_scatter_p (memory_access_type))
> {
> - gcc_assert (!grouped_load && !slp_perm);
> + gcc_assert ((!grouped_load && !slp_perm) || ls.pun_vectype);
> +
> + /* If we pun the original vectype the loads as well as costing, length,
> + etc. is performed with the new type. After loading we VIEW_CONVERT
> + the data to the original vectype. */
> + tree original_vectype = vectype;
> + if (ls.pun_vectype)
> + vectype = ls.pun_vectype;
>
> /* 1. Create the vector or array pointer update chain. */
> if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> @@ -10754,6 +10922,17 @@ vectorizable_load (vec_info *vinfo,
> new_temp = new_temp2;
> }
>
> + if (ls.pun_vectype)
> + {
> + new_stmt = gimple_build_assign (make_ssa_name
> + (original_vectype),
> + VIEW_CONVERT_EXPR,
> + build1 (VIEW_CONVERT_EXPR,
> + original_vectype,
> + new_temp));
> + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
> + }
> +
> /* Store vector loads in the corresponding SLP_NODE. */
> slp_node->push_vec_def (new_stmt);
> }
> diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
> index df805c6ade9..c7533d7a35b 100644
> --- a/gcc/tree-vectorizer.h
> +++ b/gcc/tree-vectorizer.h
> @@ -287,6 +287,7 @@ struct vect_load_store_data : vect_data {
> tree decl; // VMAT_GATHER_SCATTER_DECL
> } gs;
> tree strided_offset_vectype; // VMAT_GATHER_SCATTER_IFN, originally strided
> + tree pun_vectype; // VMAT_GATHER_SCATTER_IFN
> auto_vec<int> elsvals;
> };
>
> --
> 2.51.0
>
