On Wed, Jun 15, 2016 at 10:51 AM, Richard Sandiford
<[email protected]> wrote:
> This patch just refactors the gather/scatter support so that all
> information is in a single structure, rather than separate variables.
> This reduces the number of arguments to a function added in patch 6.
>
> Tested on aarch64-linux-gnu and x86_64-linux-gnu. OK to install?
Please pack it by moving offset_dt after offset_vectype.
Ok with that change.
Thanks,
Richard.
> Thanks,
> Richard
>
>
> gcc/
> * tree-vectorizer.h (gather_scatter_info): New structure.
> (vect_check_gather_scatter): Return a bool rather than a decl.
> Replace return-by-pointer arguments with a single
> gather_scatter_info *.
> * tree-vect-data-refs.c (vect_check_gather_scatter): Likewise.
> (vect_analyze_data_refs): Update call accordingly.
> * tree-vect-stmts.c (vect_mark_stmts_to_be_vectorized): Likewise.
> (vectorizable_mask_load_store): Likewise. Also record the
> offset dt and vectype in the gather_scatter_info.
> (vectorizable_store): Likewise.
> (vectorizable_load): Likewise.
>
> Index: gcc/tree-vectorizer.h
> ===================================================================
> --- gcc/tree-vectorizer.h
> +++ gcc/tree-vectorizer.h
> @@ -612,6 +612,28 @@ typedef struct _stmt_vec_info {
> unsigned int num_slp_uses;
> } *stmt_vec_info;
>
> +/* Information about a gather/scatter call. */
> +struct gather_scatter_info {
> + /* The FUNCTION_DECL for the built-in gather/scatter function. */
> + tree decl;
> +
> + /* The loop-invariant base value. */
> + tree base;
> +
> + /* The original scalar offset, which is a non-loop-invariant SSA_NAME. */
> + tree offset;
> +
> + /* The definition type for the vectorized offset. */
> + enum vect_def_type offset_dt;
> +
> + /* The type of the vectorized offset. */
> + tree offset_vectype;
> +
> + /* Each offset element should be multiplied by this amount before
> + being added to the base. */
> + int scale;
> +};
> +
> /* Access Functions. */
> #define STMT_VINFO_TYPE(S) (S)->type
> #define STMT_VINFO_STMT(S) (S)->stmt
> @@ -1035,8 +1057,8 @@ extern bool vect_verify_datarefs_alignment
> (loop_vec_info);
> extern bool vect_slp_analyze_and_verify_instance_alignment (slp_instance);
> extern bool vect_analyze_data_ref_accesses (vec_info *);
> extern bool vect_prune_runtime_alias_test_list (loop_vec_info);
> -extern tree vect_check_gather_scatter (gimple *, loop_vec_info, tree *, tree
> *,
> - int *);
> +extern bool vect_check_gather_scatter (gimple *, loop_vec_info,
> + gather_scatter_info *);
> extern bool vect_analyze_data_refs (vec_info *, int *);
> extern tree vect_create_data_ref_ptr (gimple *, tree, struct loop *, tree,
> tree *, gimple_stmt_iterator *,
> Index: gcc/tree-vect-data-refs.c
> ===================================================================
> --- gcc/tree-vect-data-refs.c
> +++ gcc/tree-vect-data-refs.c
> @@ -3174,12 +3174,12 @@ vect_prune_runtime_alias_test_list (loop_vec_info
> loop_vinfo)
> return true;
> }
>
> -/* Check whether a non-affine read or write in stmt is suitable for gather
> load
> - or scatter store and if so, return a builtin decl for that operation. */
> +/* Return true if a non-affine read or write in STMT is suitable for a
> + gather load or scatter store. Describe the operation in *INFO if so. */
>
> -tree
> -vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree
> *basep,
> - tree *offp, int *scalep)
> +bool
> +vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo,
> + gather_scatter_info *info)
> {
> HOST_WIDE_INT scale = 1, pbitpos, pbitsize;
> struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
> @@ -3253,7 +3253,7 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info
> loop_vinfo, tree *basep,
> if (!expr_invariant_in_loop_p (loop, base))
> {
> if (!integer_zerop (off))
> - return NULL_TREE;
> + return false;
> off = base;
> base = size_int (pbitpos / BITS_PER_UNIT);
> }
> @@ -3279,7 +3279,7 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info
> loop_vinfo, tree *basep,
> gimple *def_stmt = SSA_NAME_DEF_STMT (off);
>
> if (expr_invariant_in_loop_p (loop, off))
> - return NULL_TREE;
> + return false;
>
> if (gimple_code (def_stmt) != GIMPLE_ASSIGN)
> break;
> @@ -3291,7 +3291,7 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info
> loop_vinfo, tree *basep,
> else
> {
> if (get_gimple_rhs_class (TREE_CODE (off)) == GIMPLE_TERNARY_RHS)
> - return NULL_TREE;
> + return false;
> code = TREE_CODE (off);
> extract_ops_from_tree (off, &code, &op0, &op1);
> }
> @@ -3366,7 +3366,7 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info
> loop_vinfo, tree *basep,
> defined in the loop, punt. */
> if (TREE_CODE (off) != SSA_NAME
> || expr_invariant_in_loop_p (loop, off))
> - return NULL_TREE;
> + return false;
>
> if (offtype == NULL_TREE)
> offtype = TREE_TYPE (off);
> @@ -3379,15 +3379,15 @@ vect_check_gather_scatter (gimple *stmt,
> loop_vec_info loop_vinfo, tree *basep,
> offtype, scale);
>
> if (decl == NULL_TREE)
> - return NULL_TREE;
> -
> - if (basep)
> - *basep = base;
> - if (offp)
> - *offp = off;
> - if (scalep)
> - *scalep = scale;
> - return decl;
> + return false;
> +
> + info->decl = decl;
> + info->base = base;
> + info->offset = off;
> + info->offset_dt = vect_unknown_def_type;
> + info->offset_vectype = NULL_TREE;
> + info->scale = scale;
> + return true;
> }
>
> /* Function vect_analyze_data_refs.
> @@ -3865,10 +3865,10 @@ again:
>
> if (gatherscatter != SG_NONE)
> {
> - tree off;
> + gather_scatter_info gs_info;
> if (!vect_check_gather_scatter (stmt, as_a <loop_vec_info> (vinfo),
> - NULL, &off, NULL)
> - || get_vectype_for_scalar_type (TREE_TYPE (off)) == NULL_TREE)
> + &gs_info)
> + || !get_vectype_for_scalar_type (TREE_TYPE (gs_info.offset)))
> {
> STMT_VINFO_DATA_REF (stmt_info) = NULL;
> free_data_ref (dr);
> Index: gcc/tree-vect-stmts.c
> ===================================================================
> --- gcc/tree-vect-stmts.c
> +++ gcc/tree-vect-stmts.c
> @@ -772,10 +772,11 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info
> loop_vinfo)
>
> if (STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo))
> {
> - tree off;
> - tree decl = vect_check_gather_scatter (stmt, loop_vinfo, NULL,
> &off, NULL);
> - gcc_assert (decl);
> - if (!process_use (stmt, off, loop_vinfo, relevant, &worklist, true))
> + gather_scatter_info gs_info;
> + if (!vect_check_gather_scatter (stmt, loop_vinfo, &gs_info))
> + gcc_unreachable ();
> + if (!process_use (stmt, gs_info.offset, loop_vinfo, relevant,
> + &worklist, true))
> return false;
> }
> } /* while worklist */
> @@ -1703,10 +1704,7 @@ vectorizable_mask_load_store (gimple *stmt,
> gimple_stmt_iterator *gsi,
> int ncopies;
> int i, j;
> bool inv_p;
> - tree gather_base = NULL_TREE, gather_off = NULL_TREE;
> - tree gather_off_vectype = NULL_TREE, gather_decl = NULL_TREE;
> - int gather_scale = 1;
> - enum vect_def_type gather_dt = vect_unknown_def_type;
> + gather_scatter_info gs_info;
> bool is_store;
> tree mask;
> gimple *def_stmt;
> @@ -1774,11 +1772,10 @@ vectorizable_mask_load_store (gimple *stmt,
> gimple_stmt_iterator *gsi,
> if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> {
> gimple *def_stmt;
> - gather_decl = vect_check_gather_scatter (stmt, loop_vinfo,
> &gather_base,
> - &gather_off, &gather_scale);
> - gcc_assert (gather_decl);
> - if (!vect_is_simple_use (gather_off, loop_vinfo, &def_stmt, &gather_dt,
> - &gather_off_vectype))
> + if (!vect_check_gather_scatter (stmt, loop_vinfo, &gs_info))
> + gcc_unreachable ();
> + if (!vect_is_simple_use (gs_info.offset, loop_vinfo, &def_stmt,
> + &gs_info.offset_dt, &gs_info.offset_vectype))
> {
> if (dump_enabled_p ())
> dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> @@ -1786,7 +1783,7 @@ vectorizable_mask_load_store (gimple *stmt,
> gimple_stmt_iterator *gsi,
> return false;
> }
>
> - tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gather_decl));
> + tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl));
> tree masktype
> = TREE_VALUE (TREE_CHAIN (TREE_CHAIN (TREE_CHAIN (arglist))));
> if (TREE_CODE (masktype) == INTEGER_TYPE)
> @@ -1825,7 +1822,7 @@ vectorizable_mask_load_store (gimple *stmt,
> gimple_stmt_iterator *gsi,
> if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> {
> tree vec_oprnd0 = NULL_TREE, op;
> - tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gather_decl));
> + tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl));
> tree rettype, srctype, ptrtype, idxtype, masktype, scaletype;
> tree ptr, vec_mask = NULL_TREE, mask_op = NULL_TREE, var, scale;
> tree perm_mask = NULL_TREE, prev_res = NULL_TREE;
> @@ -1834,9 +1831,9 @@ vectorizable_mask_load_store (gimple *stmt,
> gimple_stmt_iterator *gsi,
> gimple_seq seq;
> basic_block new_bb;
> enum { NARROW, NONE, WIDEN } modifier;
> - int gather_off_nunits = TYPE_VECTOR_SUBPARTS (gather_off_vectype);
> + int gather_off_nunits = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype);
>
> - rettype = TREE_TYPE (TREE_TYPE (gather_decl));
> + rettype = TREE_TYPE (TREE_TYPE (gs_info.decl));
> srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
> ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
> idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
> @@ -1855,7 +1852,7 @@ vectorizable_mask_load_store (gimple *stmt,
> gimple_stmt_iterator *gsi,
> for (i = 0; i < gather_off_nunits; ++i)
> sel[i] = i | nunits;
>
> - perm_mask = vect_gen_perm_mask_checked (gather_off_vectype, sel);
> + perm_mask = vect_gen_perm_mask_checked (gs_info.offset_vectype,
> sel);
> }
> else if (nunits == gather_off_nunits * 2)
> {
> @@ -1877,7 +1874,7 @@ vectorizable_mask_load_store (gimple *stmt,
> gimple_stmt_iterator *gsi,
>
> vec_dest = vect_create_destination_var (gimple_call_lhs (stmt),
> vectype);
>
> - ptr = fold_convert (ptrtype, gather_base);
> + ptr = fold_convert (ptrtype, gs_info.base);
> if (!is_gimple_min_invariant (ptr))
> {
> ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE);
> @@ -1885,7 +1882,7 @@ vectorizable_mask_load_store (gimple *stmt,
> gimple_stmt_iterator *gsi,
> gcc_assert (!new_bb);
> }
>
> - scale = build_int_cst (scaletype, gather_scale);
> + scale = build_int_cst (scaletype, gs_info.scale);
>
> prev_stmt_info = NULL;
> for (j = 0; j < ncopies; ++j)
> @@ -1895,10 +1892,10 @@ vectorizable_mask_load_store (gimple *stmt,
> gimple_stmt_iterator *gsi,
> perm_mask, stmt, gsi);
> else if (j == 0)
> op = vec_oprnd0
> - = vect_get_vec_def_for_operand (gather_off, stmt);
> + = vect_get_vec_def_for_operand (gs_info.offset, stmt);
> else
> op = vec_oprnd0
> - = vect_get_vec_def_for_stmt_copy (gather_dt, vec_oprnd0);
> + = vect_get_vec_def_for_stmt_copy (gs_info.offset_dt,
> vec_oprnd0);
>
> if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
> {
> @@ -1940,7 +1937,7 @@ vectorizable_mask_load_store (gimple *stmt,
> gimple_stmt_iterator *gsi,
> }
>
> new_stmt
> - = gimple_build_call (gather_decl, 5, mask_op, ptr, op, mask_op,
> + = gimple_build_call (gs_info.decl, 5, mask_op, ptr, op, mask_op,
> scale);
>
> if (!useless_type_conversion_p (vectype, rettype))
> @@ -5207,10 +5204,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator
> *gsi, gimple **vec_stmt,
> bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
> vec_info *vinfo = stmt_info->vinfo;
> tree aggr_type;
> - tree scatter_base = NULL_TREE, scatter_off = NULL_TREE;
> - tree scatter_off_vectype = NULL_TREE, scatter_decl = NULL_TREE;
> - int scatter_scale = 1;
> - enum vect_def_type scatter_idx_dt = vect_unknown_def_type;
> + gather_scatter_info gs_info;
> enum vect_def_type scatter_src_dt = vect_unknown_def_type;
> gimple *new_stmt;
>
> @@ -5378,11 +5372,10 @@ vectorizable_store (gimple *stmt,
> gimple_stmt_iterator *gsi, gimple **vec_stmt,
> if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> {
> gimple *def_stmt;
> - scatter_decl = vect_check_gather_scatter (stmt, loop_vinfo,
> &scatter_base,
> - &scatter_off, &scatter_scale);
> - gcc_assert (scatter_decl);
> - if (!vect_is_simple_use (scatter_off, vinfo, &def_stmt,
> &scatter_idx_dt,
> - &scatter_off_vectype))
> + if (!vect_check_gather_scatter (stmt, loop_vinfo, &gs_info))
> + gcc_unreachable ();
> + if (!vect_is_simple_use (gs_info.offset, vinfo, &def_stmt,
> + &gs_info.offset_dt, &gs_info.offset_vectype))
> {
> if (dump_enabled_p ())
> dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> @@ -5408,14 +5401,14 @@ vectorizable_store (gimple *stmt,
> gimple_stmt_iterator *gsi, gimple **vec_stmt,
> if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> {
> tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE, op, src;
> - tree arglist = TYPE_ARG_TYPES (TREE_TYPE (scatter_decl));
> + tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl));
> tree rettype, srctype, ptrtype, idxtype, masktype, scaletype;
> tree ptr, mask, var, scale, perm_mask = NULL_TREE;
> edge pe = loop_preheader_edge (loop);
> gimple_seq seq;
> basic_block new_bb;
> enum { NARROW, NONE, WIDEN } modifier;
> - int scatter_off_nunits = TYPE_VECTOR_SUBPARTS (scatter_off_vectype);
> + int scatter_off_nunits = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype);
>
> if (nunits == (unsigned int) scatter_off_nunits)
> modifier = NONE;
> @@ -5427,7 +5420,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator
> *gsi, gimple **vec_stmt,
> for (i = 0; i < (unsigned int) scatter_off_nunits; ++i)
> sel[i] = i | nunits;
>
> - perm_mask = vect_gen_perm_mask_checked (scatter_off_vectype, sel);
> + perm_mask = vect_gen_perm_mask_checked (gs_info.offset_vectype,
> sel);
> gcc_assert (perm_mask != NULL_TREE);
> }
> else if (nunits == (unsigned int) scatter_off_nunits * 2)
> @@ -5445,7 +5438,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator
> *gsi, gimple **vec_stmt,
> else
> gcc_unreachable ();
>
> - rettype = TREE_TYPE (TREE_TYPE (scatter_decl));
> + rettype = TREE_TYPE (TREE_TYPE (gs_info.decl));
> ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
> masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
> idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
> @@ -5455,7 +5448,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator
> *gsi, gimple **vec_stmt,
> gcc_checking_assert (TREE_CODE (masktype) == INTEGER_TYPE
> && TREE_CODE (rettype) == VOID_TYPE);
>
> - ptr = fold_convert (ptrtype, scatter_base);
> + ptr = fold_convert (ptrtype, gs_info.base);
> if (!is_gimple_min_invariant (ptr))
> {
> ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE);
> @@ -5468,7 +5461,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator
> *gsi, gimple **vec_stmt,
> mask = build_int_cst (masktype, -1);
> mask = vect_init_vector (stmt, mask, masktype, NULL);
>
> - scale = build_int_cst (scaletype, scatter_scale);
> + scale = build_int_cst (scaletype, gs_info.scale);
>
> prev_stmt_info = NULL;
> for (j = 0; j < ncopies; ++j)
> @@ -5478,7 +5471,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator
> *gsi, gimple **vec_stmt,
> src = vec_oprnd1
> = vect_get_vec_def_for_operand (gimple_assign_rhs1 (stmt),
> stmt);
> op = vec_oprnd0
> - = vect_get_vec_def_for_operand (scatter_off, stmt);
> + = vect_get_vec_def_for_operand (gs_info.offset, stmt);
> }
> else if (modifier != NONE && (j & 1))
> {
> @@ -5494,7 +5487,8 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator
> *gsi, gimple **vec_stmt,
> src = permute_vec_elements (vec_oprnd1, vec_oprnd1,
> perm_mask,
> stmt, gsi);
> op = vec_oprnd0
> - = vect_get_vec_def_for_stmt_copy (scatter_idx_dt,
> vec_oprnd0);
> + = vect_get_vec_def_for_stmt_copy (gs_info.offset_dt,
> + vec_oprnd0);
> }
> else
> gcc_unreachable ();
> @@ -5504,7 +5498,8 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator
> *gsi, gimple **vec_stmt,
> src = vec_oprnd1
> = vect_get_vec_def_for_stmt_copy (scatter_src_dt, vec_oprnd1);
> op = vec_oprnd0
> - = vect_get_vec_def_for_stmt_copy (scatter_idx_dt, vec_oprnd0);
> + = vect_get_vec_def_for_stmt_copy (gs_info.offset_dt,
> + vec_oprnd0);
> }
>
> if (!useless_type_conversion_p (srctype, TREE_TYPE (src)))
> @@ -5530,7 +5525,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator
> *gsi, gimple **vec_stmt,
> }
>
> new_stmt
> - = gimple_build_call (scatter_decl, 5, ptr, mask, op, src, scale);
> + = gimple_build_call (gs_info.decl, 5, ptr, mask, op, src, scale);
>
> vect_finish_stmt_generation (stmt, new_stmt, gsi);
>
> @@ -6204,10 +6199,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator
> *gsi, gimple **vec_stmt,
> bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
> int vf;
> tree aggr_type;
> - tree gather_base = NULL_TREE, gather_off = NULL_TREE;
> - tree gather_off_vectype = NULL_TREE, gather_decl = NULL_TREE;
> - int gather_scale = 1;
> - enum vect_def_type gather_dt = vect_unknown_def_type;
> + gather_scatter_info gs_info;
> vec_info *vinfo = stmt_info->vinfo;
>
> if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
> @@ -6404,11 +6396,10 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator
> *gsi, gimple **vec_stmt,
> if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> {
> gimple *def_stmt;
> - gather_decl = vect_check_gather_scatter (stmt, loop_vinfo,
> &gather_base,
> - &gather_off, &gather_scale);
> - gcc_assert (gather_decl);
> - if (!vect_is_simple_use (gather_off, vinfo, &def_stmt, &gather_dt,
> - &gather_off_vectype))
> + if (!vect_check_gather_scatter (stmt, loop_vinfo, &gs_info))
> + gcc_unreachable ();
> + if (!vect_is_simple_use (gs_info.offset, vinfo, &def_stmt,
> + &gs_info.offset_dt, &gs_info.offset_vectype))
> {
> if (dump_enabled_p ())
> dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> @@ -6483,14 +6474,14 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator
> *gsi, gimple **vec_stmt,
> if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> {
> tree vec_oprnd0 = NULL_TREE, op;
> - tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gather_decl));
> + tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl));
> tree rettype, srctype, ptrtype, idxtype, masktype, scaletype;
> tree ptr, mask, var, scale, merge, perm_mask = NULL_TREE, prev_res =
> NULL_TREE;
> edge pe = loop_preheader_edge (loop);
> gimple_seq seq;
> basic_block new_bb;
> enum { NARROW, NONE, WIDEN } modifier;
> - int gather_off_nunits = TYPE_VECTOR_SUBPARTS (gather_off_vectype);
> + int gather_off_nunits = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype);
>
> if (nunits == gather_off_nunits)
> modifier = NONE;
> @@ -6502,7 +6493,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator
> *gsi, gimple **vec_stmt,
> for (i = 0; i < gather_off_nunits; ++i)
> sel[i] = i | nunits;
>
> - perm_mask = vect_gen_perm_mask_checked (gather_off_vectype, sel);
> + perm_mask = vect_gen_perm_mask_checked (gs_info.offset_vectype,
> sel);
> }
> else if (nunits == gather_off_nunits * 2)
> {
> @@ -6519,7 +6510,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator
> *gsi, gimple **vec_stmt,
> else
> gcc_unreachable ();
>
> - rettype = TREE_TYPE (TREE_TYPE (gather_decl));
> + rettype = TREE_TYPE (TREE_TYPE (gs_info.decl));
> srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
> ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
> idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
> @@ -6529,7 +6520,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator
> *gsi, gimple **vec_stmt,
>
> vec_dest = vect_create_destination_var (scalar_dest, vectype);
>
> - ptr = fold_convert (ptrtype, gather_base);
> + ptr = fold_convert (ptrtype, gs_info.base);
> if (!is_gimple_min_invariant (ptr))
> {
> ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE);
> @@ -6561,7 +6552,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator
> *gsi, gimple **vec_stmt,
> else
> gcc_unreachable ();
>
> - scale = build_int_cst (scaletype, gather_scale);
> + scale = build_int_cst (scaletype, gs_info.scale);
>
> if (TREE_CODE (TREE_TYPE (rettype)) == INTEGER_TYPE)
> merge = build_int_cst (TREE_TYPE (rettype), 0);
> @@ -6587,10 +6578,10 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator
> *gsi, gimple **vec_stmt,
> perm_mask, stmt, gsi);
> else if (j == 0)
> op = vec_oprnd0
> - = vect_get_vec_def_for_operand (gather_off, stmt);
> + = vect_get_vec_def_for_operand (gs_info.offset, stmt);
> else
> op = vec_oprnd0
> - = vect_get_vec_def_for_stmt_copy (gather_dt, vec_oprnd0);
> + = vect_get_vec_def_for_stmt_copy (gs_info.offset_dt,
> vec_oprnd0);
>
> if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
> {
> @@ -6605,7 +6596,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator
> *gsi, gimple **vec_stmt,
> }
>
> new_stmt
> - = gimple_build_call (gather_decl, 5, merge, ptr, op, mask, scale);
> + = gimple_build_call (gs_info.decl, 5, merge, ptr, op, mask,
> scale);
>
> if (!useless_type_conversion_p (vectype, rettype))
> {
