The following removes the STMT_VINFO_VECTYPE usage from
vect_check_gather_scatter and instead gets the vector type passed
in from the caller. For the two early callers, the one from
dataref analysis and the one from vect_mark_stmts_to_be_vectorized
we add a mode that works with a NULL vector type - this mode
is intended to look for a base/offset pair that's suitable
for processing, meaning an invariant base and a SSA offset, but
without influencing this discovery by target capabilities
(as if forced to do emulation). This is to allow, in the next
step, to elide setting STMT_VINFO_VECTYPE during dataref
analysis.
* tree-vectorizer.h (vect_check_gather_scatter):
Add vectype argument.
* tree-vect-data-refs.cc (vect_check_gather_scatter):
Get vectype as argument, do early processing when NULL.
(vect_describe_gather_scatter_call): Likewise.
(vect_analyze_data_refs): Do not perform vectype
specific analysis on gather/scatter refs.
* tree-vect-patterns.cc (vect_recog_gather_scatter_pattern):
Use the data element vector type to perform gather/scatter
analysis.
* tree-vect-slp.cc (vect_get_and_check_slp_defs): Get
the vector type as argument and pass it along.
(vect_build_slp_tree_2): Adjust.
(vect_detect_hybrid_slp): Use the basic analysis from
vect_check_gather_scatter.
* tree-vect-stmts.cc (vect_mark_stmts_to_be_vectorized):
Likewise.
(vect_use_strided_gather_scatters_p): Get vectype and
pass along.
(get_group_load_store_type): Pass down vectype.
---
gcc/tree-vect-data-refs.cc | 46 ++++++++++++++++++++++----------------
gcc/tree-vect-patterns.cc | 9 +++++---
gcc/tree-vect-slp.cc | 8 +++----
gcc/tree-vect-stmts.cc | 12 +++++-----
gcc/tree-vectorizer.h | 2 +-
5 files changed, 45 insertions(+), 32 deletions(-)
diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc
index 0601d592ca3..b62498b34bd 100644
--- a/gcc/tree-vect-data-refs.cc
+++ b/gcc/tree-vect-data-refs.cc
@@ -4535,10 +4535,10 @@ vect_gather_scatter_fn_p (vec_info *vinfo, bool read_p,
bool masked_p,
static void
vect_describe_gather_scatter_call (stmt_vec_info stmt_info,
+ tree vectype,
gather_scatter_info *info)
{
gcall *call = as_a <gcall *> (stmt_info->stmt);
- tree vectype = STMT_VINFO_VECTYPE (stmt_info);
data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
info->ifn = gimple_call_internal_fn (call);
@@ -4548,17 +4548,20 @@ vect_describe_gather_scatter_call (stmt_vec_info
stmt_info,
info->offset_dt = vect_unknown_def_type;
info->offset_vectype = NULL_TREE;
info->scale = TREE_INT_CST_LOW (gimple_call_arg (call, 2));
- info->element_type = TREE_TYPE (vectype);
+ info->element_type = vectype ? TREE_TYPE (vectype) : NULL_TREE;
info->memory_type = TREE_TYPE (DR_REF (dr));
}
/* Return true if a non-affine read or write in STMT_INFO is suitable for a
- gather load or scatter store. Describe the operation in *INFO if so.
+ gather load or scatter store. Describe the operation in *INFO if so,
+ when INFO is NULL only ensure there's a loop invariant base and a loop
+ variant offset.
If it is suitable and ELSVALS is nonzero store the supported else values
in the vector it points to. */
bool
-vect_check_gather_scatter (stmt_vec_info stmt_info, loop_vec_info loop_vinfo,
+vect_check_gather_scatter (stmt_vec_info stmt_info, tree vectype,
+ loop_vec_info loop_vinfo,
gather_scatter_info *info, vec<int> *elsvals)
{
HOST_WIDE_INT scale = 1;
@@ -4567,7 +4570,6 @@ vect_check_gather_scatter (stmt_vec_info stmt_info,
loop_vec_info loop_vinfo,
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
tree offtype = NULL_TREE;
tree decl = NULL_TREE, base, off;
- tree vectype = STMT_VINFO_VECTYPE (stmt_info);
tree memory_type = TREE_TYPE (DR_REF (dr));
machine_mode pmode;
int punsignedp, reversep, pvolatilep = 0;
@@ -4583,7 +4585,7 @@ vect_check_gather_scatter (stmt_vec_info stmt_info,
loop_vec_info loop_vinfo,
ifn = gimple_call_internal_fn (call);
if (internal_gather_scatter_fn_p (ifn))
{
- vect_describe_gather_scatter_call (stmt_info, info);
+ vect_describe_gather_scatter_call (stmt_info, vectype, info);
/* In pattern recog we simply used a ZERO else value that
we need to correct here. To that end just re-use the
@@ -4605,10 +4607,11 @@ vect_check_gather_scatter (stmt_vec_info stmt_info,
loop_vec_info loop_vinfo,
/* True if we should aim to use internal functions rather than
built-in functions. */
- bool use_ifn_p = (DR_IS_READ (dr)
- ? supports_vec_gather_load_p (TYPE_MODE (vectype),
- elsvals)
- : supports_vec_scatter_store_p (TYPE_MODE (vectype)));
+ bool use_ifn_p = false;
+ if (vectype)
+ use_ifn_p = (DR_IS_READ (dr)
+ ? supports_vec_gather_load_p (TYPE_MODE (vectype), elsvals)
+ : supports_vec_scatter_store_p (TYPE_MODE (vectype)));
base = DR_REF (dr);
/* For masked loads/stores, DR_REF (dr) is an artificial MEM_REF,
@@ -4708,7 +4711,7 @@ vect_check_gather_scatter (stmt_vec_info stmt_info,
loop_vec_info loop_vinfo,
if (expr_invariant_in_loop_p (loop, off))
return false;
- if (gimple_code (def_stmt) != GIMPLE_ASSIGN)
+ if (!vectype || gimple_code (def_stmt) != GIMPLE_ASSIGN)
break;
op0 = gimple_assign_rhs1 (def_stmt);
@@ -4759,7 +4762,8 @@ vect_check_gather_scatter (stmt_vec_info stmt_info,
loop_vec_info loop_vinfo,
int new_scale = tree_to_shwi (op1);
/* Only treat this as a scaling operation if the target
supports it for at least some offset type. */
- if (use_ifn_p
+ if (TREE_CODE (op0) == SSA_NAME
+ && use_ifn_p
&& !vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr),
masked_p, vectype, memory_type,
signed_char_type_node,
@@ -4810,7 +4814,8 @@ vect_check_gather_scatter (stmt_vec_info stmt_info,
loop_vec_info loop_vinfo,
data vector element size. */
if ((TYPE_PRECISION (TREE_TYPE (op0))
< TYPE_PRECISION (TREE_TYPE (off)))
- && (use_ifn_p
+ && (TREE_CODE (off) != SSA_NAME
+ || use_ifn_p
|| (DR_IS_READ (dr)
? (targetm.vectorize.builtin_gather
&& targetm.vectorize.builtin_gather (vectype,
@@ -4844,7 +4849,12 @@ vect_check_gather_scatter (stmt_vec_info stmt_info,
loop_vec_info loop_vinfo,
if (offtype == NULL_TREE)
offtype = TREE_TYPE (off);
- if (use_ifn_p)
+ if (! vectype)
+ {
+ ifn = IFN_LAST;
+ decl = NULL_TREE;
+ }
+ else if (use_ifn_p)
{
if (!vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr), masked_p,
vectype, memory_type, offtype, scale,
@@ -4876,7 +4886,7 @@ vect_check_gather_scatter (stmt_vec_info stmt_info,
loop_vec_info loop_vinfo,
info->offset_dt = vect_unknown_def_type;
info->offset_vectype = offset_vectype;
info->scale = scale;
- info->element_type = TREE_TYPE (vectype);
+ info->element_type = vectype ? TREE_TYPE (vectype) : NULL_TREE;
info->memory_type = memory_type;
return true;
}
@@ -5279,11 +5289,9 @@ vect_analyze_data_refs (vec_info *vinfo, bool *fatal)
if (gatherscatter != SG_NONE)
{
gather_scatter_info gs_info;
- if (!vect_check_gather_scatter (stmt_info,
+ if (!vect_check_gather_scatter (stmt_info, NULL,
as_a <loop_vec_info> (vinfo),
- &gs_info)
- || !get_vectype_for_scalar_type (vinfo,
- TREE_TYPE (gs_info.offset)))
+ &gs_info))
{
if (fatal)
*fatal = false;
diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
index 0f6d6b77ea1..340c36c73f2 100644
--- a/gcc/tree-vect-patterns.cc
+++ b/gcc/tree-vect-patterns.cc
@@ -6003,16 +6003,19 @@ vect_recog_gather_scatter_pattern (vec_info *vinfo,
This is null if the operation is unconditional. */
tree mask = vect_get_load_store_mask (stmt_info);
+ tree gs_vectype = get_vectype_for_scalar_type (loop_vinfo,
+ TREE_TYPE (DR_REF (dr)));
+
/* Make sure that the target supports an appropriate internal
function for the gather/scatter operation. */
gather_scatter_info gs_info;
- if (!vect_check_gather_scatter (stmt_info, loop_vinfo, &gs_info)
+ if (!gs_vectype
+ || !vect_check_gather_scatter (stmt_info, gs_vectype, loop_vinfo,
+ &gs_info)
|| gs_info.ifn == IFN_LAST)
return NULL;
/* Convert the mask to the right form. */
- tree gs_vectype = get_vectype_for_scalar_type (loop_vinfo,
- gs_info.element_type);
if (mask)
mask = vect_convert_mask_for_vectype (mask, gs_vectype, stmt_info,
loop_vinfo);
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index 5f414790354..826b0f341dc 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -636,7 +636,7 @@ vect_slp_child_index_for_operand (const gimple *stmt, int
op,
swapping operands of father node of this one, return 1; if everything is
ok return 0. */
static int
-vect_get_and_check_slp_defs (vec_info *vinfo, unsigned char swap,
+vect_get_and_check_slp_defs (vec_info *vinfo, tree vectype, unsigned char swap,
bool *skip_args,
vec<stmt_vec_info> stmts, unsigned stmt_num,
vec<slp_oprnd_info> *oprnds_info)
@@ -697,7 +697,7 @@ vect_get_and_check_slp_defs (vec_info *vinfo, unsigned char
swap,
{
gcc_assert (STMT_VINFO_GATHER_SCATTER_P (stmt_info));
if (!is_a <loop_vec_info> (vinfo)
- || !vect_check_gather_scatter (stmt_info,
+ || !vect_check_gather_scatter (stmt_info, vectype,
as_a <loop_vec_info> (vinfo),
first ? &oprnd_info->first_gs_info
: &gs_info))
@@ -2604,7 +2604,7 @@ out:
slp_oprnd_info oprnd_info;
FOR_EACH_VEC_ELT (stmts, i, stmt_info)
{
- int res = vect_get_and_check_slp_defs (vinfo, swap[i], skip_args,
+ int res = vect_get_and_check_slp_defs (vinfo, vectype, swap[i],
skip_args,
stmts, i, &oprnds_info);
if (res != 0)
matches[(res == -1) ? 0 : i] = false;
@@ -7817,7 +7817,7 @@ vect_detect_hybrid_slp (loop_vec_info loop_vinfo)
can be a scaling and conversion away. */
gather_scatter_info gs_info;
if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
- && vect_check_gather_scatter (stmt_info, loop_vinfo, &gs_info))
+ && vect_check_gather_scatter (stmt_info, NULL, loop_vinfo, &gs_info))
{
int dummy;
vect_detect_hybrid_slp (&gs_info.offset, &dummy, &wi);
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 26913481e25..0b5e5af7992 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -884,7 +884,8 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo,
bool *fatal)
if (STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo))
{
gather_scatter_info gs_info;
- if (!vect_check_gather_scatter (stmt_vinfo, loop_vinfo, &gs_info))
+ if (!vect_check_gather_scatter (stmt_vinfo, NULL,
+ loop_vinfo, &gs_info))
gcc_unreachable ();
opt_result res
= process_use (stmt_vinfo, gs_info.offset, loop_vinfo, relevant,
@@ -1831,12 +1832,13 @@ vect_truncate_gather_scatter_offset (stmt_vec_info
stmt_info,
else values will be stored in the vector ELSVALS points to. */
static bool
-vect_use_strided_gather_scatters_p (stmt_vec_info stmt_info,
+vect_use_strided_gather_scatters_p (stmt_vec_info stmt_info, tree vectype,
loop_vec_info loop_vinfo, bool masked_p,
gather_scatter_info *gs_info,
vec<int> *elsvals)
{
- if (!vect_check_gather_scatter (stmt_info, loop_vinfo, gs_info, elsvals)
+ if (!vect_check_gather_scatter (stmt_info, vectype, loop_vinfo,
+ gs_info, elsvals)
|| gs_info->ifn == IFN_LAST)
return vect_truncate_gather_scatter_offset (stmt_info, loop_vinfo,
masked_p, gs_info, elsvals);
@@ -2354,7 +2356,7 @@ get_group_load_store_type (vec_info *vinfo, stmt_vec_info
stmt_info,
&& single_element_p
&& SLP_TREE_LANES (slp_node) == 1
&& loop_vinfo
- && vect_use_strided_gather_scatters_p (stmt_info, loop_vinfo,
+ && vect_use_strided_gather_scatters_p (stmt_info, vectype, loop_vinfo,
masked_p, gs_info, elsvals))
*memory_access_type = VMAT_GATHER_SCATTER;
@@ -2448,7 +2450,7 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info
stmt_info,
if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
{
*memory_access_type = VMAT_GATHER_SCATTER;
- if (!vect_check_gather_scatter (stmt_info, loop_vinfo, gs_info,
+ if (!vect_check_gather_scatter (stmt_info, vectype, loop_vinfo, gs_info,
elsvals))
gcc_unreachable ();
/* When using internal functions, we rely on pattern recognition
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index e15b0492791..ddf31e7b0ee 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -2541,7 +2541,7 @@ extern opt_result vect_prune_runtime_alias_test_list
(loop_vec_info);
extern bool vect_gather_scatter_fn_p (vec_info *, bool, bool, tree, tree,
tree, int, internal_fn *, tree *,
vec<int> * = nullptr);
-extern bool vect_check_gather_scatter (stmt_vec_info, loop_vec_info,
+extern bool vect_check_gather_scatter (stmt_vec_info, tree, loop_vec_info,
gather_scatter_info *,
vec<int> * = nullptr);
extern opt_result vect_find_stmt_data_reference (loop_p, gimple *,
--
2.43.0
