https://gcc.gnu.org/g:2c25af0e52a631e46a1731594301e5e63bc28992
commit r15-5013-g2c25af0e52a631e46a1731594301e5e63bc28992
Author: Richard Biener <rguent...@suse.de>
Date: Mon Nov 4 13:09:21 2024 +0100
Add LOOP_VINFO_DRS_ADVANCED_BY
The following remembers how we advanced DRs when vectorizing an
epilogue. When we want to vectorize the epilogue of such epilogue
we have to retain that advancement and add the advancement for this
vectorized epilogue. Due to the way we copy and re-associate
stmt_vec_infos and DRs recording this advancement and re-applying
it for the next epilogue is simplest.
* tree-vectorizer.h (_loop_vec_info::drs_advanced_by): New.
(LOOP_VINFO_DRS_ADVANCED_BY): Likewise.
* tree-vect-loop.cc (_loop_vec_info::_loop_vec_info): Initialize
drs_advanced_by.
(update_epilogue_loop_vinfo): Remember the DR advancement made.
(vect_transform_loop): Accumulate past advancements.
Diff:
---
gcc/tree-vect-loop.cc | 9 +++++++++
gcc/tree-vectorizer.h | 4 ++++
2 files changed, 13 insertions(+)
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index e91549a643b9..6059ce031d14 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -1072,6 +1072,7 @@ _loop_vec_info::_loop_vec_info (class loop *loop_in,
vec_info_shared *shared)
scalar_loop_scaling (profile_probability::uninitialized ()),
scalar_loop (NULL),
orig_loop_info (NULL),
+ drs_advanced_by (NULL_TREE),
vec_loop_iv_exit (NULL),
vec_epilogue_loop_iv_exit (NULL),
scalar_loop_iv_exit (NULL)
@@ -12302,6 +12303,9 @@ update_epilogue_loop_vinfo (class loop *epilogue, tree
advance)
loop and its prologue. */
vect_update_inits_of_drs (epilogue_vinfo, advance, PLUS_EXPR);
+ /* Remember the advancement made. */
+ LOOP_VINFO_DRS_ADVANCED_BY (epilogue_vinfo) = advance;
+
epilogue_vinfo->shared->datarefs_copy.release ();
epilogue_vinfo->shared->save_datarefs ();
}
@@ -12849,6 +12853,11 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple
*loop_vectorized_call)
if (epilogue)
{
+ /* Accumulate past advancements made. */
+ if (LOOP_VINFO_DRS_ADVANCED_BY (loop_vinfo))
+ advance = fold_build2 (PLUS_EXPR, TREE_TYPE (advance),
+ LOOP_VINFO_DRS_ADVANCED_BY (loop_vinfo),
+ advance);
update_epilogue_loop_vinfo (epilogue, advance);
epilogue->simduid = loop->simduid;
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 24227a69d4ac..5a1bd237beb6 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -994,6 +994,9 @@ public:
analysis. */
vec<_loop_vec_info *> epilogue_vinfos;
+ /* If this is an epilogue loop the DR advancement applied. */
+ tree drs_advanced_by;
+
/* The controlling loop IV for the current loop when vectorizing. This IV
controls the natural exits of the loop. */
edge vec_loop_iv_exit;
@@ -1097,6 +1100,7 @@ public:
#define LOOP_VINFO_SIMD_IF_COND(L) (L)->simd_if_cond
#define LOOP_VINFO_INNER_LOOP_COST_FACTOR(L) (L)->inner_loop_cost_factor
#define LOOP_VINFO_INV_PATTERN_DEF_SEQ(L) (L)->inv_pattern_def_seq
+#define LOOP_VINFO_DRS_ADVANCED_BY(L) (L)->drs_advanced_by
#define LOOP_VINFO_FULLY_MASKED_P(L) \
(LOOP_VINFO_USING_PARTIAL_VECTORS_P (L) \
