The pass aborted analysis of the entire remaining basic block when it
encountered a complex memory operation (non-simple store/load).
Store-forwarding opportunities after the complex operation were silently
missed.
Replace the early return with a flush-and-continue: clear the pending
store candidates and keep scanning. The two other flush sites
(throwing insns, unknown-size memory) already use this pattern.
gcc/ChangeLog:
* avoid-store-forwarding.cc
(store_forwarding_analyzer::avoid_store_forwarding): Replace
return with flush-and-continue for complex memory operations.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/avoid-store-forwarding-6.c: New test.
Signed-off-by: Konstantinos Eleftheriou <[email protected]>
---
gcc/avoid-store-forwarding.cc | 39 +++++++++++--------
.../aarch64/avoid-store-forwarding-6.c | 18 +++++++++
2 files changed, 40 insertions(+), 17 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/aarch64/avoid-store-forwarding-6.c
diff --git a/gcc/avoid-store-forwarding.cc b/gcc/avoid-store-forwarding.cc
index 0c065ba1a75a..bb015681c458 100644
--- a/gcc/avoid-store-forwarding.cc
+++ b/gcc/avoid-store-forwarding.cc
@@ -691,30 +691,35 @@ store_forwarding_analyzer::avoid_store_forwarding
(basic_block bb)
process_store_forwarding (forwardings, insn, load_mem);
}
- /* Abort in case that we encounter a memory read/write that is not a
- simple store/load, as we can't make safe assumptions about the
- side-effects of this. */
- if ((writes_mem && !is_simple_store)
- || (reads_mem && !is_simple_load))
- return;
-
- if (removed_count)
+ /* If we encounter a memory read/write that is not a simple
+ store/load, flush all pending store candidates and continue.
+ We can't make safe assumptions about the side-effects, but
+ store-forwarding opportunities later in the BB should still
+ be analyzed. */
+ if ((writes_mem && !is_simple_store)
+ || (reads_mem && !is_simple_load))
+ {
+ store_exprs.truncate (0);
+ continue;
+ }
+
+ if (removed_count)
{
unsigned int i, j;
store_fwd_info *it;
VEC_ORDERED_REMOVE_IF (store_exprs, i, j, it, it->remove);
}
- /* Don't consider store forwarding if the RTL instruction distance is
- more than PARAM_STORE_FORWARDING_MAX_DISTANCE and the cost checks
- are not disabled. */
- const bool unlimited_cost = (param_store_forwarding_max_distance == 0);
- if (!unlimited_cost && !store_exprs.is_empty ()
- && (store_exprs[0].insn_cnt
- + param_store_forwarding_max_distance <= insn_cnt))
- store_exprs.ordered_remove (0);
+ /* Don't consider store forwarding if the RTL instruction distance is
+ more than PARAM_STORE_FORWARDING_MAX_DISTANCE and the cost checks
+ are not disabled. */
+ const bool unlimited_cost = (param_store_forwarding_max_distance == 0);
+ if (!unlimited_cost && !store_exprs.is_empty ()
+ && (store_exprs[0].insn_cnt
+ + param_store_forwarding_max_distance <= insn_cnt))
+ store_exprs.ordered_remove (0);
- insn_cnt++;
+ insn_cnt++;
}
}
diff --git a/gcc/testsuite/gcc.target/aarch64/avoid-store-forwarding-6.c
b/gcc/testsuite/gcc.target/aarch64/avoid-store-forwarding-6.c
new file mode 100644
index 000000000000..150235e9649a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/avoid-store-forwarding-6.c
@@ -0,0 +1,18 @@
+/* Check that the pass continues after a complex memory op (volatile load). */
+/* { dg-do compile } */
+/* { dg-options "-O2 -favoid-store-forwarding
-fdump-rtl-avoid_store_forwarding" } */
+
+typedef union {
+ char arr_8[8];
+ long long_value;
+} DataUnion;
+
+long ssll_complex_mem (DataUnion *data, char x, volatile int *v)
+{
+ (void)*v;
+ data->arr_8[4] = x;
+ return data->long_value;
+}
+
+/* { dg-final { scan-rtl-dump-times "Store forwarding detected" 1
"avoid_store_forwarding" } } */
+/* { dg-final { scan-rtl-dump-times "Store forwarding avoided" 1
"avoid_store_forwarding" } } */
--
2.52.0
