> Ah, I see. How the code is factored makes this a bit iffy IMO.
> Can't we refactor things somehow to avoid this "special value"?
Time for a store else value? :-x
Attached is a v2, there is a bit of code duplication now but I think not too
bad. Sniff regtest on riscv64 and x86 was successful, rest of the targets are
running.
Regards
Robin
[PATCH v2] fold: Elide MASK_LEN_LOAD/STORE with zero length [PR122635].
This patch adds zero-length handling to gimple_fold_partial_store and
gimple_fold_partial_load. If length + bias is zero, we replace the
load with its else value and remove a store altogether.
At the same time the patch removes the mask_p argument of
gimple_fold_partial_load and _store. We can easily get the mask index
from the IFN.
PR tree-optimization/122635
gcc/ChangeLog:
* gimple-fold.cc (gimple_fold_partial_load_store_mem_ref):
Replace zero-length load with zero constant.
(partial_load_store_elidible_p): New function.
(gimple_fold_partial_load): Remove mask_p argument.
(gimple_fold_partial_store): Remove store if elidible.
(gimple_fold_call): Remove mask_p argument.
gcc/testsuite/ChangeLog:
* gcc.target/riscv/rvv/autovec/pr122635-1.c: New test.
* gcc.target/riscv/rvv/autovec/pr122635-2.c: New test.
* gcc.target/powerpc/p9-vec-length-epil-8.c: Expect two lxvl
less.
---
gcc/gimple-fold.cc | 116 ++++++++++++------
.../gcc.target/powerpc/p9-vec-length-epil-8.c | 2 +-
.../gcc.target/riscv/rvv/autovec/pr122635-1.c | 20 +++
.../gcc.target/riscv/rvv/autovec/pr122635-2.c | 18 +++
4 files changed, 116 insertions(+), 40 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/pr122635-1.c
create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/pr122635-2.c
diff --git a/gcc/gimple-fold.cc b/gcc/gimple-fold.cc
index 3fc76313622..60169f92655 100644
--- a/gcc/gimple-fold.cc
+++ b/gcc/gimple-fold.cc
@@ -5757,48 +5757,80 @@ arith_overflowed_p (enum tree_code code, const_tree
type,
return wi::min_precision (wres, sign) > TYPE_PRECISION (type);
}
+/* Check if IFN_{MASK,LEN,MASK_LEN}_LOAD/STORE call CALL is unconditional and
+ has a zero length. If so, it can be safely elided. */
+
+static bool
+partial_load_store_elidible_p (gcall *call)
+{
+ internal_fn ifn = gimple_call_internal_fn (call);
+
+ int mask_index = internal_fn_mask_index (ifn);
+ if (mask_index != -1)
+ {
+ tree mask = gimple_call_arg (call, mask_index);
+ if (!integer_all_onesp (mask))
+ return false;
+ }
+
+ int len_index = internal_fn_len_index (ifn);
+ tree len = gimple_call_arg (call, len_index);
+ tree bias = gimple_call_arg (call, len_index + 1);
+ gcc_assert (TREE_CODE (bias) == INTEGER_CST);
+
+ if (poly_int_tree_p (len))
+ {
+ poly_widest_int wlen = wi::to_poly_widest (len)
+ + wi::to_widest (bias);
+ return known_eq (wlen, 0);
+ }
+
+ return false;
+}
+
+
/* If IFN_{MASK,LEN,MASK_LEN}_LOAD/STORE call CALL is unconditional,
return a MEM_REF for the memory it references, otherwise return null.
- VECTYPE is the type of the memory vector. MASK_P indicates it's for
- MASK if true, otherwise it's for LEN. */
+ VECTYPE is the type of the memory vector. */
static tree
-gimple_fold_partial_load_store_mem_ref (gcall *call, tree vectype, bool mask_p)
+gimple_fold_partial_load_store_mem_ref (gcall *call, tree vectype)
{
tree ptr = gimple_call_arg (call, 0);
tree alias_align = gimple_call_arg (call, 1);
if (!tree_fits_uhwi_p (alias_align))
return NULL_TREE;
- if (mask_p)
+ internal_fn ifn = gimple_call_internal_fn (call);
+ int mask_index = internal_fn_mask_index (ifn);
+ int len_index = internal_fn_len_index (ifn);
+
+ if (mask_index != -1)
{
- tree mask = gimple_call_arg (call, 2);
+ tree mask = gimple_call_arg (call, mask_index);
if (!integer_all_onesp (mask))
return NULL_TREE;
}
- else
+
+ if (len_index != -1)
{
- internal_fn ifn = gimple_call_internal_fn (call);
- int len_index = internal_fn_len_index (ifn);
- tree basic_len = gimple_call_arg (call, len_index);
- if (!poly_int_tree_p (basic_len))
+ tree len = gimple_call_arg (call, len_index);
+ if (!poly_int_tree_p (len))
return NULL_TREE;
tree bias = gimple_call_arg (call, len_index + 1);
gcc_assert (TREE_CODE (bias) == INTEGER_CST);
- /* For LEN_LOAD/LEN_STORE/MASK_LEN_LOAD/MASK_LEN_STORE,
- we don't fold when (bias + len) != VF. */
- if (maybe_ne (wi::to_poly_widest (basic_len) + wi::to_widest (bias),
- GET_MODE_NUNITS (TYPE_MODE (vectype))))
- return NULL_TREE;
-
- /* For MASK_LEN_{LOAD,STORE}, we should also check whether
- the mask is all ones mask. */
- if (ifn == IFN_MASK_LEN_LOAD || ifn == IFN_MASK_LEN_STORE)
+ poly_widest_int wlen = wi::to_poly_widest (len)
+ + wi::to_widest (bias);
+ if (known_eq (wlen, 0) && internal_fn_else_index (ifn))
{
- tree mask = gimple_call_arg (call, internal_fn_mask_index (ifn));
- if (!integer_all_onesp (mask))
- return NULL_TREE;
+ /* Length is 0. Replace with the else operand. */
+ int else_index = internal_fn_else_index (ifn);
+ return gimple_call_arg (call, else_index);
}
+ /* For LEN_LOAD/LEN_STORE/MASK_LEN_LOAD/MASK_LEN_STORE,
+ we don't fold when len + bias != VF. */
+ else if (maybe_ne (wlen, GET_MODE_NUNITS (TYPE_MODE (vectype))))
+ return NULL_TREE;
}
unsigned HOST_WIDE_INT align = tree_to_uhwi (alias_align);
@@ -5808,18 +5840,17 @@ gimple_fold_partial_load_store_mem_ref (gcall *call,
tree vectype, bool mask_p)
return fold_build2 (MEM_REF, vectype, ptr, offset);
}
-/* Try to fold IFN_{MASK,LEN}_LOAD call CALL. Return true on success.
- MASK_P indicates it's for MASK if true, otherwise it's for LEN. */
+/* Try to fold IFN_{MASK,LEN}_LOAD call CALL. Return true on success. */
static bool
-gimple_fold_partial_load (gimple_stmt_iterator *gsi, gcall *call, bool mask_p)
+gimple_fold_partial_load (gimple_stmt_iterator *gsi, gcall *call)
{
tree lhs = gimple_call_lhs (call);
if (!lhs)
return false;
if (tree rhs
- = gimple_fold_partial_load_store_mem_ref (call, TREE_TYPE (lhs), mask_p))
+ = gimple_fold_partial_load_store_mem_ref (call, TREE_TYPE (lhs)))
{
gassign *new_stmt = gimple_build_assign (lhs, rhs);
gimple_set_location (new_stmt, gimple_location (call));
@@ -5830,17 +5861,16 @@ gimple_fold_partial_load (gimple_stmt_iterator *gsi,
gcall *call, bool mask_p)
return false;
}
-/* Try to fold IFN_{MASK,LEN}_STORE call CALL. Return true on success.
- MASK_P indicates it's for MASK if true, otherwise it's for LEN. */
+/* Try to fold IFN_{MASK,LEN}_STORE call CALL. Return true on success. */
static bool
-gimple_fold_partial_store (gimple_stmt_iterator *gsi, gcall *call,
- bool mask_p)
+gimple_fold_partial_store (gimple_stmt_iterator *gsi, gcall *call)
{
internal_fn ifn = gimple_call_internal_fn (call);
+
tree rhs = gimple_call_arg (call, internal_fn_stored_value_index (ifn));
if (tree lhs
- = gimple_fold_partial_load_store_mem_ref (call, TREE_TYPE (rhs), mask_p))
+ = gimple_fold_partial_load_store_mem_ref (call, TREE_TYPE (rhs)))
{
gassign *new_stmt = gimple_build_assign (lhs, rhs);
gimple_set_location (new_stmt, gimple_location (call));
@@ -5848,6 +5878,18 @@ gimple_fold_partial_store (gimple_stmt_iterator *gsi,
gcall *call,
gsi_replace (gsi, new_stmt, false);
return true;
}
+
+ /* gimple_fold_partial_load_store_mem_ref can replace a zero-length load
+ with its else value but not a store as those don't have an else value.
+ Therefore remove those here. */
+ if (partial_load_store_elidible_p (call))
+ {
+ unlink_stmt_vdef (call);
+ release_defs (call);
+ gsi_replace (gsi, gimple_build_nop (), true);
+ return true;
+ }
+
return false;
}
@@ -6075,19 +6117,15 @@ gimple_fold_call (gimple_stmt_iterator *gsi, bool
inplace)
cplx_result = true;
uaddc_usubc = true;
break;
- case IFN_MASK_LOAD:
- changed |= gimple_fold_partial_load (gsi, stmt, true);
- break;
- case IFN_MASK_STORE:
- changed |= gimple_fold_partial_store (gsi, stmt, true);
- break;
case IFN_LEN_LOAD:
+ case IFN_MASK_LOAD:
case IFN_MASK_LEN_LOAD:
- changed |= gimple_fold_partial_load (gsi, stmt, false);
+ changed |= gimple_fold_partial_load (gsi, stmt);
break;
case IFN_LEN_STORE:
+ case IFN_MASK_STORE:
case IFN_MASK_LEN_STORE:
- changed |= gimple_fold_partial_store (gsi, stmt, false);
+ changed |= gimple_fold_partial_store (gsi, stmt);
break;
default:
break;
diff --git a/gcc/testsuite/gcc.target/powerpc/p9-vec-length-epil-8.c
b/gcc/testsuite/gcc.target/powerpc/p9-vec-length-epil-8.c
index 34a2c8eb11b..5dff0d0ceb9 100644
--- a/gcc/testsuite/gcc.target/powerpc/p9-vec-length-epil-8.c
+++ b/gcc/testsuite/gcc.target/powerpc/p9-vec-length-epil-8.c
@@ -13,5 +13,5 @@
#include "p9-vec-length-8.h"
-/* { dg-final { scan-assembler-times {\mlxvl\M} 16 } } */
+/* { dg-final { scan-assembler-times {\mlxvl\M} 14 } } */
/* { dg-final { scan-assembler-times {\mstxvl\M} 7 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/pr122635-1.c
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/pr122635-1.c
new file mode 100644
index 00000000000..0beb3d70866
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/pr122635-1.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=rv64gcv_zvl256b -mabi=lp64d -mrvv-vector-bits=zvl
-mno-autovec-segment" } */
+
+typedef struct {
+ int a[6];
+ float b[3];
+} c;
+
+int d(c *e) {
+ int f =0;
+ for (; f < 3; f++) {
+ e->a[2 * f] = e->b[f];
+ e->a[2 * f + 1] = -e->a[2 * f];
+ e->a[2 * f] = f + 3 * e->a[2 * f];
+ e->a[2 * f + 1] = f + 3 * e->a[2 * f + 1];
+ }
+ return 0;
+}
+
+/* { dg-final { scan-assembler-not "vsetivli.*zero,0" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/pr122635-2.c
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/pr122635-2.c
new file mode 100644
index 00000000000..0de69b52cb0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/pr122635-2.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=rv64gcv_zvl256b -mabi=lp64d -mrvv-vector-bits=zvl
-mno-autovec-segment" } */
+
+typedef struct {
+ int A[6];
+ float b[];
+} a;
+
+int b(a *a) {
+ int b = 0;
+ for (; b < 3; b++) {
+ a->A[2 * b] = a->b[b] - b + a->A[2 * b];
+ a->A[2 * b + 1] = b * a->A[2 * b + 1];
+ }
+ return 0;
+}
+
+/* { dg-final { scan-assembler-not "vsetivli.*zero,0" } } */
--
2.51.1
