The following makes it possible to have can_vec_perm_const_p guards
that can tell us whether vector lowering would have lowered the
permute if false. For this purpose we have to enhance the API
to allow anticipation of the vec_shl/vec_shr expansion path.
So this adds two boolean flags indicating whether the permuted
vectors are known zero and move the vector lowering detection
code to can_vec_perm_const_p which makes both consistent.
For now I have adjusted only vector lowering and match.pd. ISTR
the vectorizer explicitly has some vec_shl/shr matching that can
be cleaned up. Up for discussion might be whether to pass
down the actual permuted vectors rather than flags, the latter
looked enough and also suitable for use from RTL.
Bootstrapped and tested on x86_64-unknown-linux-gnu.
OK?
For branches I'll probably add a after-vector-lowering guard
to the patterns instead.
Thanks,
Richard.
PR middle-end/125875
* optabs-query.h (can_vec_perm_const_p): Add defaulted
params to indicate the permuted vectors are bitwise zero.
* optabs-query.cc (can_vec_perm_const_p): Handle permute
patterns mapping to vec_shl/vec_shr.
* tree-vect-generic.cc (lower_vec_perm): Move vec_shl/vec_shr
detection to can_vec_perm_const_p and simplify.
* match.pd: When checking for !can_vec_perm_const_p also
allow variable permutes and pass in whether the permuted
vectors are known constant zero.
* gcc.dg/torture/pr125875.c: New testcase.
---
gcc/match.pd | 36 +++++++++++----
gcc/optabs-query.cc | 55 ++++++++++++++++++++---
gcc/optabs-query.h | 3 +-
gcc/testsuite/gcc.dg/torture/pr125875.c | 29 ++++++++++++
gcc/tree-vect-generic.cc | 59 ++-----------------------
5 files changed, 111 insertions(+), 71 deletions(-)
create mode 100644 gcc/testsuite/gcc.dg/torture/pr125875.c
diff --git a/gcc/match.pd b/gcc/match.pd
index 8c410c2f3b3..7fecb4466bb 100644
--- a/gcc/match.pd
+++ b/gcc/match.pd
@@ -11720,6 +11720,9 @@ and,
vec_perm_indices sel2 (builder2, 2, nelts);
+ bool op2_zerop = initializer_zerop (@2);
+ bool op3_zerop = initializer_zerop (@3);
+ bool op0_zerop = initializer_zerop (@0);
tree op0 = NULL_TREE;
/* If the new VEC_PERM_EXPR can't be handled but both
original VEC_PERM_EXPRs can, punt.
@@ -11728,9 +11731,12 @@ and,
number of VEC_PERM_EXPRs that can't be handled. */
if (can_vec_perm_const_p (result_mode, op_mode, sel2, false)
|| (single_use (@0)
- ? (!can_vec_perm_const_p (result_mode, op_mode, sel0, false)
- || !can_vec_perm_const_p (result_mode, op_mode, sel1, false))
- : !can_vec_perm_const_p (result_mode, op_mode, sel1, false)))
+ ? (!can_vec_perm_const_p (result_mode, op_mode, sel0, true,
+ op2_zerop, op3_zerop)
+ || !can_vec_perm_const_p (result_mode, op_mode, sel1, true,
+ false, op0_zerop))
+ : !can_vec_perm_const_p (result_mode, op_mode, sel1, true,
+ false, op0_zerop)))
op0 = vec_perm_indices_to_tree (TREE_TYPE (@5), sel2);
}
(if (op0)
@@ -11788,6 +11794,9 @@ and,
{
vec_perm_indices sel2 (builder2, 2, nelts);
tree op0 = NULL_TREE;
+ bool op1_zerop = initializer_zerop (@1);
+ bool op2_zerop = initializer_zerop (@2);
+ bool op5_zerop = initializer_zerop (@5);
/* If the new VEC_PERM_EXPR can't be handled but both
original VEC_PERM_EXPRs can, punt.
If one or both of the original VEC_PERM_EXPRs can't be
@@ -11795,9 +11804,12 @@ and,
number of VEC_PERM_EXPRs that can't be handled. */
if (can_vec_perm_const_p (result_mode, op_mode, sel2, false)
|| (single_use (@0)
- ? (!can_vec_perm_const_p (result_mode, op_mode, sel0, false)
- || !can_vec_perm_const_p (result_mode, op_mode, sel1,
false))
- : !can_vec_perm_const_p (result_mode, op_mode, sel1, false)))
+ ? (!can_vec_perm_const_p (result_mode, op_mode, sel0, true,
+ op1_zerop, op2_zerop)
+ || !can_vec_perm_const_p (result_mode, op_mode, sel1, true,
+ op5_zerop, false))
+ : !can_vec_perm_const_p (result_mode, op_mode, sel1, true,
+ op5_zerop, false)))
op0 = vec_perm_indices_to_tree (TREE_TYPE (@4), sel2);
}
(if (op0)
@@ -11852,6 +11864,9 @@ and,
{
vec_perm_indices sel2 (builder2, 2, nelts);
tree op0 = NULL_TREE;
+ bool op1_zerop = initializer_zerop (@1);
+ bool op2_zerop = initializer_zerop (@2);
+ bool op5_zerop = initializer_zerop (@5);
/* If the new VEC_PERM_EXPR can't be handled but both
original VEC_PERM_EXPRs can, punt.
If one or both of the original VEC_PERM_EXPRs can't be
@@ -11859,9 +11874,12 @@ and,
number of VEC_PERM_EXPRs that can't be handled. */
if (can_vec_perm_const_p (result_mode, op_mode, sel2, false)
|| (single_use (@0)
- ? (!can_vec_perm_const_p (result_mode, op_mode, sel0, false)
- || !can_vec_perm_const_p (result_mode, op_mode, sel1,
false))
- : !can_vec_perm_const_p (result_mode, op_mode, sel1, false)))
+ ? (!can_vec_perm_const_p (result_mode, op_mode, sel0, true,
+ op1_zerop, op2_zerop)
+ || !can_vec_perm_const_p (result_mode, op_mode, sel1, true,
+ false, op5_zerop))
+ : !can_vec_perm_const_p (result_mode, op_mode, sel1, true,
+ false, op5_zerop)))
op0 = vec_perm_indices_to_tree (TREE_TYPE (@4), sel2);
}
(if (op0)
diff --git a/gcc/optabs-query.cc b/gcc/optabs-query.cc
index 58842e40ed6..2e74acb0664 100644
--- a/gcc/optabs-query.cc
+++ b/gcc/optabs-query.cc
@@ -417,14 +417,13 @@ can_vec_perm_var_p (machine_mode mode)
ALLOW_VARIABLE_P is true if it is acceptable to force the selector into a
register and use a variable permute (if the target supports that).
- Note that additional permutations representing whole-vector shifts may
- also be handled via the vec_shr or vec_shl optab, but only where the
- second input vector is entirely constant zeroes; this case is not dealt
- with here. */
+ If OP0_ZEROP or OP1_ZEROP additional permutations representing whole-vector
+ shifts may also be handled via the vec_shr or vec_shl optab. */
bool
can_vec_perm_const_p (machine_mode mode, machine_mode op_mode,
- const vec_perm_indices &sel, bool allow_variable_p)
+ const vec_perm_indices &sel, bool allow_variable_p,
+ bool op0_zerop, bool op1_zerop)
{
/* If the target doesn't implement a vector mode for the vector type,
then no operations are supported. */
@@ -465,6 +464,52 @@ can_vec_perm_const_p (machine_mode mode, machine_mode
op_mode,
into integer operations. */
}
+ unsigned elements;
+ if (mode == op_mode
+ && GET_MODE_NUNITS (mode).is_constant (&elements))
+ {
+ if (op0_zerop
+ && can_implement_p (vec_shl_optab, mode))
+ {
+ unsigned int first = 0, i;
+ for (i = 0; i < elements; ++i)
+ if (known_eq (poly_uint64 (sel[i]), elements))
+ {
+ if (i == 0 || first)
+ break;
+ first = i;
+ }
+ else if (first
+ ? maybe_ne (poly_uint64 (sel[i]),
+ elements + i - first)
+ : maybe_ge (poly_uint64 (sel[i]), elements))
+ break;
+ if (first && i == elements)
+ return true;
+ }
+ if (op1_zerop
+ && maybe_ne (sel[0], 0)
+ && known_lt (sel[0], elements)
+ && can_implement_p (vec_shr_optab, mode))
+ {
+ if (sel.series_p (0, 1, sel[0], 1))
+ return true;
+ unsigned i;
+ for (i = 1; i < elements; ++i)
+ {
+ poly_uint64 actual = sel[i];
+ poly_uint64 expected = i + sel[0];
+ /* Indices into the second vector are all equivalent. */
+ if (maybe_lt (actual, elements)
+ ? maybe_ne (actual, expected)
+ : maybe_lt (expected, elements))
+ break;
+ }
+ if (i == elements)
+ return true;
+ }
+ }
+
return false;
}
diff --git a/gcc/optabs-query.h b/gcc/optabs-query.h
index faa688a0609..4a7eb1036f2 100644
--- a/gcc/optabs-query.h
+++ b/gcc/optabs-query.h
@@ -156,7 +156,8 @@ opt_machine_mode qimode_for_vec_perm (machine_mode);
bool selector_fits_mode_p (machine_mode, const vec_perm_indices &);
bool can_vec_perm_var_p (machine_mode);
bool can_vec_perm_const_p (machine_mode, machine_mode,
- const vec_perm_indices &, bool = true);
+ const vec_perm_indices &, bool = true,
+ bool op0_zerop = false, bool op1_zerop = false);
/* Find a widening optab even if it doesn't widen as much as we want. */
#define find_widening_optab_handler(A, B, C) \
find_widening_optab_handler_and_mode (A, B, C, NULL)
diff --git a/gcc/testsuite/gcc.dg/torture/pr125875.c
b/gcc/testsuite/gcc.dg/torture/pr125875.c
new file mode 100644
index 00000000000..0cccb190c73
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr125875.c
@@ -0,0 +1,29 @@
+/* { dg-do compile } */
+
+#include <stdint.h>
+typedef int16_t v4i16 __attribute__((vector_size(8)));
+v4i16 g7, g19;
+int64_t g16;
+_Bool f16_a0_c3, f16_a0_ob6;
+void f16_a0()
+{
+ int32_t ov10;
+ uint64_t __ov_tmp_g16 = __builtin_sub_overflow(0, 0, &__ov_tmp_g16);
+ g16 = __ov_tmp_g16;
+lbl_b1:
+ __builtin_mul_overflow(g16, g16, &ov10);
+ switch (ov10)
+ case 33:
+ goto lbl_sw_def57;
+lbl_sw_def15:
+ if (f16_a0_c3) goto lbl_b1;
+ g7 = __builtin_shufflevector(g7, g7, 4, 3, 4, 5);
+ f16_a0_ob6 = f16_a0;
+ g19 = g19 <= g19;
+lbl_bf43:
+ g19 = ~g19;
+ g7 = __builtin_shufflevector(g7, g19, 6, 7, 0, 1);
+lbl_sw_def57:
+ if (f16_a0_ob6) goto lbl_sw_def15;
+ goto lbl_bf43;
+}
diff --git a/gcc/tree-vect-generic.cc b/gcc/tree-vect-generic.cc
index fddb44bfe86..e43b749e8cc 100644
--- a/gcc/tree-vect-generic.cc
+++ b/gcc/tree-vect-generic.cc
@@ -1649,67 +1649,14 @@ lower_vec_perm (gimple_stmt_iterator *gsi)
machine_mode vmode = TYPE_MODE (vect_type);
tree lhs_type = TREE_TYPE (gimple_assign_lhs (stmt));
machine_mode lhs_mode = TYPE_MODE (lhs_type);
- if (can_vec_perm_const_p (lhs_mode, vmode, indices))
+ if (can_vec_perm_const_p (lhs_mode, vmode, indices, true,
+ initializer_zerop (vec0),
+ initializer_zerop (vec1)))
{
gimple_assign_set_rhs3 (stmt, mask);
update_stmt (stmt);
return;
}
- /* Also detect vec_shr pattern - VEC_PERM_EXPR with zero
- vector as VEC1 and a right element shift MASK. */
- if (can_implement_p (vec_shr_optab, TYPE_MODE (vect_type))
- && TREE_CODE (vec1) == VECTOR_CST
- && initializer_zerop (vec1)
- && maybe_ne (indices[0], 0)
- && known_lt (poly_uint64 (indices[0]), elements))
- {
- bool ok_p = indices.series_p (0, 1, indices[0], 1);
- if (!ok_p)
- {
- for (i = 1; i < elements; ++i)
- {
- poly_uint64 actual = indices[i];
- poly_uint64 expected = i + indices[0];
- /* Indices into the second vector are all equivalent. */
- if (maybe_lt (actual, elements)
- ? maybe_ne (actual, expected)
- : maybe_lt (expected, elements))
- break;
- }
- ok_p = i == elements;
- }
- if (ok_p)
- {
- gimple_assign_set_rhs3 (stmt, mask);
- update_stmt (stmt);
- return;
- }
- }
- /* And similarly vec_shl pattern. */
- if (can_implement_p (vec_shl_optab, TYPE_MODE (vect_type))
- && TREE_CODE (vec0) == VECTOR_CST
- && initializer_zerop (vec0))
- {
- unsigned int first = 0;
- for (i = 0; i < elements; ++i)
- if (known_eq (poly_uint64 (indices[i]), elements))
- {
- if (i == 0 || first)
- break;
- first = i;
- }
- else if (first
- ? maybe_ne (poly_uint64 (indices[i]),
- elements + i - first)
- : maybe_ge (poly_uint64 (indices[i]), elements))
- break;
- if (first && i == elements)
- {
- gimple_assign_set_rhs3 (stmt, mask);
- update_stmt (stmt);
- return;
- }
- }
}
else if (can_vec_perm_var_p (TYPE_MODE (vect_type)))
return;
--
2.51.0
