gcc/ChangeLog:
* config/i386/sse.md (V_ANY_CBRANCH): New mode iterator.
(cond_vec_cbranch_any<mode>): New define_expand.
(cond_vec_cbranch_all<mode>): Ditto.
gcc/testsuite/ChangeLog:
* gcc.target/i386/vect-cbranch-cond-any-128.c: New test.
* gcc.target/i386/vect-cbranch-cond-any-256.c: Ditto.
* gcc.target/i386/vect-cbranch-cond-any-run.c: Ditto.
* gcc.target/i386/vect-cbranch-cond-all-128.c: Ditto.
* gcc.target/i386/vect-cbranch-cond-all-256.c: Ditto.
* gcc.target/i386/vect-cbranch-cond-all-run.c: Ditto.
---
gcc/config/i386/sse.md | 72 +++++++++++++++
.../i386/vect-cbranch-cond-all-128.c | 87 +++++++++++++++++++
.../i386/vect-cbranch-cond-all-256.c | 47 ++++++++++
.../i386/vect-cbranch-cond-all-run.c | 72 +++++++++++++++
.../i386/vect-cbranch-cond-any-128.c | 86 ++++++++++++++++++
.../i386/vect-cbranch-cond-any-256.c | 46 ++++++++++
.../i386/vect-cbranch-cond-any-run.c | 67 ++++++++++++++
7 files changed, 477 insertions(+)
create mode 100644 gcc/testsuite/gcc.target/i386/vect-cbranch-cond-all-128.c
create mode 100644 gcc/testsuite/gcc.target/i386/vect-cbranch-cond-all-256.c
create mode 100644 gcc/testsuite/gcc.target/i386/vect-cbranch-cond-all-run.c
create mode 100644 gcc/testsuite/gcc.target/i386/vect-cbranch-cond-any-128.c
create mode 100644 gcc/testsuite/gcc.target/i386/vect-cbranch-cond-any-256.c
create mode 100644 gcc/testsuite/gcc.target/i386/vect-cbranch-cond-any-run.c
diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md
index a632e256e91..7d0415b5add 100644
--- a/gcc/config/i386/sse.md
+++ b/gcc/config/i386/sse.md
@@ -29765,6 +29765,78 @@ (define_expand "cbranch<mode>4"
DONE;
})
+;; vec_cbranch modes with fewer than 8 elements (integer + floating-point).
+;; Limited to AVX512VL+DQ so that kortestb is available for the QImode mask.
+(define_mode_iterator V_ANY_CBRANCH
+ [(V4SI "TARGET_AVX512VL && TARGET_AVX512DQ")
+ (V2DI "TARGET_AVX512VL && TARGET_AVX512DQ")
+ (V4DI "TARGET_AVX512VL && TARGET_AVX512DQ")
+ (V4SF "TARGET_AVX512VL && TARGET_AVX512DQ")
+ (V4DF "TARGET_AVX512VL && TARGET_AVX512DQ")
+ (V2DF "TARGET_AVX512VL && TARGET_AVX512DQ")])
+
+;; Masked variant of vec_cbranch_any. Eliminates the redundant kmov + and
+;; + test sequence the vectorizer would otherwise emit when an early-break
+;; is guarded by a loop mask.
+(define_expand "cond_vec_cbranch_any<mode>"
+ [(set (pc)
+ (if_then_else
+ (match_operator 0 ""
+ [(match_operand:<avx512fmaskmode> 1 "register_operand")
+ (match_operand:V_ANY_CBRANCH 2 "register_operand")
+ (match_operand:V_ANY_CBRANCH 3 "nonimmediate_operand")])
+ (label_ref (match_operand 4 ""))
+ (pc)))]
+ ""
+{
+ machine_mode mask_mode = <avx512fmaskmode>mode;
+ rtx cmp_mask = gen_reg_rtx (mask_mode);
+ rtx and_mask = gen_reg_rtx (mask_mode);
+
+ emit_insn (gen_vec_cmp<mode><avx512fmaskmodelower> (cmp_mask,
+ operands[0],
+ operands[2],
+ operands[3]));
+ and_mask = expand_simple_binop (mask_mode, AND, cmp_mask, operands[1],
+ and_mask, 1, OPTAB_DIRECT);
+
+ rtx cc_reg = gen_rtx_REG (CCZmode, FLAGS_REG);
+ emit_insn (gen_rtx_SET (cc_reg,
+ gen_rtx_COMPARE (CCZmode, and_mask, const0_rtx)));
+ ix86_expand_branch (NE, cc_reg, const0_rtx, operands[4]);
+ DONE;
+})
+
+;; Reached when the middle-end reverses an NE comparison to EQ and swaps the
+;; branch labels.
+(define_expand "cond_vec_cbranch_all<mode>"
+ [(set (pc)
+ (if_then_else
+ (match_operator 0 ""
+ [(match_operand:<avx512fmaskmode> 1 "register_operand")
+ (match_operand:V_ANY_CBRANCH 2 "register_operand")
+ (match_operand:V_ANY_CBRANCH 3 "nonimmediate_operand")])
+ (label_ref (match_operand 4 ""))
+ (pc)))]
+ ""
+{
+ machine_mode mask_mode = <avx512fmaskmode>mode;
+ rtx cmp_mask = gen_reg_rtx (mask_mode);
+ rtx and_mask = gen_reg_rtx (mask_mode);
+
+ emit_insn (gen_vec_cmp<mode><avx512fmaskmodelower> (cmp_mask,
+ operands[0],
+ operands[2],
+ operands[3]));
+ and_mask = expand_simple_binop (mask_mode, AND, cmp_mask, operands[1],
+ and_mask, 1, OPTAB_DIRECT);
+
+ rtx cc_reg = gen_rtx_REG (CCZmode, FLAGS_REG);
+ emit_insn (gen_rtx_SET (cc_reg,
+ gen_rtx_COMPARE (CCZmode, and_mask, const0_rtx)));
+ ix86_expand_branch (EQ, cc_reg, const0_rtx, operands[4]);
+ DONE;
+})
(define_insn_and_split "avx_<castmode><avxsizesuffix>_<castmode>"
[(set (match_operand:AVX256MODE2P 0 "nonimmediate_operand" "=x,m")
diff --git a/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-all-128.c
b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-all-128.c
new file mode 100644
index 00000000000..f12178d4a90
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-all-128.c
@@ -0,0 +1,87 @@
+/* Test cond_vec_cbranch_all<mode> for the 128-bit V_ANY_CBRANCH modes
+ (V4SI, V2DI, V4SF, V2DF). */
+/* { dg-do compile } */
+/* { dg-options "-Ofast -mavx512f -mavx512vl -mavx512dq
-mprefer-vector-width=128 -fno-schedule-insns -fno-reorder-blocks
-fno-schedule-insns2" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#define N 100000
+int a_int[N], b_int[N], c_int[N];
+long long a_long[N], b_long[N], c_long[N];
+float a_float[N], b_float[N], c_float[N];
+double a_double[N], b_double[N], c_double[N];
+
+/*
+** f_v4si:
+** ...
+** vpcmpd \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+
+** vpcmpd \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+\{%k[0-9]+\}
+** kortestb %k[0-9]+, %k[0-9]+
+** je \.L[0-9]+
+** ...
+*/
+void f_v4si (int d, int e)
+{
+ for (int i = 0; i < N; i++)
+ {
+ b_int[i] += a_int[i];
+ if (c_int[i] == e) continue;
+ if (a_int[i] != d) break;
+ }
+}
+
+/*
+** f_v2di:
+** ...
+** vpcmpq \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+
+** vpcmpq \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+\{%k[0-9]+\}
+** kortestb %k[0-9]+, %k[0-9]+
+** je \.L[0-9]+
+** ...
+*/
+void f_v2di (long long d, long long e)
+{
+ for (int i = 0; i < N; i++)
+ {
+ b_long[i] += a_long[i];
+ if (c_long[i] == e) continue;
+ if (a_long[i] != d) break;
+ }
+}
+
+/*
+** f_v4sf:
+** ...
+** vcmpps \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+
+** vcmpps \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+\{%k[0-9]+\}
+** kortestb %k[0-9]+, %k[0-9]+
+** je \.L[0-9]+
+** ...
+*/
+void f_v4sf (float d, float e)
+{
+ for (int i = 0; i < N; i++)
+ {
+ b_float[i] += a_float[i];
+ if (c_float[i] == e) continue;
+ if (a_float[i] != d) break;
+ }
+}
+
+/*
+** f_v2df:
+** ...
+** vcmppd \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+
+** vcmppd \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+\{%k[0-9]+\}
+** kortestb %k[0-9]+, %k[0-9]+
+** je \.L[0-9]+
+** ...
+*/
+void f_v2df (double d, double e)
+{
+ for (int i = 0; i < N; i++)
+ {
+ b_double[i] += a_double[i];
+ if (c_double[i] == e) continue;
+ if (a_double[i] != d) break;
+ }
+}
diff --git a/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-all-256.c
b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-all-256.c
new file mode 100644
index 00000000000..8abad799280
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-all-256.c
@@ -0,0 +1,47 @@
+/* Test cond_vec_cbranch_all<mode> for the 256-bit V_ANY_CBRANCH modes
+ (V4DI, V4DF). */
+/* { dg-do compile } */
+/* { dg-options "-Ofast -mavx512f -mavx512vl -mavx512dq
-mprefer-vector-width=256 -fno-schedule-insns -fno-reorder-blocks
-fno-schedule-insns2" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#define N 100000
+long long a_long[N], b_long[N], c_long[N];
+double a_double[N], b_double[N], c_double[N];
+
+/*
+** f_v4di:
+** ...
+** vpcmpq \$4, %ymm[0-9]+, %ymm[0-9]+, %k[0-9]+
+** vpcmpq \$4, %ymm[0-9]+, %ymm[0-9]+, %k[0-9]+\{%k[0-9]+\}
+** kortestb %k[0-9]+, %k[0-9]+
+** je \.L[0-9]+
+** ...
+*/
+void f_v4di (long long d, long long e)
+{
+ for (int i = 0; i < N; i++)
+ {
+ b_long[i] += a_long[i];
+ if (c_long[i] == e) continue;
+ if (a_long[i] != d) break;
+ }
+}
+
+/*
+** f_v4df:
+** ...
+** vcmppd \$4, %ymm[0-9]+, %ymm[0-9]+, %k[0-9]+
+** vcmppd \$4, %ymm[0-9]+, %ymm[0-9]+, %k[0-9]+\{%k[0-9]+\}
+** kortestb %k[0-9]+, %k[0-9]+
+** je \.L[0-9]+
+** ...
+*/
+void f_v4df (double d, double e)
+{
+ for (int i = 0; i < N; i++)
+ {
+ b_double[i] += a_double[i];
+ if (c_double[i] == e) continue;
+ if (a_double[i] != d) break;
+ }
+}
diff --git a/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-all-run.c
b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-all-run.c
new file mode 100644
index 00000000000..7c459e995da
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-all-run.c
@@ -0,0 +1,72 @@
+/* { dg-do run { target { avx512vl && avx512dq } } } */
+/* { dg-options "-Ofast -mavx512f -mavx512vl -mavx512dq
-mprefer-vector-width=128" } */
+
+#define AVX512VL
+#define AVX512DQ
+#define DO_TEST test_cbranch_all
+static void test_cbranch_all (void);
+#include "avx512-check.h"
+
+#define N 100
+long long a[N];
+long long b[N];
+long long c[N];
+
+__attribute__((noipa))
+void f (long long d, long long e)
+{
+ for (int i = 0; i < N; i++)
+ {
+ b[i] += a[i];
+ if (c[i] == e) continue;
+ if (a[i] != d) break;
+ }
+}
+
+static void reset (void)
+{
+ for (int i = 0; i < N; i++) { a[i] = 0; b[i] = 0; c[i] = 0; }
+}
+
+#define CHECK(_i, _v) do { if (b[_i] != (_v)) __builtin_abort (); } while (0)
+#define CHECK_RANGE_EQ(_lo, _hi, _v) \
+ do { \
+ for (int i = _lo; i < _hi; i++) \
+ if (b[i] != (_v)) __builtin_abort (); \
+ } while (0)
+
+static void
+test_cbranch_all (void)
+{
+ /* No break — all a[i] == d. Whole loop runs; b[i] = a[i]. */
+ reset ();
+ for (int i = 0; i < N; i++) { a[i] = 7; c[i] = 0; }
+ f (7 /*d*/, 99 /*e*/);
+ CHECK_RANGE_EQ (0, N, 7);
+
+ /* Break at index 50: a[i]==d for i<50, a[50]!=d AND c[50]!=e. */
+ reset ();
+ for (int i = 0; i < N; i++) { a[i] = 7; c[i] = 0; }
+ a[50] = 99;
+ f (7, 5);
+ CHECK_RANGE_EQ (0, 50, 7); CHECK (50, 99);
+ CHECK_RANGE_EQ (51, N, 0);
+
+ /* "continue" gates the break: c[50]==e at the same i where a[i]!=d
+ should *not* break, loop keeps going. */
+ reset ();
+ for (int i = 0; i < N; i++) { a[i] = 7; c[i] = 0; }
+ a[50] = 99; c[50] = 5; /* c==e -> continue, skip break check */
+ f (7, 5);
+ CHECK (50, 99);
+ for (int i = 0; i < N; i++)
+ if (i != 50 && b[i] != 7) __builtin_abort ();
+
+ /* Break at index 0. */
+ reset ();
+ a[0] = 99; c[0] = 0; /* a!=d AND c!=e at i=0 */
+ for (int i = 1; i < N; i++) { a[i] = 7; c[i] = 0; }
+ f (7, 5);
+ CHECK (0, 99);
+ CHECK_RANGE_EQ (1, N, 0);
+}
diff --git a/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-any-128.c
b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-any-128.c
new file mode 100644
index 00000000000..b8e321306a0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-any-128.c
@@ -0,0 +1,86 @@
+/* Test cond_vec_cbranch_any<mode> for the 128-bit V_ANY_CBRANCH modes
+ (V4SI, V2DI, V4SF, V2DF). */
+/* { dg-do compile } */
+/* { dg-options "-Ofast -mavx512f -mavx512vl -mavx512dq
-mprefer-vector-width=128 -fno-schedule-insns -fno-reorder-blocks
-fno-schedule-insns2" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+int a_int[5], b_int[5], c_int[5];
+long long a_long[3], b_long[3], c_long[3];
+float a_float[5], b_float[5], c_float[5];
+double a_double[3], b_double[3], c_double[3];
+
+/*
+** f_v4si:
+** ...
+** vpcmpd \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+
+** vpcmpd \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+\{%k[0-9]+\}
+** kortestb %k[0-9]+, %k[0-9]+
+** jne \.L[0-9]+
+** ...
+*/
+void f_v4si (int d, int e)
+{
+ for (int i = 0; i < 5; i++)
+ {
+ b_int[i] += a_int[i];
+ if (c_int[i] == e) continue;
+ if (a_int[i] != d) break;
+ }
+}
+
+/*
+** f_v2di:
+** ...
+** vpcmpq \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+
+** vpcmpq \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+\{%k[0-9]+\}
+** kortestb %k[0-9]+, %k[0-9]+
+** jne \.L[0-9]+
+** ...
+*/
+void f_v2di (long long d, long long e)
+{
+ for (int i = 0; i < 3; i++)
+ {
+ b_long[i] += a_long[i];
+ if (c_long[i] == e) continue;
+ if (a_long[i] != d) break;
+ }
+}
+
+/*
+** f_v4sf:
+** ...
+** vcmpps \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+
+** vcmpps \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+\{%k[0-9]+\}
+** kortestb %k[0-9]+, %k[0-9]+
+** jne \.L[0-9]+
+** ...
+*/
+void f_v4sf (float d, float e)
+{
+ for (int i = 0; i < 5; i++)
+ {
+ b_float[i] += a_float[i];
+ if (c_float[i] == e) continue;
+ if (a_float[i] != d) break;
+ }
+}
+
+/*
+** f_v2df:
+** ...
+** vcmppd \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+
+** vcmppd \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+\{%k[0-9]+\}
+** kortestb %k[0-9]+, %k[0-9]+
+** jne \.L[0-9]+
+** ...
+*/
+void f_v2df (double d, double e)
+{
+ for (int i = 0; i < 3; i++)
+ {
+ b_double[i] += a_double[i];
+ if (c_double[i] == e) continue;
+ if (a_double[i] != d) break;
+ }
+}
diff --git a/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-any-256.c
b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-any-256.c
new file mode 100644
index 00000000000..7e88834f7c5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-any-256.c
@@ -0,0 +1,46 @@
+/* Test cond_vec_cbranch_any<mode> for the 256-bit V_ANY_CBRANCH modes
+ (V4DI, V4DF). */
+/* { dg-do compile } */
+/* { dg-options "-Ofast -mavx512f -mavx512vl -mavx512dq
-mprefer-vector-width=256 -fno-schedule-insns -fno-reorder-blocks
-fno-schedule-insns2" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+long long a_long[5], b_long[5], c_long[5];
+double a_double[5], b_double[5], c_double[5];
+
+/*
+** f_v4di:
+** ...
+** vpcmpq \$4, %ymm[0-9]+, %ymm[0-9]+, %k[0-9]+
+** vpcmpq \$4, %ymm[0-9]+, %ymm[0-9]+, %k[0-9]+\{%k[0-9]+\}
+** kortestb %k[0-9]+, %k[0-9]+
+** jne \.L[0-9]+
+** ...
+*/
+void f_v4di (long long d, long long e)
+{
+ for (int i = 0; i < 5; i++)
+ {
+ b_long[i] += a_long[i];
+ if (c_long[i] == e) continue;
+ if (a_long[i] != d) break;
+ }
+}
+
+/*
+** f_v4df:
+** ...
+** vcmppd \$4, %ymm[0-9]+, %ymm[0-9]+, %k[0-9]+
+** vcmppd \$4, %ymm[0-9]+, %ymm[0-9]+, %k[0-9]+\{%k[0-9]+\}
+** kortestb %k[0-9]+, %k[0-9]+
+** jne \.L[0-9]+
+** ...
+*/
+void f_v4df (double d, double e)
+{
+ for (int i = 0; i < 5; i++)
+ {
+ b_double[i] += a_double[i];
+ if (c_double[i] == e) continue;
+ if (a_double[i] != d) break;
+ }
+}
diff --git a/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-any-run.c
b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-any-run.c
new file mode 100644
index 00000000000..323b6d1e545
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-any-run.c
@@ -0,0 +1,67 @@
+/* { dg-do run { target { avx512vl && avx512dq } } } */
+/* { dg-options "-Ofast -mavx512f -mavx512vl -mavx512dq
-mprefer-vector-width=128" } */
+
+#define AVX512VL
+#define AVX512DQ
+#define DO_TEST test_cbranch_any
+static void test_cbranch_any (void);
+#include "avx512-check.h"
+
+#define N 3 /* V2DI nelem (2) + 1 */
+long long a[N];
+long long b[N];
+long long c[N];
+
+__attribute__((noipa))
+void f (long long d, long long e)
+{
+ for (int i = 0; i < N; i++)
+ {
+ b[i] += a[i];
+ if (c[i] == e) continue;
+ if (a[i] != d) break;
+ }
+}
+
+static void reset (void)
+{
+ for (int i = 0; i < N; i++) { a[i] = 0; b[i] = 0; c[i] = 0; }
+}
+
+#define CHECK(_i, _v) do { if (b[_i] != (_v)) __builtin_abort (); } while (0)
+
+static void
+test_cbranch_any (void)
+{
+ /* a[0] != d AND c[0] != e → break immediately, only b[0] updated. */
+ reset ();
+ a[0] = 5; c[0] = 5;
+ a[1] = 1; c[1] = 7;
+ a[2] = 1; c[2] = 7;
+ f (1 /*d*/, 7 /*e*/);
+ CHECK (0, 5); CHECK (1, 0); CHECK (2, 0);
+
+ /* All a[i]==d → continue cond never breaks. All updated. */
+ reset ();
+ a[0] = 1; c[0] = 7;
+ a[1] = 1; c[1] = 7;
+ a[2] = 1; c[2] = 7;
+ f (1, 7);
+ CHECK (0, 1); CHECK (1, 1); CHECK (2, 1);
+
+ /* All c[i]==e → outer cond hits "continue" each iter, no break. */
+ reset ();
+ a[0] = 5; c[0] = 7; /* c==e (e=7) — continue */
+ a[1] = 6; c[1] = 7;
+ a[2] = 8; c[2] = 7;
+ f (1, 7);
+ CHECK (0, 5); CHECK (1, 6); CHECK (2, 8);
+
+ /* break at index 1. */
+ reset ();
+ a[0] = 1; c[0] = 0; /* a==d, no break */
+ a[1] = 9; c[1] = 0; /* a!=d AND c!=e → break here */
+ a[2] = 0; c[2] = 0;
+ f (1, 7);
+ CHECK (0, 1); CHECK (1, 9); CHECK (2, 0);
+}
--
2.31.1
