From: Richard Ball <[email protected]>
This patch adds the following advsimd intrinsics:
*vfdot_f32_f16
*vfdotq_f32_f16
*vfdot_lane_f32_f16
*vfdot_laneq_f32_f16
---
gcc/config/aarch64/aarch64-c.cc | 3 ++
gcc/config/aarch64/aarch64-simd-builtins.def | 5 ++
gcc/config/aarch64/aarch64-simd.md | 32 ++++++++++++
gcc/config/aarch64/aarch64.h | 2 +
gcc/config/aarch64/arm_neon.h | 34 +++++++++++++
gcc/doc/invoke.texi | 2 +
.../aarch64/advsimd-intrinsics/fdot-1.c | 48 ++++++++++++++++++
.../aarch64/advsimd-intrinsics/fdot-2.c | 50 +++++++++++++++++++
.../aarch64/advsimd-intrinsics/fdot-3.c | 17 +++++++
gcc/testsuite/lib/target-supports.exp | 41 +++++++++++++++
10 files changed, 234 insertions(+)
create mode 100644 gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-1.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-2.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-3.c
diff --git a/gcc/config/aarch64/aarch64-c.cc b/gcc/config/aarch64/aarch64-c.cc
index 21ced615c10..8047f51f92b 100644
--- a/gcc/config/aarch64/aarch64-c.cc
+++ b/gcc/config/aarch64/aarch64-c.cc
@@ -324,6 +324,9 @@ aarch64_update_cpp_builtins (cpp_reader *pfile)
builtin_define ("__ARM_PREFETCH_RANGE");
aarch64_def_or_undef (AARCH64_HAVE_ISA (SME_TMOP),
"__ARM_FEATURE_SME_TMOP", pfile);
+ aarch64_def_or_undef (AARCH64_HAVE_ISA (SSVE_AES), "__ARM_FEATURE_SSVE_AES",
+ pfile);
+ aarch64_def_or_undef (TARGET_F16F32DOT, "__ARM_FEATURE_F16F32DOT", pfile);
// Function multi-versioning defines
aarch64_def_or_undef (targetm.has_ifunc_p (),
diff --git a/gcc/config/aarch64/aarch64-simd-builtins.def
b/gcc/config/aarch64/aarch64-simd-builtins.def
index 1b09191a0c0..5b5c17a7f3a 100644
--- a/gcc/config/aarch64/aarch64-simd-builtins.def
+++ b/gcc/config/aarch64/aarch64-simd-builtins.def
@@ -957,6 +957,11 @@
VAR2 (QUADOP_LANE_PAIR, bfdot_lane, 0, QUIET, v2sf, v4sf)
VAR2 (QUADOP_LANE_PAIR, bfdot_laneq, 0, QUIET, v2sf, v4sf)
+ /* Implemented by aarch64_simdfdot{_lane}{q}<mode>. */
+ VAR2 (TERNOP, simdfdot, 0, QUIET, v2sf, v4sf)
+ VAR1 (QUADOP_LANE_PAIR, simdfdot_lane, 0, QUIET, v2sf)
+ VAR1 (QUADOP_LANE_PAIR, simdfdot_laneq, 0, QUIET, v4sf)
+
/* Implemented by aarch64_bfmmlaqv4sf */
VAR1 (TERNOP, bfmmlaq, 0, QUIET, v4sf)
diff --git a/gcc/config/aarch64/aarch64-simd.md
b/gcc/config/aarch64/aarch64-simd.md
index 843ad6cb076..8a5059f4ec1 100644
--- a/gcc/config/aarch64/aarch64-simd.md
+++ b/gcc/config/aarch64/aarch64-simd.md
@@ -10645,6 +10645,38 @@ (define_insn
"aarch64_bfdot_lane<VBF:isquadop><VDQSF:mode>"
[(set_attr "type" "neon_dot<VDQSF:q>")]
)
+(define_insn "aarch64_simdfdot<mode>"
+ [(set (match_operand:VDQSF 0 "register_operand" "=w")
+ (plus:VDQSF
+ (unspec:VDQSF
+ [(match_operand:<VFMLA_W> 2 "register_operand" "w")
+ (match_operand:<VFMLA_W> 3 "register_operand" "w")]
+ UNSPEC_FDOT)
+ (match_operand:VDQSF 1 "register_operand" "0")))]
+ "TARGET_F16F32DOT"
+ "fdot\t%0.<Vtype>, %2.<Vbfdottype>, %3.<Vbfdottype>"
+ [(set_attr "type" "neon_dot<q>")]
+)
+
+(define_insn "aarch64_simdfdot_lane<f16quad><mode>"
+ [(set (match_operand:VDQSF 0 "register_operand" "=w")
+ (plus:VDQSF
+ (unspec:VDQSF
+ [(match_operand:<VFMLA_W> 2 "register_operand" "w")
+ (match_operand:<VFMLA_W> 3 "register_operand" "w")
+ (match_operand:SI 4 "const_int_operand" "n")]
+ UNSPEC_FDOT)
+ (match_operand:VDQSF 1 "register_operand" "0")))]
+ "TARGET_F16F32DOT"
+{
+ int nunits = GET_MODE_NUNITS (<MODE>mode).to_constant ();
+ int lane = INTVAL (operands[4]);
+ operands[4] = gen_int_mode (ENDIAN_LANE_N (nunits / 2, lane), SImode);
+ return "fdot\t%0.<Vtype>, %2.<Vbfdottype>, %3.2h[%4]";
+}
+ [(set_attr "type" "neon_dot<q>")]
+)
+
;; bfmmla
(define_insn "aarch64_bfmmlaqv4sf"
[(set (match_operand:V4SF 0 "register_operand" "=w")
diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h
index fbdae6ec183..1bf6765a74a 100644
--- a/gcc/config/aarch64/aarch64.h
+++ b/gcc/config/aarch64/aarch64.h
@@ -436,6 +436,8 @@ constexpr auto AARCH64_FL_DEFAULT_ISA_MODE ATTRIBUTE_UNUSED
#define TARGET_F8F16MM (AARCH64_HAVE_ISA (F8F16MM))
/* SVE_F16F32MM instructions, enabled through +sve-f16f32mm. */
#define TARGET_SVE_F16F32MM (AARCH64_HAVE_ISA (SVE_F16F32MM))
+/* F16F32DOT instructions enabled through +f16f32dot. */
+#define TARGET_F16F32DOT (AARCH64_HAVE_ISA (F16F32DOT))
/* Make sure this is always defined so we don't have to check for ifdefs
but rather use normal ifs. */
diff --git a/gcc/config/aarch64/arm_neon.h b/gcc/config/aarch64/arm_neon.h
index d7505a16edb..1cd905d6b7d 100644
--- a/gcc/config/aarch64/arm_neon.h
+++ b/gcc/config/aarch64/arm_neon.h
@@ -28559,6 +28559,40 @@ vst4q_lane_bf16 (bfloat16_t *__ptr, bfloat16x8x4_t
__val, const int __lane)
#pragma GCC pop_options
+#pragma GCC push_options
+#pragma GCC target ("+nothing+f16f32dot")
+
+__extension__ extern __inline float32x2_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vfdot_f32_f16 (float32x2_t __r, float16x4_t __a, float16x4_t __b)
+{
+ return __builtin_aarch64_simdfdotv2sf (__r, __a, __b);
+}
+
+__extension__ extern __inline float32x4_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vfdotq_f32_f16 (float32x4_t __r, float16x8_t __a, float16x8_t __b)
+{
+ return __builtin_aarch64_simdfdotv4sf (__r, __a, __b);
+}
+
+__extension__ extern __inline float32x2_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vfdot_lane_f32_f16 (float32x2_t __r, float16x4_t __a, float16x4_t __b,
+ const int __index)
+{
+ return __builtin_aarch64_simdfdot_lanev2sf (__r, __a, __b, __index);
+}
+
+__extension__ extern __inline float32x4_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vfdot_laneq_f32_f16 (float32x4_t __r, float16x8_t __a, float16x8_t __b,
+ const int __index)
+{
+ return __builtin_aarch64_simdfdot_laneqv4sf (__r, __a, __b, __index);
+}
+
+#pragma GCC pop_options
/* AdvSIMD 8-bit Integer Matrix Multiply (I8MM) intrinsics. */
#pragma GCC push_options
diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index d6cf1831ceb..3099ca9228d 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -22051,6 +22051,8 @@ extension.
@item ssve-fp8dot2
Enable the fp8 (8-bit floating point) to half-precision 2-way dot product
extension in streaming mode.
+@item f16f32dot
+Enable the Half-precision to single-precision dot product extension.
@item faminmax
Enable the Floating Point Absolute Maximum/Minimum extension.
@item lut
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-1.c
b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-1.c
new file mode 100644
index 00000000000..d8e4a76c9c6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-1.c
@@ -0,0 +1,48 @@
+/* { dg-do assemble { target { aarch64*-*-* } } } */
+/* { dg-require-effective-target aarch64_v8_2a_f16f32dot_neon_ok } */
+/* { dg-add-options aarch64_v8_2a_f16f32dot_neon } */
+/* { dg-additional-options "-save-temps" } */
+/* { dg-final { check-function-bodies "**" "" {-O[^0]} } } */
+/* { dg-skip-if "" { *-*-* } { "-fno-fat-lto-objects" } } */
+
+#include <arm_neon.h>
+
+/*
+**ufoo:
+** fdot v0.2s, (v1.4h, v2.4h|v2.4h, v1.4h)
+** ret
+*/
+float32x2_t ufoo(float32x2_t r, float16x4_t x, float16x4_t y)
+{
+ return vfdot_f32_f16 (r, x, y);
+}
+
+/*
+**ufooq:
+** fdot v0.4s, (v1.8h, v2.8h|v2.8h, v1.8h)
+** ret
+*/
+float32x4_t ufooq(float32x4_t r, float16x8_t x, float16x8_t y)
+{
+ return vfdotq_f32_f16 (r, x, y);
+}
+
+/*
+**ufoo_lane:
+** fdot v0.2s, v1.4h, v2.2h\[0\]
+** ret
+*/
+float32x2_t ufoo_lane(float32x2_t r, float16x4_t x, float16x4_t y)
+{
+ return vfdot_lane_f32_f16 (r, x, y, 0);
+}
+
+/*
+**ufooq_laneq:
+** fdot v0.4s, v1.8h, v2.2h\[2\]
+** ret
+*/
+float32x4_t ufooq_laneq(float32x4_t r, float16x8_t x, float16x8_t y)
+{
+ return vfdot_laneq_f32_f16 (r, x, y, 2);
+}
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-2.c
b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-2.c
new file mode 100644
index 00000000000..b2fc0ce807e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-2.c
@@ -0,0 +1,50 @@
+/* { dg-do assemble { target { aarch64*-*-* } } } */
+/* { dg-require-effective-target stdint_types_mbig_endian } */
+/* { dg-require-effective-target aarch64_v8_2a_f16f32dot_neon_ok } */
+/* { dg-add-options aarch64_v8_2a_f16f32dot_neon } */
+/* { dg-additional-options "-mbig-endian --save-temps" } */
+/* { dg-final { check-function-bodies "**" "" {-O[^0]} } } */
+/* { dg-skip-if "" { *-*-* } { "-fno-fat-lto-objects" } } */
+
+
+#include <arm_neon.h>
+
+/*
+**ufoo:
+** fdot v0.2s, (v1.4h, v2.4h|v2.4h, v1.4h)
+** ret
+*/
+float32x2_t ufoo(float32x2_t r, float16x4_t x, float16x4_t y)
+{
+ return vfdot_f32_f16 (r, x, y);
+}
+
+/*
+**ufooq:
+** fdot v0.4s, (v1.8h, v2.8h|v2.8h, v1.8h)
+** ret
+*/
+float32x4_t ufooq(float32x4_t r, float16x8_t x, float16x8_t y)
+{
+ return vfdotq_f32_f16 (r, x, y);
+}
+
+/*
+**ufoo_lane:
+** fdot v0.2s, v1.4h, v2.2h\[0\]
+** ret
+*/
+float32x2_t ufoo_lane(float32x2_t r, float16x4_t x, float16x4_t y)
+{
+ return vfdot_lane_f32_f16 (r, x, y, 0);
+}
+
+/*
+**ufooq_laneq:
+** fdot v0.4s, v1.8h, v2.2h\[2\]
+** ret
+*/
+float32x4_t ufooq_laneq(float32x4_t r, float16x8_t x, float16x8_t y)
+{
+ return vfdot_laneq_f32_f16 (r, x, y, 2);
+}
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-3.c
b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-3.c
new file mode 100644
index 00000000000..08ec38d2dfe
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-3.c
@@ -0,0 +1,17 @@
+/* { dg-do assemble { target { aarch64*-*-* } } } */
+/* { dg-skip-if "" { *-*-* } { "-fno-fat-lto-objects" } } */
+/* { dg-require-effective-target aarch64_v8_2a_f16f32dot_neon_ok } */
+/* { dg-add-options aarch64_v8_2a_f16f32dot_neon } */
+/* { dg-additional-options "--save-temps" } */
+
+#include <arm_neon.h>
+
+float32x2_t ufoo_lane(float32x2_t r, float16x4_t x, float16x4_t y)
+{
+ return vfdot_lane_f32_f16 (r, x, y, 2); /* { dg-error {lane 2 out of range 0
- 1} "" { target *-*-* } 0 } */
+}
+
+float32x4_t ufooq_laneq(float32x4_t r, float16x8_t x, float16x8_t y)
+{
+ return vfdot_laneq_f32_f16 (r, x, y, 4); /* { dg-error {lane 4 out of range
0 - 3} "" { target *-*-* } 0 } */
+}
diff --git a/gcc/testsuite/lib/target-supports.exp
b/gcc/testsuite/lib/target-supports.exp
index b5426dbc92d..ba7b5053f20 100644
--- a/gcc/testsuite/lib/target-supports.exp
+++ b/gcc/testsuite/lib/target-supports.exp
@@ -7291,6 +7291,47 @@ proc add_options_for_arm_fp16fml_neon { flags } {
return "$flags $et_arm_fp16fml_neon_flags"
}
+# Return 1 if the target supports F16F32DOT
+# instructions, 0 otherwise. This test is valid
+# for AARCH64.
+# Record the command line options needed.
+
+proc check_effective_target_aarch64_v8_2a_f16f32dot_neon_ok_nocache { } {
+ global et_aarch64_v8_2a_f16f32dot_neon_flags
+ set et_aarch64_v8_2a_f16f32dot_neon_flags ""
+
+ if { ![istarget aarch64*-*-*] } {
+ return 0;
+ }
+
+ foreach flags {"" "-mfloat-abi=softfp -mfpu=neon-fp-armv8"
"-mfloat-abi=hard -mfpu=neon-fp-armv8" } {
+ if { [check_no_compiler_messages_nocache
aarch64_v8_2a_f16f32dot_neon_ok object {
+ #include <arm_neon.h>
+ #if !defined (__ARM_FEATURE_F16F32DOT)
+ #error "__ARM_FEATURE_F16F32DOT not defined"
+ #endif
+ } "$flags -march=armv8.2-a+f16f32dot"] } {
+ set et_aarch64_v8_2a_f16f32dot_neon_flags "$flags
-march=armv8.2-a+f16f32dot"
+ return 1
+ }
+ }
+
+ return 0;
+}
+
+proc check_effective_target_aarch64_v8_2a_f16f32dot_neon_ok { } {
+ return [check_cached_effective_target aarch64_v8_2a_f16f32dot_neon_ok \
+ check_effective_target_aarch64_v8_2a_f16f32dot_neon_ok_nocache]
+}
+
+proc add_options_for_aarch64_v8_2a_f16f32dot_neon { flags } {
+ if { ! [check_effective_target_aarch64_v8_2a_f16f32dot_neon_ok] } {
+ return "$flags"
+ }
+ global et_aarch64_v8_2a_f16f32dot_neon_flags
+ return "$flags $et_aarch64_v8_2a_f16f32dot_neon_flags"
+}
+
# Return 1 if the target supports BFloat16 SIMD instructions, 0 otherwise.
# The test is valid for ARM and for AArch64.
--
2.43.0
