https://github.com/MacDue updated
https://github.com/llvm/llvm-project/pull/121763
>From 42b096396f36adc6f1ad34de263a80dd7e4e0960 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxw...@arm.com>
Date: Mon, 6 Jan 2025 11:49:48 +0000
Subject: [PATCH 1/3] [clang] Lower non-builtin sincos[f|l] calls to
llvm.sincos.* when -fno-math-errno is set
This will allow vectorizing these calls (after a few more patches). This
should not change the codegen for targets that enable the use of AA
during the codegen (in `TargetSubtargetInfo::useAA()`). This includes
targets such as AArch64. This notably does not include x86 but can be
worked around by passing `-mllvm -combiner-global-alias-analysis=true`
to clang.
---
clang/lib/CodeGen/CGBuiltin.cpp | 3 +++
clang/test/CodeGen/AArch64/sincos.c | 24 +++++++++++++++++++-----
2 files changed, 22 insertions(+), 5 deletions(-)
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index d57f491a20c8e..d3f41563aed9d 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -3377,6 +3377,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl
GD, unsigned BuiltinID,
return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(
*this, E, Intrinsic::sinh,
Intrinsic::experimental_constrained_sinh));
+ case Builtin::BIsincos:
+ case Builtin::BIsincosf:
+ case Builtin::BIsincosl:
case Builtin::BI__builtin_sincos:
case Builtin::BI__builtin_sincosf:
case Builtin::BI__builtin_sincosf16:
diff --git a/clang/test/CodeGen/AArch64/sincos.c
b/clang/test/CodeGen/AArch64/sincos.c
index b77d98ceab486..fde277716dddd 100644
--- a/clang/test/CodeGen/AArch64/sincos.c
+++ b/clang/test/CodeGen/AArch64/sincos.c
@@ -1,5 +1,19 @@
-// RUN: %clang_cc1 -triple=aarch64-gnu-linux -emit-llvm -O1 %s -o - |
FileCheck --check-prefix=NO-MATH-ERRNO %s
-// RUN: %clang_cc1 -triple=aarch64-gnu-linux -emit-llvm -fmath-errno %s -o - |
FileCheck --check-prefix=MATH-ERRNO %s
+// RUN: %clang_cc1 -triple=aarch64-gnu-linux -emit-llvm -O1 %s -o -
-DUSE_BUILTIN | FileCheck --check-prefix=NO-MATH-ERRNO %s
+// RUN: %clang_cc1 -triple=aarch64-gnu-linux -emit-llvm -fmath-errno %s -o -
-DUSE_BUILTIN | FileCheck --check-prefix=MATH-ERRNO %s
+// RUN: %clang_cc1 -triple=aarch64-gnu-linux -emit-llvm -O1 %s -o -
-DUSE_C_DECL | FileCheck --check-prefix=NO-MATH-ERRNO %s
+// RUN: %clang_cc1 -triple=aarch64-gnu-linux -emit-llvm -fmath-errno %s -o -
-DUSE_C_DECL | FileCheck --check-prefix=MATH-ERRNO %s
+
+#if defined(USE_BUILTIN)
+ #define sincos __builtin_sincos
+ #define sincosf __builtin_sincosf
+ #define sincosl __builtin_sincosl
+#elif defined(USE_C_DECL)
+ void sincos(double, double*, double*);
+ void sincosf(float, float*, float*);
+ void sincosl(long double, long double*, long double*);
+#else
+ #error Expected USE_BUILTIN or USE_C_DECL to be defined.
+#endif
// NO-MATH-ERRNO-LABEL: @sincos_f32
// NO-MATH-ERRNO: [[SINCOS:%.*]] = tail call { float, float }
@llvm.sincos.f32(float {{.*}})
@@ -12,7 +26,7 @@
// MATH-ERRNO: call void @sincosf(
//
void sincos_f32(float x, float* fp0, float* fp1) {
- __builtin_sincosf(x, fp0, fp1);
+ sincosf(x, fp0, fp1);
}
// NO-MATH-ERRNO-LABEL: @sincos_f64
@@ -26,7 +40,7 @@ void sincos_f32(float x, float* fp0, float* fp1) {
// MATH-ERRNO: call void @sincos(
//
void sincos_f64(double x, double* dp0, double* dp1) {
- __builtin_sincos(x, dp0, dp1);
+ sincos(x, dp0, dp1);
}
// NO-MATH-ERRNO-LABEL: @sincos_f128
@@ -40,5 +54,5 @@ void sincos_f64(double x, double* dp0, double* dp1) {
// MATH-ERRNO: call void @sincosl(
//
void sincos_f128(long double x, long double* ldp0, long double* ldp1) {
- __builtin_sincosl(x, ldp0, ldp1);
+ sincosl(x, ldp0, ldp1);
}
>From 35de429ac0e259998121d911706454377a2d6d24 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxw...@arm.com>
Date: Mon, 6 Jan 2025 17:06:01 +0000
Subject: [PATCH 2/3] Tweak tests
---
clang/test/CodeGen/AArch64/sincos.c | 62 ++++++++++++++++++++++-------
1 file changed, 47 insertions(+), 15 deletions(-)
diff --git a/clang/test/CodeGen/AArch64/sincos.c
b/clang/test/CodeGen/AArch64/sincos.c
index fde277716dddd..736c0892ed741 100644
--- a/clang/test/CodeGen/AArch64/sincos.c
+++ b/clang/test/CodeGen/AArch64/sincos.c
@@ -1,19 +1,9 @@
-// RUN: %clang_cc1 -triple=aarch64-gnu-linux -emit-llvm -O1 %s -o -
-DUSE_BUILTIN | FileCheck --check-prefix=NO-MATH-ERRNO %s
-// RUN: %clang_cc1 -triple=aarch64-gnu-linux -emit-llvm -fmath-errno %s -o -
-DUSE_BUILTIN | FileCheck --check-prefix=MATH-ERRNO %s
-// RUN: %clang_cc1 -triple=aarch64-gnu-linux -emit-llvm -O1 %s -o -
-DUSE_C_DECL | FileCheck --check-prefix=NO-MATH-ERRNO %s
-// RUN: %clang_cc1 -triple=aarch64-gnu-linux -emit-llvm -fmath-errno %s -o -
-DUSE_C_DECL | FileCheck --check-prefix=MATH-ERRNO %s
+// RUN: %clang_cc1 -triple=aarch64-gnu-linux -emit-llvm -O1 %s -o - |
FileCheck --check-prefix=NO-MATH-ERRNO %s
+// RUN: %clang_cc1 -triple=aarch64-gnu-linux -emit-llvm -fmath-errno %s -o - |
FileCheck --check-prefix=MATH-ERRNO %s
-#if defined(USE_BUILTIN)
- #define sincos __builtin_sincos
- #define sincosf __builtin_sincosf
- #define sincosl __builtin_sincosl
-#elif defined(USE_C_DECL)
- void sincos(double, double*, double*);
- void sincosf(float, float*, float*);
- void sincosl(long double, long double*, long double*);
-#else
- #error Expected USE_BUILTIN or USE_C_DECL to be defined.
-#endif
+void sincos(double, double*, double*);
+void sincosf(float, float*, float*);
+void sincosl(long double, long double*, long double*);
// NO-MATH-ERRNO-LABEL: @sincos_f32
// NO-MATH-ERRNO: [[SINCOS:%.*]] = tail call { float, float }
@llvm.sincos.f32(float {{.*}})
@@ -29,6 +19,20 @@ void sincos_f32(float x, float* fp0, float* fp1) {
sincosf(x, fp0, fp1);
}
+// NO-MATH-ERRNO-LABEL: @sincos_builtin_f32
+// NO-MATH-ERRNO: [[SINCOS:%.*]] = tail call { float, float }
@llvm.sincos.f32(float {{.*}})
+// NO-MATH-ERRNO-NEXT: [[SIN:%.*]] = extractvalue { float, float }
[[SINCOS]], 0
+// NO-MATH-ERRNO-NEXT: [[COS:%.*]] = extractvalue { float, float }
[[SINCOS]], 1
+// NO-MATH-ERRNO-NEXT: store float [[SIN]], ptr {{.*}}, align 4,
!alias.scope [[SINCOS_ALIAS_SCOPE:![0-9]+]]
+// NO-MATH-ERRNO-NEXT: store float [[COS]], ptr {{.*}}, align 4, !noalias
[[SINCOS_ALIAS_SCOPE]]
+//
+// MATH-ERRNO-LABEL: @sincos_builtin_f32
+// MATH-ERRNO: call void @sincosf(
+//
+void sincos_builtin_f32(float x, float* fp0, float* fp1) {
+ __builtin_sincosf(x, fp0, fp1);
+}
+
// NO-MATH-ERRNO-LABEL: @sincos_f64
// NO-MATH-ERRNO: [[SINCOS:%.*]] = tail call { double, double }
@llvm.sincos.f64(double {{.*}})
// NO-MATH-ERRNO-NEXT: [[SIN:%.*]] = extractvalue { double, double }
[[SINCOS]], 0
@@ -43,6 +47,20 @@ void sincos_f64(double x, double* dp0, double* dp1) {
sincos(x, dp0, dp1);
}
+// NO-MATH-ERRNO-LABEL: @sincos_builtin_f64
+// NO-MATH-ERRNO: [[SINCOS:%.*]] = tail call { double, double }
@llvm.sincos.f64(double {{.*}})
+// NO-MATH-ERRNO-NEXT: [[SIN:%.*]] = extractvalue { double, double }
[[SINCOS]], 0
+// NO-MATH-ERRNO-NEXT: [[COS:%.*]] = extractvalue { double, double }
[[SINCOS]], 1
+// NO-MATH-ERRNO-NEXT: store double [[SIN]], ptr {{.*}}, align 8,
!alias.scope [[SINCOS_ALIAS_SCOPE:![0-9]+]]
+// NO-MATH-ERRNO-NEXT: store double [[COS]], ptr {{.*}}, align 8, !noalias
[[SINCOS_ALIAS_SCOPE]]
+//
+// MATH-ERRNO-LABEL: @sincos_builtin_f64
+// MATH-ERRNO: call void @sincos(
+//
+void sincos_builtin_f64(double x, double* dp0, double* dp1) {
+ __builtin_sincos(x, dp0, dp1);
+}
+
// NO-MATH-ERRNO-LABEL: @sincos_f128
// NO-MATH-ERRNO: [[SINCOS:%.*]] = tail call { fp128, fp128 }
@llvm.sincos.f128(fp128 {{.*}})
// NO-MATH-ERRNO-NEXT: [[SIN:%.*]] = extractvalue { fp128, fp128 }
[[SINCOS]], 0
@@ -56,3 +74,17 @@ void sincos_f64(double x, double* dp0, double* dp1) {
void sincos_f128(long double x, long double* ldp0, long double* ldp1) {
sincosl(x, ldp0, ldp1);
}
+
+// NO-MATH-ERRNO-LABEL: @sincos_builtin_f128
+// NO-MATH-ERRNO: [[SINCOS:%.*]] = tail call { fp128, fp128 }
@llvm.sincos.f128(fp128 {{.*}})
+// NO-MATH-ERRNO-NEXT: [[SIN:%.*]] = extractvalue { fp128, fp128 }
[[SINCOS]], 0
+// NO-MATH-ERRNO-NEXT: [[COS:%.*]] = extractvalue { fp128, fp128 }
[[SINCOS]], 1
+// NO-MATH-ERRNO-NEXT: store fp128 [[SIN]], ptr {{.*}}, align 16,
!alias.scope [[SINCOS_ALIAS_SCOPE:![0-9]+]]
+// NO-MATH-ERRNO-NEXT: store fp128 [[COS]], ptr {{.*}}, align 16, !noalias
[[SINCOS_ALIAS_SCOPE]]
+//
+// MATH-ERRNO-LABEL: @sincos_builtin_f128
+// MATH-ERRNO: call void @sincosl(
+//
+void sincos_builtin_f128(long double x, long double* ldp0, long double* ldp1) {
+ __builtin_sincosl(x, ldp0, ldp1);
+}
>From aafed97efb71225893965378d2125085924a2293 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxw...@arm.com>
Date: Tue, 18 Feb 2025 14:47:17 +0000
Subject: [PATCH 3/3] Add hook to limit intrinsic use
---
clang/lib/CodeGen/CGBuiltin.cpp | 31 +++++++++++++++++----------
clang/lib/CodeGen/TargetInfo.h | 17 +++++++++++++++
clang/lib/CodeGen/Targets/AArch64.cpp | 4 ++++
clang/test/CodeGen/math-libcalls.c | 11 ++++++++++
4 files changed, 52 insertions(+), 11 deletions(-)
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index d3f41563aed9d..45ea335eab7d0 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2937,12 +2937,21 @@ RValue CodeGenFunction::EmitBuiltinExpr(const
GlobalDecl GD, unsigned BuiltinID,
&getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
BuiltinID = mutateLongDoubleBuiltin(BuiltinID);
- // If the builtin has been declared explicitly with an assembler label,
- // disable the specialized emitting below. Ideally we should communicate the
- // rename in IR, or at least avoid generating the intrinsic calls that are
- // likely to get lowered to the renamed library functions.
- const unsigned BuiltinIDIfNoAsmLabel =
- FD->hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
+ const unsigned BuiltinIDIfEmitIntrinsics = [&] {
+ // If the builtin has been declared explicitly with an assembler label,
+ // disable the specialized emitting below. Ideally we should communicate
the
+ // rename in IR, or at least avoid generating the intrinsic calls that are
+ // likely to get lowered to the renamed library functions.
+ if (FD->hasAttr<AsmLabelAttr>())
+ return 0U;
+
+ // If the target requests not to use intrinsics for a builtin, disable the
+ // specialized emitting below.
+ if (!getTargetHooks().shouldUseIntrinsicsForBuiltin(BuiltinID))
+ return 0U;
+
+ return BuiltinID;
+ }();
std::optional<bool> ErrnoOverriden;
// ErrnoOverriden is true if math-errno is overriden via the
@@ -3035,7 +3044,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl
GD, unsigned BuiltinID,
ConstWithoutErrnoOrExceptions && ErrnoOverridenToFalseWithOpt;
}
if (GenerateIntrinsics) {
- switch (BuiltinIDIfNoAsmLabel) {
+ switch (BuiltinIDIfEmitIntrinsics) {
case Builtin::BIacos:
case Builtin::BIacosf:
case Builtin::BIacosl:
@@ -3515,7 +3524,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl
GD, unsigned BuiltinID,
}
};
- switch (BuiltinIDIfNoAsmLabel) {
+ switch (BuiltinIDIfEmitIntrinsics) {
default: break;
case Builtin::BI__builtin___CFStringMakeConstantString:
case Builtin::BI__builtin___NSStringMakeConstantString:
@@ -3645,7 +3654,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl
GD, unsigned BuiltinID,
case Builtin::BI__builtin_ctzl:
case Builtin::BI__builtin_ctzll:
case Builtin::BI__builtin_ctzg: {
- bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_ctzg &&
+ bool HasFallback = BuiltinIDIfEmitIntrinsics == Builtin::BI__builtin_ctzg
&&
E->getNumArgs() > 1;
Value *ArgValue =
@@ -3677,7 +3686,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl
GD, unsigned BuiltinID,
case Builtin::BI__builtin_clzl:
case Builtin::BI__builtin_clzll:
case Builtin::BI__builtin_clzg: {
- bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_clzg &&
+ bool HasFallback = BuiltinIDIfEmitIntrinsics == Builtin::BI__builtin_clzg
&&
E->getNumArgs() > 1;
Value *ArgValue =
@@ -4347,7 +4356,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl
GD, unsigned BuiltinID,
Ty = VecTy->getElementType();
bool IsSigned = Ty->isSignedIntegerType();
unsigned Opc;
- if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
+ if (BuiltinIDIfEmitIntrinsics == Builtin::BI__builtin_elementwise_add_sat)
Opc = IsSigned ? llvm::Intrinsic::sadd_sat : llvm::Intrinsic::uadd_sat;
else
Opc = IsSigned ? llvm::Intrinsic::ssub_sat : llvm::Intrinsic::usub_sat;
diff --git a/clang/lib/CodeGen/TargetInfo.h b/clang/lib/CodeGen/TargetInfo.h
index 4a66683a3b91f..280e51401f66c 100644
--- a/clang/lib/CodeGen/TargetInfo.h
+++ b/clang/lib/CodeGen/TargetInfo.h
@@ -454,6 +454,23 @@ class TargetCodeGenInfo {
initBranchProtectionFnAttributes(const TargetInfo::BranchProtectionInfo &BPI,
llvm::AttrBuilder &FuncAttrs);
+ /// Returns true if an intrinsic should be emitted for a specific builtin. By
+ /// default, this disables emitting intrinsics for (non-prefixed) sincos
+ /// builtins, as on targets that do not set
llvm::TargetSubtargetInfo::useAA()
+ /// the intrinsics can worsen codegen.
+ /// TODO: Remove once the intrinsic lowering works well for all targets.
+ virtual bool shouldUseIntrinsicsForBuiltin(unsigned BuiltinID) const {
+ switch (BuiltinID) {
+ case Builtin::BIsincos:
+ case Builtin::BIsincosf:
+ case Builtin::BIsincosl:
+ return false;
+ default:
+ break;
+ }
+ return true;
+ }
+
protected:
static std::string qualifyWindowsLibrary(StringRef Lib);
diff --git a/clang/lib/CodeGen/Targets/AArch64.cpp
b/clang/lib/CodeGen/Targets/AArch64.cpp
index d6e0e720a0941..8ea0b7c5bac46 100644
--- a/clang/lib/CodeGen/Targets/AArch64.cpp
+++ b/clang/lib/CodeGen/Targets/AArch64.cpp
@@ -181,6 +181,10 @@ class AArch64TargetCodeGenInfo : public TargetCodeGenInfo {
bool wouldInliningViolateFunctionCallABI(
const FunctionDecl *Caller, const FunctionDecl *Callee) const override;
+ bool shouldUseIntrinsicsForBuiltin(unsigned BuiltinID) const override {
+ return true;
+ }
+
private:
// Diagnose calls between functions with incompatible Streaming SVE
// attributes.
diff --git a/clang/test/CodeGen/math-libcalls.c
b/clang/test/CodeGen/math-libcalls.c
index bcc61c8f046b4..d9ef7f444d8c3 100644
--- a/clang/test/CodeGen/math-libcalls.c
+++ b/clang/test/CodeGen/math-libcalls.c
@@ -660,6 +660,17 @@ void foo(double *d, float f, float *fp, long double *l,
int *i, const char *c) {
// HAS_MAYTRAP: declare float @llvm.experimental.constrained.sinh.f32(
// HAS_MAYTRAP: declare x86_fp80 @llvm.experimental.constrained.sinh.f80(
+sincos(f, d, d); sincosf(f, fp, fp); sincosl(f, l, l);
+
+// NO__ERRNO: declare void @sincos(double noundef, ptr noundef, ptr noundef)
[[NOT_READNONE]]
+// NO__ERRNO: declare void @sincosf(float noundef, ptr noundef, ptr noundef)
[[NOT_READNONE]]
+// NO__ERRNO: declare void @sincosl(x86_fp80 noundef, ptr noundef, ptr
noundef) [[NOT_READNONE]]
+// HAS__ERRNO: declare void @sincos(double noundef, ptr noundef, ptr noundef)
[[NOT_READNONE]]
+// HAS__ERRNO: declare void @sincosf(float noundef, ptr noundef, ptr noundef)
[[NOT_READNONE]]
+// HAS__ERRNO: declare void @sincosl(x86_fp80 noundef, ptr noundef, ptr
noundef) [[NOT_READNONE]]
+// HAS_MAYTRAP: declare void @sincos(double noundef, ptr noundef, ptr noundef)
[[NOT_READNONE]]
+// HAS_MAYTRAP: declare void @sincosf(float noundef, ptr noundef, ptr noundef)
[[NOT_READNONE]]
+// HAS_MAYTRAP: declare void @sincosl(x86_fp80 noundef, ptr noundef, ptr
noundef) [[NOT_READNONE]]
sqrt(f); sqrtf(f); sqrtl(f);
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
