https://github.com/rohitaggarwal007 updated https://github.com/llvm/llvm-project/pull/213676
>From 43babe906d7e458f826e38e2122fdccd0f3e48fa Mon Sep 17 00:00:00 2001 From: Rohit Aggarwal <[email protected]> Date: Mon, 3 Aug 2026 19:00:50 +0530 Subject: [PATCH 1/2] [X86][AMDLIBM] - Add scalar AMD AOCL fast-call lowering (-fsclrlib=AMDLIBM) Under fast-math at -O3, scalar math library calls (e.g. tan, exp) are rewritten to their AMD AOCL fast-call equivalents (e.g. amd_fasttan, amd_fastexp) on X86 targets. LLVM: - New X86 MachineFunctionPass X86GenScalarAmdFastCalls, run in addMachineSSAOptimization at CodeGenOptLevel::Aggressive. - TargetLibraryInfo gains a scalar-math-library selection (ScalarLibrary enum, addScalarFunctionsFromMathLib / getScalarFunctionFromMathLib / getScalarMathLib / setScalarMathLib), populated from the new ScalarAOCLFuncs.def mapping and driven by the -scalar-library=AMDLIBM cl::opt (usable with llc and the LTO plugin). - The rewrite is gated on -scalar-library=AMDLIBM plus a function-level fast-math signal (per-operation fast-math flags are no longer available at this late machine pass in upstream codegen). Clang: - New -fsclrlib= driver/CC1 flag with a CodeGenOptions ScalarLib enum, applied in BackendUtil and forwarded to LTO as -plugin-opt=-scalar-library=. Tests: - llvm/test/CodeGen/X86/aocl-fast-scalar-calls.ll (llc path, incl. a negative no-fast-math case). - clang/test/CodeGen/X86/aocl-fast-scalar-calls.c (driver path). - aocl-fast-scalar-calls-mappings.ll: float variants (tanf, powf), a *_finite alias (__exp_finite), inverse-trig (acos, atan), and negative cases for math calls with no AOCL mapping (pow(double), cbrt). - aocl-fast-scalar-calls-i686.ll: 32-bit X86 coverage (tan, expf). - aocl-fast-scalar-calls.c: add a single-precision case (tanf) and an unmapped negative case (cbrt) to the clang driver test. --- clang/include/clang/Basic/CodeGenOptions.def | 3 + clang/include/clang/Basic/CodeGenOptions.h | 6 + clang/include/clang/Options/Options.td | 7 + clang/lib/CodeGen/BackendUtil.cpp | 15 + clang/lib/Driver/ToolChains/Clang.cpp | 11 + clang/lib/Driver/ToolChains/CommonArgs.cpp | 8 + .../test/CodeGen/X86/aocl-fast-scalar-calls.c | 40 +++ .../include/llvm/Analysis/ScalarAOCLFuncs.def | 80 ++++++ .../include/llvm/Analysis/TargetLibraryInfo.h | 33 +++ llvm/lib/Analysis/TargetLibraryInfo.cpp | 52 ++++ llvm/lib/Target/X86/CMakeLists.txt | 1 + llvm/lib/Target/X86/X86.h | 6 + .../Target/X86/X86GenScalarAmdFastCalls.cpp | 176 ++++++++++++ llvm/lib/Target/X86/X86TargetMachine.cpp | 5 + .../X86/aocl-fast-scalar-calls-i686.ll | 31 +++ .../X86/aocl-fast-scalar-calls-mappings.ll | 124 +++++++++ .../CodeGen/X86/aocl-fast-scalar-calls.ll | 63 +++++ llvm/test/CodeGen/X86/veclib-llvm.sincos.s | 258 ++++++++++++++++++ 18 files changed, 919 insertions(+) create mode 100644 clang/test/CodeGen/X86/aocl-fast-scalar-calls.c create mode 100644 llvm/include/llvm/Analysis/ScalarAOCLFuncs.def create mode 100644 llvm/lib/Target/X86/X86GenScalarAmdFastCalls.cpp create mode 100644 llvm/test/CodeGen/X86/aocl-fast-scalar-calls-i686.ll create mode 100644 llvm/test/CodeGen/X86/aocl-fast-scalar-calls-mappings.ll create mode 100644 llvm/test/CodeGen/X86/aocl-fast-scalar-calls.ll create mode 100644 llvm/test/CodeGen/X86/veclib-llvm.sincos.s diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def index 7e54e75752f39..16e353ae76f20 100644 --- a/clang/include/clang/Basic/CodeGenOptions.def +++ b/clang/include/clang/Basic/CodeGenOptions.def @@ -420,6 +420,9 @@ VALUE_CODEGENOPT(InlineMaxStackSize, 32, UINT_MAX, Benign) // Vector functions library to use. ENUM_CODEGENOPT(VecLib, VectorLibrary, 4, VectorLibrary::NoLibrary, Benign) +// Scalar math functions library to use. +ENUM_CODEGENOPT(ScalarLib, ScalarLibrary, 1, Default_Scalar_Library, Benign) + /// The default TLS model to use. ENUM_CODEGENOPT(DefaultTLSModel, TLSModel, 2, GeneralDynamicTLSModel, Benign) diff --git a/clang/include/clang/Basic/CodeGenOptions.h b/clang/include/clang/Basic/CodeGenOptions.h index c12434135a198..06cca30d9c5aa 100644 --- a/clang/include/clang/Basic/CodeGenOptions.h +++ b/clang/include/clang/Basic/CodeGenOptions.h @@ -110,6 +110,12 @@ class CodeGenOptions : public CodeGenOptionsBase { OnlyAlwaysInlining // Only run the always inlining pass. }; + /// Scalar math functions library to use with -fsclrlib=. + enum ScalarLibrary { + Default_Scalar_Library, // Use default library. + SCALAR_AMDLIBM // AMD scalar math library. + }; + enum ObjCDispatchMethodKind { Legacy = 0, NonLegacy = 1, diff --git a/clang/include/clang/Options/Options.td b/clang/include/clang/Options/Options.td index 2467ebd0abe19..8eadef417fa35 100644 --- a/clang/include/clang/Options/Options.td +++ b/clang/include/clang/Options/Options.td @@ -3965,6 +3965,13 @@ def fveclib : Joined<["-"], "fveclib=">, Group<f_Group>, NormalizedValues<["Accelerate", "LIBMVEC", "MASSV", "SVML", "SLEEF", "Darwin_libsystem_m", "ArmPL", "AMDLIBM", "NoLibrary"]>, MarshallingInfoEnum<CodeGenOpts<"VecLib">, "NoLibrary">; +def fsclrlib : Joined<["-"], "fsclrlib=">, Group<f_Group>, + Visibility<[ClangOption, CC1Option]>, + HelpText<"Use the given scalar math functions library.">, + Values<"AMDLIBM,none">, + NormalizedValuesScope<"CodeGenOptions">, + NormalizedValues<["SCALAR_AMDLIBM", "Default_Scalar_Library"]>, + MarshallingInfoEnum<CodeGenOpts<"ScalarLib">, "Default_Scalar_Library">; def fno_lax_vector_conversions : Flag<["-"], "fno-lax-vector-conversions">, Group<f_Group>, Alias<flax_vector_conversions_EQ>, AliasArgs<["none"]>; def fno_implicit_module_maps : Flag <["-"], "fno-implicit-module-maps">, Group<f_Group>; diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp index 068b1b4c262c8..aa0797adddfc3 100644 --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -142,6 +142,19 @@ static std::string getProfileGenName(const CodeGenOptions &CodeGenOpts) { return FileName; } +/// Populate the scalar math library mappings on \p TLII according to the +/// -fsclrlib= selection. +static void addScalarMathLibrary(TargetLibraryInfoImpl &TLII, + const CodeGenOptions &CodeGenOpts) { + switch (CodeGenOpts.getScalarLib()) { + case CodeGenOptions::SCALAR_AMDLIBM: + TLII.addScalarFunctionsFromMathLib(TargetLibraryInfoImpl::SCALAR_AMDLIBM); + break; + case CodeGenOptions::Default_Scalar_Library: + break; + } +} + namespace { class EmitAssemblyHelper { @@ -1002,6 +1015,7 @@ void EmitAssemblyHelper::RunOptimizationPipeline( // preset TLI. std::unique_ptr<TargetLibraryInfoImpl> TLII( llvm::driver::createTLII(TargetTriple, CodeGenOpts.getVecLib())); + addScalarMathLibrary(*TLII, CodeGenOpts); FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); }); // Register all the basic analyses with the managers. @@ -1271,6 +1285,7 @@ void EmitAssemblyHelper::RunCodegenPipelineLegacy( // Add LibraryInfo. std::unique_ptr<TargetLibraryInfoImpl> TLII( llvm::driver::createTLII(TargetTriple, CodeGenOpts.getVecLib())); + addScalarMathLibrary(*TLII, CodeGenOpts); CodeGenPasses.add(new TargetLibraryInfoWrapperPass(*TLII)); const llvm::TargetOptions &Options = TM->Options; diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index 9e90994c178dd..934b504478a8c 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -5982,6 +5982,17 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, A->render(Args, CmdArgs); } + if (Arg *A = Args.getLastArg(options::OPT_fsclrlib)) { + StringRef Name = A->getValue(); + if (Name == "AMDLIBM") { + if (Triple.getArch() != llvm::Triple::x86 && + Triple.getArch() != llvm::Triple::x86_64) + D.Diag(diag::err_drv_unsupported_opt_for_target) + << Name << Triple.getArchName(); + } + A->render(Args, CmdArgs); + } + if (Args.hasFlag(options::OPT_fmerge_all_constants, options::OPT_fno_merge_all_constants, false)) CmdArgs.push_back("-fmerge-all-constants"); diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp index 883296e43111b..2ca621ad1123a 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -1145,6 +1145,14 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args, Twine(PluginOptPrefix) + "-vector-library=" + OptVal.value())); } + // Pass scalar math library arguments to LTO. + if (Arg *ArgScalarLib = Args.getLastArg(options::OPT_fsclrlib)) { + StringRef Name = ArgScalarLib->getValue(); + if (Name == "AMDLIBM") + CmdArgs.push_back(Args.MakeArgString(Twine(PluginOptPrefix) + + "-scalar-library=" + Name)); + } + // Try to pass driver level flags relevant to LTO code generation down to // the plugin. diff --git a/clang/test/CodeGen/X86/aocl-fast-scalar-calls.c b/clang/test/CodeGen/X86/aocl-fast-scalar-calls.c new file mode 100644 index 0000000000000..50a512ebc2a5e --- /dev/null +++ b/clang/test/CodeGen/X86/aocl-fast-scalar-calls.c @@ -0,0 +1,40 @@ +// Verify that the -fsclrlib=AMDLIBM driver flag, together with fast-math at -O3, +// rewrites scalar math library calls into their AMD AOCL fast-call equivalents +// for X86, and leaves them untouched without the flag. + +// REQUIRES: x86-registered-target + +// RUN: %clang --target=x86_64-unknown-linux-gnu -O3 -ffast-math \ +// RUN: -fsclrlib=AMDLIBM -S %s -o - | FileCheck %s --check-prefix=AMD +// RUN: %clang --target=x86_64-unknown-linux-gnu -O3 -ffast-math \ +// RUN: -S %s -o - | FileCheck %s --check-prefix=STD + +double tan(double); +double exp(double); +float tanf(float); +double cbrt(double); + +double call_tan(double x) { return tan(x) + x; } +// AMD-LABEL: call_tan: +// AMD: callq{{.*}}amd_fasttan +// STD-LABEL: call_tan: +// STD: callq{{.*}}tan + +double call_exp(double x) { return exp(x) + x; } +// AMD-LABEL: call_exp: +// AMD: callq{{.*}}amd_fastexp +// STD-LABEL: call_exp: +// STD: callq{{.*}}exp + +// Single-precision variant is rewritten too. +float call_tanf(float x) { return tanf(x) + x; } +// AMD-LABEL: call_tanf: +// AMD: callq{{.*}}amd_fasttanf +// STD-LABEL: call_tanf: +// STD: callq{{.*}}tanf + +// cbrt has no AOCL mapping and must stay even with the option enabled. +double call_cbrt(double x) { return cbrt(x) + x; } +// AMD-LABEL: call_cbrt: +// AMD-NOT: amd_fast +// AMD: callq{{.*}}cbrt diff --git a/llvm/include/llvm/Analysis/ScalarAOCLFuncs.def b/llvm/include/llvm/Analysis/ScalarAOCLFuncs.def new file mode 100644 index 0000000000000..906b1612afa5c --- /dev/null +++ b/llvm/include/llvm/Analysis/ScalarAOCLFuncs.def @@ -0,0 +1,80 @@ +//===-- ScalarAOCLFuncs.def - AMD scalar math library mappings --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This .def file creates a mapping from standard scalar math functions to +// their corresponding fast entry points in the AMD AOCL scalar math library. +// The lowering is only legal under fast-math semantics. +// +//===----------------------------------------------------------------------===// + +#if defined(TLI_DEFINE_SCALAR_AOCL_FUNCS) +#define TLI_DEFINE_SCALAR_AOCL_FUNC(SCAL, AOCLENTRY) {SCAL, AOCLENTRY}, + +TLI_DEFINE_SCALAR_AOCL_FUNC("acosf", "amd_fastacosf") +TLI_DEFINE_SCALAR_AOCL_FUNC("__acosf_finite", "amd_fastacosf") +TLI_DEFINE_SCALAR_AOCL_FUNC("acos", "amd_fastacos") +TLI_DEFINE_SCALAR_AOCL_FUNC("__acos_finite", "amd_fastacos") + +TLI_DEFINE_SCALAR_AOCL_FUNC("asinf", "amd_fastasinf") +TLI_DEFINE_SCALAR_AOCL_FUNC("__asinf_finite", "amd_fastasinf") +TLI_DEFINE_SCALAR_AOCL_FUNC("asin", "amd_fastasin") +TLI_DEFINE_SCALAR_AOCL_FUNC("__asin_finite", "amd_fastasin") + +TLI_DEFINE_SCALAR_AOCL_FUNC("atanf", "amd_fastatanf") +TLI_DEFINE_SCALAR_AOCL_FUNC("__atanf_finite", "amd_fastatanf") +TLI_DEFINE_SCALAR_AOCL_FUNC("atan", "amd_fastatan") +TLI_DEFINE_SCALAR_AOCL_FUNC("__atan_finite", "amd_fastatan") + +TLI_DEFINE_SCALAR_AOCL_FUNC("cosf", "amd_fastcosf") +TLI_DEFINE_SCALAR_AOCL_FUNC("__cosf_finite", "amd_fastcosf") +TLI_DEFINE_SCALAR_AOCL_FUNC("llvm.cos.f32", "amd_fastcosf") +TLI_DEFINE_SCALAR_AOCL_FUNC("cos", "amd_fastcos") +TLI_DEFINE_SCALAR_AOCL_FUNC("__cos_finite", "amd_fastcos") +TLI_DEFINE_SCALAR_AOCL_FUNC("llvm.cos.f64", "amd_fastcos") + +TLI_DEFINE_SCALAR_AOCL_FUNC("erff", "amd_fasterff") +TLI_DEFINE_SCALAR_AOCL_FUNC("__erff_finite", "amd_fasterff") +TLI_DEFINE_SCALAR_AOCL_FUNC("erf", "amd_fasterf") +TLI_DEFINE_SCALAR_AOCL_FUNC("__erf_finite", "amd_fasterf") + +TLI_DEFINE_SCALAR_AOCL_FUNC("expf", "amd_fastexpf") +TLI_DEFINE_SCALAR_AOCL_FUNC("__expf_finite", "amd_fastexpf") +TLI_DEFINE_SCALAR_AOCL_FUNC("llvm.exp.f32", "amd_fastexpf") +TLI_DEFINE_SCALAR_AOCL_FUNC("exp", "amd_fastexp") +TLI_DEFINE_SCALAR_AOCL_FUNC("__exp_finite", "amd_fastexp") +TLI_DEFINE_SCALAR_AOCL_FUNC("llvm.exp.f64", "amd_fastexp") + +TLI_DEFINE_SCALAR_AOCL_FUNC("logf", "amd_fastlogf") +TLI_DEFINE_SCALAR_AOCL_FUNC("__logf_finite", "amd_fastlogf") +TLI_DEFINE_SCALAR_AOCL_FUNC("llvm.log.f32", "amd_fastlogf") +TLI_DEFINE_SCALAR_AOCL_FUNC("log", "amd_fastlog") +TLI_DEFINE_SCALAR_AOCL_FUNC("__log_finite", "amd_fastlog") +TLI_DEFINE_SCALAR_AOCL_FUNC("llvm.log.f64", "amd_fastlog") + +TLI_DEFINE_SCALAR_AOCL_FUNC("powf", "amd_fastpowf") +TLI_DEFINE_SCALAR_AOCL_FUNC("__powf_finite", "amd_fastpowf") +TLI_DEFINE_SCALAR_AOCL_FUNC("llvm.pow.f32", "amd_fastpowf") +TLI_DEFINE_SCALAR_AOCL_FUNC("pow", "amd_fastpow") +TLI_DEFINE_SCALAR_AOCL_FUNC("__pow_finite", "amd_fastpow") +TLI_DEFINE_SCALAR_AOCL_FUNC("llvm.pow.f64", "amd_fastpow") + +TLI_DEFINE_SCALAR_AOCL_FUNC("sinf", "amd_fastsinf") +TLI_DEFINE_SCALAR_AOCL_FUNC("__sinf_finite", "amd_fastsinf") +TLI_DEFINE_SCALAR_AOCL_FUNC("llvm.sin.f32", "amd_fastsinf") +TLI_DEFINE_SCALAR_AOCL_FUNC("sin", "amd_fastsin") +TLI_DEFINE_SCALAR_AOCL_FUNC("__sin_finite", "amd_fastsin") +TLI_DEFINE_SCALAR_AOCL_FUNC("llvm.sin.f64", "amd_fastsin") + +TLI_DEFINE_SCALAR_AOCL_FUNC("tanf", "amd_fasttanf") +TLI_DEFINE_SCALAR_AOCL_FUNC("__tanf_finite", "amd_fasttanf") +TLI_DEFINE_SCALAR_AOCL_FUNC("tan", "amd_fasttan") +TLI_DEFINE_SCALAR_AOCL_FUNC("__tan_finite", "amd_fasttan") +#endif + +#undef TLI_DEFINE_SCALAR_AOCL_FUNCS +#undef TLI_DEFINE_SCALAR_AOCL_FUNC diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.h b/llvm/include/llvm/Analysis/TargetLibraryInfo.h index cd61b925ee5d7..4df61a4ec0791 100644 --- a/llvm/include/llvm/Analysis/TargetLibraryInfo.h +++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.h @@ -109,12 +109,25 @@ class TargetLibraryInfoImpl { /// on VectorFnName rather than ScalarFnName. std::vector<VecDesc> ScalarDescs; + /// Mapping from a standard scalar math function name to its AMD scalar math + /// library fast-call equivalent (e.g. "tan" -> "amd_fasttan"). Populated when + /// an AMD scalar math library is selected. + DenseMap<StringRef, StringRef> LibScalarFunctions; + /// Return true if the function type FTy is valid for the library function /// F, regardless of whether the function is available. LLVM_ABI bool isValidProtoForLibFunc(const FunctionType &FTy, LibFunc F, const Module &M) const; public: + /// Scalar math library selection used for lowering standard scalar math + /// calls to faster, library-specific entry points. + enum ScalarLibrary { + Default_Scalar_Library, // Use default library. + SCALAR_AMDLIBM // AMD scalar math library. + }; + ScalarLibrary ScalarMathLib = Default_Scalar_Library; + TargetLibraryInfoImpl() = delete; LLVM_ABI explicit TargetLibraryInfoImpl( const Triple &T, VectorLibrary VecLib = VectorLibrary::NoLibrary); @@ -182,6 +195,20 @@ class TargetLibraryInfoImpl { addVectorizableFunctionsFromVecLib(enum VectorLibrary VecLib, const llvm::Triple &TargetTriple); + /// Populate the scalar math function mappings for the given scalar library + /// and record it as the selected scalar math library. + LLVM_ABI void addScalarFunctionsFromMathLib(enum ScalarLibrary ScalarLib); + + /// Return the library-specific scalar function name for \p F, or an empty + /// StringRef if no mapping exists. + LLVM_ABI StringRef getScalarFunctionFromMathLib(StringRef F) const; + + /// Return the currently selected scalar math library. + LLVM_ABI ScalarLibrary getScalarMathLib() const; + + /// Set the selected scalar math library. + LLVM_ABI void setScalarMathLib(enum ScalarLibrary ScalarLib); + /// Return true if the function F has a vector equivalent with vectorization /// factor VF. bool isFunctionVectorizable(StringRef F, const ElementCount &VF) const { @@ -387,6 +414,12 @@ class TargetLibraryInfo { bool Masked) const { return Impl->getVectorMappingInfo(F, VF, Masked); } + StringRef getScalarFunctionFromMathLib(StringRef F) const { + return Impl->getScalarFunctionFromMathLib(F); + } + TargetLibraryInfoImpl::ScalarLibrary getScalarMathLib() const { + return Impl->getScalarMathLib(); + } /// Tests if the function is both available and a candidate for optimized code /// generation. diff --git a/llvm/lib/Analysis/TargetLibraryInfo.cpp b/llvm/lib/Analysis/TargetLibraryInfo.cpp index 763c04b9b06f7..45a20b8398b4e 100644 --- a/llvm/lib/Analysis/TargetLibraryInfo.cpp +++ b/llvm/lib/Analysis/TargetLibraryInfo.cpp @@ -18,9 +18,18 @@ #include "llvm/IR/Module.h" #include "llvm/IR/SystemLibraries.h" #include "llvm/InitializePasses.h" +#include "llvm/Support/CommandLine.h" #include "llvm/TargetParser/Triple.h" using namespace llvm; +static cl::opt<TargetLibraryInfoImpl::ScalarLibrary> ClScalarLibrary( + "scalar-library", cl::Hidden, cl::desc("Scalar functions library"), + cl::init(TargetLibraryInfoImpl::Default_Scalar_Library), + cl::values(clEnumValN(TargetLibraryInfoImpl::Default_Scalar_Library, "none", + "Use default library"), + clEnumValN(TargetLibraryInfoImpl::SCALAR_AMDLIBM, "AMDLIBM", + "AMD scalar math library"))); + #define GET_TARGET_LIBRARY_INFO_STRING_TABLE #include "llvm/Analysis/TargetLibraryInfo.inc" @@ -902,6 +911,7 @@ TargetLibraryInfoImpl::TargetLibraryInfoImpl(const Triple &T, memset(AvailableArray, -1, sizeof(AvailableArray)); initialize(*this, T, StandardNamesStrTable, VecLib); + addScalarFunctionsFromMathLib(ClScalarLibrary); } TargetLibraryInfoImpl::TargetLibraryInfoImpl(const TargetLibraryInfoImpl &TLI) @@ -913,6 +923,8 @@ TargetLibraryInfoImpl::TargetLibraryInfoImpl(const TargetLibraryInfoImpl &TLI) memcpy(AvailableArray, TLI.AvailableArray, sizeof(AvailableArray)); VectorDescs = TLI.VectorDescs; ScalarDescs = TLI.ScalarDescs; + LibScalarFunctions = TLI.LibScalarFunctions; + ScalarMathLib = TLI.ScalarMathLib; } TargetLibraryInfoImpl::TargetLibraryInfoImpl(TargetLibraryInfoImpl &&TLI) @@ -926,6 +938,8 @@ TargetLibraryInfoImpl::TargetLibraryInfoImpl(TargetLibraryInfoImpl &&TLI) AvailableArray); VectorDescs = TLI.VectorDescs; ScalarDescs = TLI.ScalarDescs; + LibScalarFunctions = TLI.LibScalarFunctions; + ScalarMathLib = TLI.ScalarMathLib; } TargetLibraryInfoImpl &TargetLibraryInfoImpl::operator=(const TargetLibraryInfoImpl &TLI) { @@ -936,6 +950,8 @@ TargetLibraryInfoImpl &TargetLibraryInfoImpl::operator=(const TargetLibraryInfoI ShouldSignExtI32Return = TLI.ShouldSignExtI32Return; SizeOfInt = TLI.SizeOfInt; memcpy(AvailableArray, TLI.AvailableArray, sizeof(AvailableArray)); + LibScalarFunctions = TLI.LibScalarFunctions; + ScalarMathLib = TLI.ScalarMathLib; return *this; } @@ -948,6 +964,8 @@ TargetLibraryInfoImpl &TargetLibraryInfoImpl::operator=(TargetLibraryInfoImpl && SizeOfInt = TLI.SizeOfInt; std::move(std::begin(TLI.AvailableArray), std::end(TLI.AvailableArray), AvailableArray); + LibScalarFunctions = TLI.LibScalarFunctions; + ScalarMathLib = TLI.ScalarMathLib; return *this; } @@ -1397,6 +1415,40 @@ void TargetLibraryInfoImpl::addVectorizableFunctionsFromVecLib( } } +void TargetLibraryInfoImpl::addScalarFunctionsFromMathLib( + enum ScalarLibrary ScalarLib) { + setScalarMathLib(ScalarLib); + switch (ScalarLib) { + case ScalarLibrary::SCALAR_AMDLIBM: { + const DenseMap<StringRef, StringRef> ScalarAOCLFuncs = { +#define TLI_DEFINE_SCALAR_AOCL_FUNCS +#include "llvm/Analysis/ScalarAOCLFuncs.def" + }; + LibScalarFunctions.insert(ScalarAOCLFuncs.begin(), ScalarAOCLFuncs.end()); + break; + } + case ScalarLibrary::Default_Scalar_Library: + break; + } +} + +void TargetLibraryInfoImpl::setScalarMathLib(enum ScalarLibrary ScalarLib) { + ScalarMathLib = ScalarLib; +} + +StringRef TargetLibraryInfoImpl::getScalarFunctionFromMathLib( + StringRef ScalarFnName) const { + auto Iter = LibScalarFunctions.find(ScalarFnName); + if (Iter == LibScalarFunctions.end()) + return StringRef(); + return Iter->second; +} + +TargetLibraryInfoImpl::ScalarLibrary +TargetLibraryInfoImpl::getScalarMathLib() const { + return ScalarMathLib; +} + bool TargetLibraryInfoImpl::isFunctionVectorizable(StringRef funcName) const { funcName = sanitizeFunctionName(funcName); if (funcName.empty()) diff --git a/llvm/lib/Target/X86/CMakeLists.txt b/llvm/lib/Target/X86/CMakeLists.txt index 62987bdbd1c2b..0c07e7c0c4b7f 100644 --- a/llvm/lib/Target/X86/CMakeLists.txt +++ b/llvm/lib/Target/X86/CMakeLists.txt @@ -56,6 +56,7 @@ set(sources X86FlagsCopyLowering.cpp X86FloatingPoint.cpp X86FrameLowering.cpp + X86GenScalarAmdFastCalls.cpp X86ISelDAGToDAG.cpp X86ISelLowering.cpp X86ISelLoweringCall.cpp diff --git a/llvm/lib/Target/X86/X86.h b/llvm/lib/Target/X86/X86.h index eef4de389a7de..e790a602e0f08 100644 --- a/llvm/lib/Target/X86/X86.h +++ b/llvm/lib/Target/X86/X86.h @@ -483,6 +483,12 @@ class X86ArgumentStackSlotPass FunctionPass *createX86ArgumentStackSlotLegacyPass(); +/// This pass rewrites scalar math library calls (e.g. tan) to their AMD AOCL +/// fast-call equivalents (e.g. amd_fasttan) under fast-math semantics. +FunctionPass *createX86GenScalarAmdFastCallsPass(); +void initializeX86GenScalarAmdFastCallsPass(PassRegistry &); +extern char &X86GenScalarAmdFastCallsID; + void initializeCompressEVEXLegacyPass(PassRegistry &); void initializeX86FixupBWInstLegacyPass(PassRegistry &); void initializeFixupLEAsLegacyPass(PassRegistry &); diff --git a/llvm/lib/Target/X86/X86GenScalarAmdFastCalls.cpp b/llvm/lib/Target/X86/X86GenScalarAmdFastCalls.cpp new file mode 100644 index 0000000000000..85544a2436c73 --- /dev/null +++ b/llvm/lib/Target/X86/X86GenScalarAmdFastCalls.cpp @@ -0,0 +1,176 @@ +//===-- X86GenScalarAmdFastCalls.cpp --------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This transformation converts standard scalar math function calls into their +// corresponding AMD AOCL scalar library entries for X86 targets, e.g.: +// tan ---> amd_fasttan +// Such lowering is only legal under fast-math semantics and when the AMD +// scalar math library has been selected (-scalar-library=AMDLIBM / +// -fsclrlib=AMDLIBM). +// +//===----------------------------------------------------------------------===// + +#include "X86.h" +#include "X86Subtarget.h" +#include "X86TargetMachine.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h" +#include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Module.h" +#include "llvm/InitializePasses.h" +#include "llvm/Pass.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "x86-gen-scalar-aocl" + +using namespace llvm; + +namespace { + +class X86GenScalarAmdFastCalls : public MachineFunctionPass { +public: + static char ID; + + X86GenScalarAmdFastCalls() : MachineFunctionPass(ID) {} + + bool runOnMachineFunction(MachineFunction &F) override; + + StringRef getPassName() const override { + return "X86 Generate Scalar AOCL Entries"; + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired<TargetLibraryInfoWrapperPass>(); + AU.addRequired<MachineOptimizationRemarkEmitterPass>(); + MachineFunctionPass::getAnalysisUsage(AU); + } + +private: + TargetLibraryInfo *TLI = nullptr; + MachineOptimizationRemarkEmitter *ORE = nullptr; + bool isCandidateSafeToLower(MachineInstr *MI) const; + bool createScalarAOCLCall(MachineInstr *MI) const; +}; + +} // namespace + +// Rewriting a scalar math call to its AOCL fast-call variant is only legal +// under fast-math semantics. By the time this late machine pass runs, the +// per-operation fast-math flags carried by the original call have already been +// lowered away, so we rely on the function-level fast-math attribute that the +// frontend sets under -ffast-math. Together with the explicit +// -scalar-library=AMDLIBM opt-in (checked in runOnMachineFunction) this gates +// the transformation. +bool X86GenScalarAmdFastCalls::isCandidateSafeToLower(MachineInstr *MI) const { + const Function &F = MI->getMF()->getFunction(); + return F.getFnAttribute("no-signed-zeros-fp-math").getValueAsBool(); +} + +/// Lowers scalar math functions to scalar AOCL functions. +/// e.g.: tan --> amd_fasttan +/// The callsite symbol is updated during lowering. +bool X86GenScalarAmdFastCalls::createScalarAOCLCall(MachineInstr *MI) const { + StringRef CallSiteName = ""; + StringRef LibScalarFnName = ""; + if (MI->getOperand(0).isSymbol()) { + CallSiteName = MI->getOperand(0).getSymbolName(); + } else if (MI->getOperand(0).isGlobal()) { + CallSiteName = MI->getOperand(0).getGlobal()->getName(); + } else { + return false; + } + + LLVM_DEBUG(dbgs() << "Candidate Func = " << CallSiteName << "\n";); + if (CallSiteName.empty()) { + return false; + } + LibScalarFnName = TLI->getScalarFunctionFromMathLib(CallSiteName); + if (LibScalarFnName.empty()) { + LLVM_DEBUG(dbgs() << "Fast call not supported\n";); + return false; + } + LLVM_DEBUG(dbgs() << "Candidate Func has fast Call variant available = " + << LibScalarFnName << "\n";); + MI->getOperand(0).ChangeToES(LibScalarFnName.data(), + MI->getOperand(0).getTargetFlags()); + + LLVM_DEBUG(dbgs() << "Successfully replaced with fastcall= " << LibScalarFnName + << "\n";); + + ORE->emit([&]() { + return MachineOptimizationRemark(DEBUG_TYPE, "Passed", MI->getDebugLoc(), + MI->getParent()) + << "Successfully replaced with fastcall= " << LibScalarFnName + << "\n"; + }); + return true; +} + +bool X86GenScalarAmdFastCalls::runOnMachineFunction(MachineFunction &MF) { + bool Changed = false; + + if (skipFunction(MF.getFunction())) + return Changed; + if (MF.getFunction().isDeclaration()) + return Changed; + SmallVector<MachineInstr *, 4> Callsites; + for (auto &BB : MF) { + for (auto &I : BB) { + if (I.isCall()) { + Callsites.push_back(&I); + } + } + } + + if (Callsites.empty()) { + return Changed; + } + + TLI = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(MF.getFunction()); + ORE = &getAnalysis<MachineOptimizationRemarkEmitterPass>().getORE(); + if (!TLI) + return Changed; + + if (TLI->getScalarMathLib() != + TargetLibraryInfoImpl::ScalarLibrary::SCALAR_AMDLIBM) { + LLVM_DEBUG(dbgs() << "-scalar-library=AMDLIBM not used so bailing out.\n";); + return Changed; + } + + for (auto *CI : Callsites) { + if (isCandidateSafeToLower(CI)) { + LLVM_DEBUG(dbgs() << "Call Inst has fastMath flags\n";); + Changed |= createScalarAOCLCall(CI); + } else + LLVM_DEBUG(dbgs() << "Call Inst does not have fastMath flags\n";); + } + return Changed; +} + +char X86GenScalarAmdFastCalls::ID = 0; + +char &llvm::X86GenScalarAmdFastCallsID = X86GenScalarAmdFastCalls::ID; + +INITIALIZE_PASS_BEGIN(X86GenScalarAmdFastCalls, DEBUG_TYPE, + "Generate Scalar AMD Fast calls", false, false) +INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) +INITIALIZE_PASS_DEPENDENCY(MachineOptimizationRemarkEmitterPass) +INITIALIZE_PASS_END(X86GenScalarAmdFastCalls, DEBUG_TYPE, + "Generate Scalar AMD Fast calls", false, false) + +FunctionPass *llvm::createX86GenScalarAmdFastCallsPass() { + return new X86GenScalarAmdFastCalls(); +} diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp index 932669b5cbac6..7bf01aac98a1b 100644 --- a/llvm/lib/Target/X86/X86TargetMachine.cpp +++ b/llvm/lib/Target/X86/X86TargetMachine.cpp @@ -110,6 +110,7 @@ extern "C" LLVM_C_ABI void LLVMInitializeX86Target() { initializeX86PreLegalizerCombinerLegacyPass(PR); initializeX86PostLegalizerCombinerLegacyPass(PR); initializeX86WinEHUnwindV3Pass(PR); + initializeX86GenScalarAmdFastCallsPass(PR); } static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) { @@ -535,6 +536,10 @@ void X86PassConfig::addPreRegAlloc() { void X86PassConfig::addMachineSSAOptimization() { addPass(createX86DomainReassignmentLegacyPass()); + // Generate x86 target-specific function calls for scalar math functions + // that are available in the AMD AOCL library. + if (getOptLevel() == CodeGenOptLevel::Aggressive) + addPass(createX86GenScalarAmdFastCallsPass()); TargetPassConfig::addMachineSSAOptimization(); } diff --git a/llvm/test/CodeGen/X86/aocl-fast-scalar-calls-i686.ll b/llvm/test/CodeGen/X86/aocl-fast-scalar-calls-i686.ll new file mode 100644 index 0000000000000..8cc4b1d9b94e2 --- /dev/null +++ b/llvm/test/CodeGen/X86/aocl-fast-scalar-calls-i686.ll @@ -0,0 +1,31 @@ +; The AOCL fast-call lowering applies to 32-bit X86 (i686) as well as x86_64. + +; RUN: llc -mtriple=i686-unknown-linux-gnu -O3 -scalar-library=AMDLIBM < %s \ +; RUN: | FileCheck %s --check-prefix=AMD +; RUN: llc -mtriple=i686-unknown-linux-gnu -O3 < %s \ +; RUN: | FileCheck %s --check-prefix=STD + +declare double @tan(double) +declare float @expf(float) + +define double @call_tan(double %x) #0 { + %r = call double @tan(double %x) + %a = fadd double %r, %x + ret double %a +} +; AMD-LABEL: call_tan: +; AMD: calll{{.*}}amd_fasttan +; STD-LABEL: call_tan: +; STD: calll{{.*}}tan + +define float @call_expf(float %x) #0 { + %r = call float @expf(float %x) + %a = fadd float %r, %x + ret float %a +} +; AMD-LABEL: call_expf: +; AMD: calll{{.*}}amd_fastexpf +; STD-LABEL: call_expf: +; STD: calll{{.*}}expf + +attributes #0 = { "approx-func-fp-math"="true" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" } diff --git a/llvm/test/CodeGen/X86/aocl-fast-scalar-calls-mappings.ll b/llvm/test/CodeGen/X86/aocl-fast-scalar-calls-mappings.ll new file mode 100644 index 0000000000000..b98704409a0f3 --- /dev/null +++ b/llvm/test/CodeGen/X86/aocl-fast-scalar-calls-mappings.ll @@ -0,0 +1,124 @@ +; Exercises the scalar->AOCL fast-call name mapping under fast-math at -O3 with +; -scalar-library=AMDLIBM on X86: float variants, *_finite aliases and +; inverse-trig functions are rewritten, while math calls with no AOCL mapping +; (e.g. cbrt) are left untouched. + +; RUN: llc -mtriple=x86_64-unknown-linux-gnu -O3 -scalar-library=AMDLIBM < %s \ +; RUN: | FileCheck %s --check-prefix=AMD +; RUN: llc -mtriple=x86_64-unknown-linux-gnu -O3 < %s \ +; RUN: | FileCheck %s --check-prefix=STD + +declare float @tanf(float) +declare float @powf(float, float) +declare double @acos(double) +declare float @acosf(float) +declare double @atan(double) +declare double @cos(double) +declare float @sinf(float) +declare double @erf(double) +declare double @__exp_finite(double) +declare double @pow(double, double) +declare double @cbrt(double) + +; Single-precision variant: tanf -> amd_fasttanf +define float @call_tanf(float %x) #0 { + %r = call float @tanf(float %x) + %a = fadd float %r, %x + ret float %a +} +; AMD-LABEL: call_tanf: +; AMD: callq{{.*}}amd_fasttanf +; STD-LABEL: call_tanf: +; STD: callq{{.*}}tanf + +; Single-precision, two-argument variant: powf -> amd_fastpowf +define float @call_powf(float %x, float %y) #0 { + %r = call float @powf(float %x, float %y) + %a = fadd float %r, %x + ret float %a +} +; AMD-LABEL: call_powf: +; AMD: callq{{.*}}amd_fastpowf + +; Inverse-trigonometric functions. +define double @call_acos(double %x) #0 { + %r = call double @acos(double %x) + %a = fadd double %r, %x + ret double %a +} +; AMD-LABEL: call_acos: +; AMD: callq{{.*}}amd_fastacos + +define double @call_atan(double %x) #0 { + %r = call double @atan(double %x) + %a = fadd double %r, %x + ret double %a +} +; AMD-LABEL: call_atan: +; AMD: callq{{.*}}amd_fastatan + +; Single-precision inverse-trig: acosf -> amd_fastacosf +define float @call_acosf(float %x) #0 { + %r = call float @acosf(float %x) + %a = fadd float %r, %x + ret float %a +} +; AMD-LABEL: call_acosf: +; AMD: callq{{.*}}amd_fastacosf + +; cos(double) -> amd_fastcos +define double @call_cos(double %x) #0 { + %r = call double @cos(double %x) + %a = fadd double %r, %x + ret double %a +} +; AMD-LABEL: call_cos: +; AMD: callq{{.*}}amd_fastcos + +; sinf -> amd_fastsinf +define float @call_sinf(float %x) #0 { + %r = call float @sinf(float %x) + %a = fadd float %r, %x + ret float %a +} +; AMD-LABEL: call_sinf: +; AMD: callq{{.*}}amd_fastsinf + +; erf(double) -> amd_fasterf +define double @call_erf(double %x) #0 { + %r = call double @erf(double %x) + %a = fadd double %r, %x + ret double %a +} +; AMD-LABEL: call_erf: +; AMD: callq{{.*}}amd_fasterf + +; pow(double) -> amd_fastpow +define double @call_pow(double %x, double %y) #0 { + %r = call double @pow(double %x, double %y) + %a = fadd double %r, %x + ret double %a +} +; AMD-LABEL: call_pow: +; AMD: callq{{.*}}amd_fastpow + +; A *_finite alias maps to the same fast entry as the base function. +define double @call_exp_finite(double %x) #0 { + %r = call double @__exp_finite(double %x) + %a = fadd double %r, %x + ret double %a +} +; AMD-LABEL: call_exp_finite: +; AMD: callq{{.*}}amd_fastexp + +; cbrt has no AOCL mapping and must not be rewritten. +define double @call_cbrt_unmapped(double %x) #0 { + %r = call double @cbrt(double %x) + %a = fadd double %r, %x + ret double %a +} +; AMD-LABEL: call_cbrt_unmapped: +; AMD-NOT: amd_fast +; AMD: callq{{.*}}cbrt + +attributes #0 = { "approx-func-fp-math"="true" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" } diff --git a/llvm/test/CodeGen/X86/aocl-fast-scalar-calls.ll b/llvm/test/CodeGen/X86/aocl-fast-scalar-calls.ll new file mode 100644 index 0000000000000..96824158fe853 --- /dev/null +++ b/llvm/test/CodeGen/X86/aocl-fast-scalar-calls.ll @@ -0,0 +1,63 @@ +; Verify that, under fast-math at -O3 with -scalar-library=AMDLIBM, scalar math +; library calls are rewritten to their AMD AOCL fast-call equivalents on X86, +; and that they are left untouched without the option. + +; RUN: llc -mtriple=x86_64-unknown-linux-gnu -O3 -scalar-library=AMDLIBM < %s \ +; RUN: | FileCheck %s --check-prefix=AMD +; RUN: llc -mtriple=x86_64-unknown-linux-gnu -O3 < %s \ +; RUN: | FileCheck %s --check-prefix=STD + +declare double @tan(double) +declare double @exp(double) +declare double @log(double) + +define double @call_tan(double %x) #0 { +entry: + %r = call double @tan(double %x) + %a = fadd double %r, %x + ret double %a +} + +; AMD-LABEL: call_tan: +; AMD: callq{{.*}}amd_fasttan +; STD-LABEL: call_tan: +; STD: callq{{.*}}tan + +define double @call_exp(double %x) #0 { +entry: + %r = call double @exp(double %x) + %a = fadd double %r, %x + ret double %a +} + +; AMD-LABEL: call_exp: +; AMD: callq{{.*}}amd_fastexp +; STD-LABEL: call_exp: +; STD: callq{{.*}}exp + +define double @call_log(double %x) #0 { +entry: + %r = call double @log(double %x) + %a = fadd double %r, %x + ret double %a +} + +; AMD-LABEL: call_log: +; AMD: callq{{.*}}amd_fastlog +; STD-LABEL: call_log: +; STD: callq{{.*}}log + +; Without the fast-math attributes the call must not be rewritten even when the +; AMD scalar library is selected. +define double @call_tan_no_fastmath(double %x) { +entry: + %r = call double @tan(double %x) + %a = fadd double %r, %x + ret double %a +} + +; AMD-LABEL: call_tan_no_fastmath: +; AMD-NOT: amd_fasttan +; AMD: callq{{.*}}tan + +attributes #0 = { "approx-func-fp-math"="true" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" } diff --git a/llvm/test/CodeGen/X86/veclib-llvm.sincos.s b/llvm/test/CodeGen/X86/veclib-llvm.sincos.s new file mode 100644 index 0000000000000..df1edb99dcd53 --- /dev/null +++ b/llvm/test/CodeGen/X86/veclib-llvm.sincos.s @@ -0,0 +1,258 @@ + .att_syntax + .file "veclib-llvm.sincos.ll" + .text + .globl test_sincos_v4f32 # -- Begin function test_sincos_v4f32 + .p2align 4 + .type test_sincos_v4f32,@function +test_sincos_v4f32: # @test_sincos_v4f32 + .cfi_startproc +# %bb.0: + pushq %rbx + .cfi_def_cfa_offset 16 + subq $16, %rsp + .cfi_def_cfa_offset 32 + .cfi_offset %rbx, -16 + movq %rsi, %rbx + movq %rsp, %rsi + callq amd_vrs4_sincosf@PLT + movaps (%rsp), %xmm0 + movaps %xmm0, (%rbx) + addq $16, %rsp + .cfi_def_cfa_offset 16 + popq %rbx + .cfi_def_cfa_offset 8 + retq +.Lfunc_end0: + .size test_sincos_v4f32, .Lfunc_end0-test_sincos_v4f32 + .cfi_endproc + # -- End function + .globl test_sincos_v8f32 # -- Begin function test_sincos_v8f32 + .p2align 4 + .type test_sincos_v8f32,@function +test_sincos_v8f32: # @test_sincos_v8f32 + .cfi_startproc +# %bb.0: + pushq %r14 + .cfi_def_cfa_offset 16 + pushq %rbx + .cfi_def_cfa_offset 24 + subq $56, %rsp + .cfi_def_cfa_offset 80 + .cfi_offset %rbx, -24 + .cfi_offset %r14, -16 + movq %rsi, %rbx + movq %rdi, %r14 + movaps %xmm0, (%rsp) # 16-byte Spill + addq $16, %rdi + leaq 16(%rsp), %rsi + movaps %xmm1, %xmm0 + callq amd_vrs4_sincosf@PLT + leaq 32(%rsp), %rsi + movaps (%rsp), %xmm0 # 16-byte Reload + movq %r14, %rdi + callq amd_vrs4_sincosf@PLT + movaps 16(%rsp), %xmm0 + movaps 32(%rsp), %xmm1 + movaps %xmm1, (%rbx) + movaps %xmm0, 16(%rbx) + addq $56, %rsp + .cfi_def_cfa_offset 24 + popq %rbx + .cfi_def_cfa_offset 16 + popq %r14 + .cfi_def_cfa_offset 8 + retq +.Lfunc_end1: + .size test_sincos_v8f32, .Lfunc_end1-test_sincos_v8f32 + .cfi_endproc + # -- End function + .globl test_sincos_v16f32 # -- Begin function test_sincos_v16f32 + .p2align 4 + .type test_sincos_v16f32,@function +test_sincos_v16f32: # @test_sincos_v16f32 + .cfi_startproc +# %bb.0: + pushq %r14 + .cfi_def_cfa_offset 16 + pushq %rbx + .cfi_def_cfa_offset 24 + subq $120, %rsp + .cfi_def_cfa_offset 144 + .cfi_offset %rbx, -24 + .cfi_offset %r14, -16 + movq %rsi, %rbx + movq %rdi, %r14 + movaps %xmm3, (%rsp) # 16-byte Spill + movaps %xmm2, 16(%rsp) # 16-byte Spill + movaps %xmm0, 32(%rsp) # 16-byte Spill + addq $16, %rdi + leaq 80(%rsp), %rsi + movaps %xmm1, %xmm0 + callq amd_vrs4_sincosf@PLT + leaq 48(%r14), %rdi + leaq 48(%rsp), %rsi + movaps (%rsp), %xmm0 # 16-byte Reload + callq amd_vrs4_sincosf@PLT + leaq 32(%r14), %rdi + leaq 64(%rsp), %rsi + movaps 16(%rsp), %xmm0 # 16-byte Reload + callq amd_vrs4_sincosf@PLT + leaq 96(%rsp), %rsi + movaps 32(%rsp), %xmm0 # 16-byte Reload + movq %r14, %rdi + callq amd_vrs4_sincosf@PLT + movaps 80(%rsp), %xmm0 + movaps 48(%rsp), %xmm1 + movaps 64(%rsp), %xmm2 + movaps 96(%rsp), %xmm3 + movaps %xmm3, (%rbx) + movaps %xmm2, 32(%rbx) + movaps %xmm1, 48(%rbx) + movaps %xmm0, 16(%rbx) + addq $120, %rsp + .cfi_def_cfa_offset 24 + popq %rbx + .cfi_def_cfa_offset 16 + popq %r14 + .cfi_def_cfa_offset 8 + retq +.Lfunc_end2: + .size test_sincos_v16f32, .Lfunc_end2-test_sincos_v16f32 + .cfi_endproc + # -- End function + .globl test_sincos_v2f64 # -- Begin function test_sincos_v2f64 + .p2align 4 + .type test_sincos_v2f64,@function +test_sincos_v2f64: # @test_sincos_v2f64 + .cfi_startproc +# %bb.0: + pushq %rbx + .cfi_def_cfa_offset 16 + subq $16, %rsp + .cfi_def_cfa_offset 32 + .cfi_offset %rbx, -16 + movq %rsi, %rbx + movq %rsp, %rsi + callq amd_vrd2_sincos@PLT + movaps (%rsp), %xmm0 + movaps %xmm0, (%rbx) + addq $16, %rsp + .cfi_def_cfa_offset 16 + popq %rbx + .cfi_def_cfa_offset 8 + retq +.Lfunc_end3: + .size test_sincos_v2f64, .Lfunc_end3-test_sincos_v2f64 + .cfi_endproc + # -- End function + .globl test_sincos_v4f64 # -- Begin function test_sincos_v4f64 + .p2align 4 + .type test_sincos_v4f64,@function +test_sincos_v4f64: # @test_sincos_v4f64 + .cfi_startproc +# %bb.0: + pushq %r14 + .cfi_def_cfa_offset 16 + pushq %rbx + .cfi_def_cfa_offset 24 + subq $56, %rsp + .cfi_def_cfa_offset 80 + .cfi_offset %rbx, -24 + .cfi_offset %r14, -16 + movq %rsi, %rbx + movq %rdi, %r14 + movaps %xmm0, (%rsp) # 16-byte Spill + addq $16, %rdi + leaq 16(%rsp), %rsi + movaps %xmm1, %xmm0 + callq amd_vrd2_sincos@PLT + leaq 32(%rsp), %rsi + movaps (%rsp), %xmm0 # 16-byte Reload + movq %r14, %rdi + callq amd_vrd2_sincos@PLT + movaps 16(%rsp), %xmm0 + movaps 32(%rsp), %xmm1 + movaps %xmm1, (%rbx) + movaps %xmm0, 16(%rbx) + addq $56, %rsp + .cfi_def_cfa_offset 24 + popq %rbx + .cfi_def_cfa_offset 16 + popq %r14 + .cfi_def_cfa_offset 8 + retq +.Lfunc_end4: + .size test_sincos_v4f64, .Lfunc_end4-test_sincos_v4f64 + .cfi_endproc + # -- End function + .globl test_sincos_v8f64 # -- Begin function test_sincos_v8f64 + .p2align 4 + .type test_sincos_v8f64,@function +test_sincos_v8f64: # @test_sincos_v8f64 + .cfi_startproc +# %bb.0: + pushq %r14 + .cfi_def_cfa_offset 16 + pushq %rbx + .cfi_def_cfa_offset 24 + subq $120, %rsp + .cfi_def_cfa_offset 144 + .cfi_offset %rbx, -24 + .cfi_offset %r14, -16 + movq %rsi, %rbx + movq %rdi, %r14 + movaps %xmm3, (%rsp) # 16-byte Spill + movaps %xmm2, 16(%rsp) # 16-byte Spill + movaps %xmm0, 32(%rsp) # 16-byte Spill + addq $16, %rdi + leaq 80(%rsp), %rsi + movaps %xmm1, %xmm0 + callq amd_vrd2_sincos@PLT + leaq 48(%r14), %rdi + leaq 48(%rsp), %rsi + movaps (%rsp), %xmm0 # 16-byte Reload + callq amd_vrd2_sincos@PLT + leaq 32(%r14), %rdi + leaq 64(%rsp), %rsi + movaps 16(%rsp), %xmm0 # 16-byte Reload + callq amd_vrd2_sincos@PLT + leaq 96(%rsp), %rsi + movaps 32(%rsp), %xmm0 # 16-byte Reload + movq %r14, %rdi + callq amd_vrd2_sincos@PLT + movaps 80(%rsp), %xmm0 + movaps 48(%rsp), %xmm1 + movaps 64(%rsp), %xmm2 + movaps 96(%rsp), %xmm3 + movaps %xmm3, (%rbx) + movaps %xmm2, 32(%rbx) + movaps %xmm1, 48(%rbx) + movaps %xmm0, 16(%rbx) + addq $120, %rsp + .cfi_def_cfa_offset 24 + popq %rbx + .cfi_def_cfa_offset 16 + popq %r14 + .cfi_def_cfa_offset 8 + retq +.Lfunc_end5: + .size test_sincos_v8f64, .Lfunc_end5-test_sincos_v8f64 + .cfi_endproc + # -- End function + .globl test_sincos_v4f32_void # -- Begin function test_sincos_v4f32_void + .p2align 4 + .type test_sincos_v4f32_void,@function +test_sincos_v4f32_void: # @test_sincos_v4f32_void + .cfi_startproc +# %bb.0: + pushq %rax + .cfi_def_cfa_offset 16 + callq sincosf@PLT + popq %rax + .cfi_def_cfa_offset 8 + retq +.Lfunc_end6: + .size test_sincos_v4f32_void, .Lfunc_end6-test_sincos_v4f32_void + .cfi_endproc + # -- End function + .section ".note.GNU-stack","",@progbits >From 7586808a60e4782bb69e27e291f41e936fb27417 Mon Sep 17 00:00:00 2001 From: Rohit Aggarwal <[email protected]> Date: Mon, 3 Aug 2026 19:29:32 +0530 Subject: [PATCH 2/2] Fix the formatting issue. --- llvm/lib/Target/X86/X86GenScalarAmdFastCalls.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Target/X86/X86GenScalarAmdFastCalls.cpp b/llvm/lib/Target/X86/X86GenScalarAmdFastCalls.cpp index 85544a2436c73..0a4b1b7b32cd2 100644 --- a/llvm/lib/Target/X86/X86GenScalarAmdFastCalls.cpp +++ b/llvm/lib/Target/X86/X86GenScalarAmdFastCalls.cpp @@ -107,8 +107,8 @@ bool X86GenScalarAmdFastCalls::createScalarAOCLCall(MachineInstr *MI) const { MI->getOperand(0).ChangeToES(LibScalarFnName.data(), MI->getOperand(0).getTargetFlags()); - LLVM_DEBUG(dbgs() << "Successfully replaced with fastcall= " << LibScalarFnName - << "\n";); + LLVM_DEBUG(dbgs() << "Successfully replaced with fastcall= " + << LibScalarFnName << "\n";); ORE->emit([&]() { return MachineOptimizationRemark(DEBUG_TYPE, "Passed", MI->getDebugLoc(), _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
