https://github.com/chichunchen updated https://github.com/llvm/llvm-project/pull/186030
>From d51728d02f499161fc4d2bfc1e365327b41d836e Mon Sep 17 00:00:00 2001 From: "Chi Chun, Chen" <[email protected]> Date: Wed, 11 Mar 2026 21:13:25 -0500 Subject: [PATCH] [Clang][OpenMP] Move declare simd codegen into OMPIRBuilder Refactor declare simd codegen by moving logic that does not depend on Clang declarations into OpenMPIRBuilder. --- clang/lib/CodeGen/CGOpenMPRuntime.cpp | 362 ++++-------------- .../llvm/Frontend/OpenMP/OMPIRBuilder.h | 71 ++++ llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 214 +++++++++++ 3 files changed, 363 insertions(+), 284 deletions(-) diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp index fa3dfcd856b30..faa762ec43e3f 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -11853,27 +11853,9 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall( } } -namespace { - /// Kind of parameter in a function with 'declare simd' directive. -enum ParamKindTy { - Linear, - LinearRef, - LinearUVal, - LinearVal, - Uniform, - Vector, -}; -/// Attribute set of the parameter. -struct ParamAttrTy { - ParamKindTy Kind = Vector; - llvm::APSInt StrideOrArg; - llvm::APSInt Alignment; - bool HasVarStride = false; -}; -} // namespace - -static unsigned evaluateCDTSize(const FunctionDecl *FD, - ArrayRef<ParamAttrTy> ParamAttrs) { +static unsigned +evaluateCDTSize(const FunctionDecl *FD, + ArrayRef<llvm::OpenMPIRBuilder::DeclareSimdAttrTy> ParamAttrs) { // Every vector variant of a SIMD-enabled function has a vector length (VLEN). // If OpenMP clause "simdlen" is used, the VLEN is the value of the argument // of that clause. The VLEN value must be power of 2. @@ -11903,13 +11885,15 @@ static unsigned evaluateCDTSize(const FunctionDecl *FD, } else { unsigned Offset = 0; if (const auto *MD = dyn_cast<CXXMethodDecl>(FD)) { - if (ParamAttrs[Offset].Kind == Vector) + if (ParamAttrs[Offset].Kind == + llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector) CDT = C.getPointerType(C.getCanonicalTagType(MD->getParent())); ++Offset; } if (CDT.isNull()) { for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) { - if (ParamAttrs[I + Offset].Kind == Vector) { + if (ParamAttrs[I + Offset].Kind == + llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector) { CDT = FD->getParamDecl(I)->getType(); break; } @@ -11924,107 +11908,6 @@ static unsigned evaluateCDTSize(const FunctionDecl *FD, return C.getTypeSize(CDT); } -/// Mangle the parameter part of the vector function name according to -/// their OpenMP classification. The mangling function is defined in -/// section 4.5 of the AAVFABI(2021Q1). -static std::string mangleVectorParameters(ArrayRef<ParamAttrTy> ParamAttrs) { - SmallString<256> Buffer; - llvm::raw_svector_ostream Out(Buffer); - for (const auto &ParamAttr : ParamAttrs) { - switch (ParamAttr.Kind) { - case Linear: - Out << 'l'; - break; - case LinearRef: - Out << 'R'; - break; - case LinearUVal: - Out << 'U'; - break; - case LinearVal: - Out << 'L'; - break; - case Uniform: - Out << 'u'; - break; - case Vector: - Out << 'v'; - break; - } - if (ParamAttr.HasVarStride) - Out << "s" << ParamAttr.StrideOrArg; - else if (ParamAttr.Kind == Linear || ParamAttr.Kind == LinearRef || - ParamAttr.Kind == LinearUVal || ParamAttr.Kind == LinearVal) { - // Don't print the step value if it is not present or if it is - // equal to 1. - if (ParamAttr.StrideOrArg < 0) - Out << 'n' << -ParamAttr.StrideOrArg; - else if (ParamAttr.StrideOrArg != 1) - Out << ParamAttr.StrideOrArg; - } - - if (!!ParamAttr.Alignment) - Out << 'a' << ParamAttr.Alignment; - } - - return std::string(Out.str()); -} - -static void -emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn, - const llvm::APSInt &VLENVal, - ArrayRef<ParamAttrTy> ParamAttrs, - OMPDeclareSimdDeclAttr::BranchStateTy State) { - struct ISADataTy { - char ISA; - unsigned VecRegSize; - }; - ISADataTy ISAData[] = { - { - 'b', 128 - }, // SSE - { - 'c', 256 - }, // AVX - { - 'd', 256 - }, // AVX2 - { - 'e', 512 - }, // AVX512 - }; - llvm::SmallVector<char, 2> Masked; - switch (State) { - case OMPDeclareSimdDeclAttr::BS_Undefined: - Masked.push_back('N'); - Masked.push_back('M'); - break; - case OMPDeclareSimdDeclAttr::BS_Notinbranch: - Masked.push_back('N'); - break; - case OMPDeclareSimdDeclAttr::BS_Inbranch: - Masked.push_back('M'); - break; - } - for (char Mask : Masked) { - for (const ISADataTy &Data : ISAData) { - SmallString<256> Buffer; - llvm::raw_svector_ostream Out(Buffer); - Out << "_ZGV" << Data.ISA << Mask; - if (!VLENVal) { - unsigned NumElts = evaluateCDTSize(FD, ParamAttrs); - assert(NumElts && "Non-zero simdlen/cdtsize expected"); - Out << llvm::APSInt::getUnsigned(Data.VecRegSize / NumElts); - } else { - Out << VLENVal; - } - Out << mangleVectorParameters(ParamAttrs); - Out << '_' << Fn->getName(); - Fn->addFnAttr(Out.str()); - } - } -} - // This are the Functions that are needed to mangle the name of the // vector functions generated by the compiler, according to the rules // defined in the "Vector Function ABI specifications for AArch64", @@ -12032,19 +11915,22 @@ emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn, // https://developer.arm.com/products/software-development-tools/hpc/arm-compiler-for-hpc/vector-function-abi. /// Maps To Vector (MTV), as defined in 4.1.1 of the AAVFABI (2021Q1). -static bool getAArch64MTV(QualType QT, ParamKindTy Kind) { +static bool getAArch64MTV(QualType QT, + llvm::OpenMPIRBuilder::DeclareSimdKindTy Kind) { QT = QT.getCanonicalType(); if (QT->isVoidType()) return false; - if (Kind == ParamKindTy::Uniform) + if (Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform) return false; - if (Kind == ParamKindTy::LinearUVal || Kind == ParamKindTy::LinearRef) + if (Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal || + Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef) return false; - if ((Kind == ParamKindTy::Linear || Kind == ParamKindTy::LinearVal) && + if ((Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear || + Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal) && !QT->isReferenceType()) return false; @@ -12077,7 +11963,9 @@ static bool getAArch64PBV(QualType QT, ASTContext &C) { /// Computes the lane size (LS) of a return type or of an input parameter, /// as defined by `LS(P)` in 3.2.1 of the AAVFABI. /// TODO: Add support for references, section 3.2.1, item 1. -static unsigned getAArch64LS(QualType QT, ParamKindTy Kind, ASTContext &C) { +static unsigned getAArch64LS(QualType QT, + llvm::OpenMPIRBuilder::DeclareSimdKindTy Kind, + ASTContext &C) { if (!getAArch64MTV(QT, Kind) && QT.getCanonicalType()->isPointerType()) { QualType PTy = QT.getCanonicalType()->getPointeeType(); if (getAArch64PBV(PTy, C)) @@ -12093,7 +11981,8 @@ static unsigned getAArch64LS(QualType QT, ParamKindTy Kind, ASTContext &C) { // signature of the scalar function, as defined in 3.2.2 of the // AAVFABI. static std::tuple<unsigned, unsigned, bool> -getNDSWDS(const FunctionDecl *FD, ArrayRef<ParamAttrTy> ParamAttrs) { +getNDSWDS(const FunctionDecl *FD, + ArrayRef<llvm::OpenMPIRBuilder::DeclareSimdAttrTy> ParamAttrs) { QualType RetType = FD->getReturnType().getCanonicalType(); ASTContext &C = FD->getASTContext(); @@ -12102,7 +11991,8 @@ getNDSWDS(const FunctionDecl *FD, ArrayRef<ParamAttrTy> ParamAttrs) { llvm::SmallVector<unsigned, 8> Sizes; if (!RetType->isVoidType()) { - Sizes.push_back(getAArch64LS(RetType, ParamKindTy::Vector, C)); + Sizes.push_back(getAArch64LS( + RetType, llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector, C)); if (!getAArch64PBV(RetType, C) && getAArch64MTV(RetType, {})) OutputBecomesInput = true; } @@ -12125,155 +12015,43 @@ getNDSWDS(const FunctionDecl *FD, ArrayRef<ParamAttrTy> ParamAttrs) { OutputBecomesInput); } -// Function used to add the attribute. The parameter `VLEN` is -// templated to allow the use of "x" when targeting scalable functions -// for SVE. -template <typename T> -static void addAArch64VectorName(T VLEN, StringRef LMask, StringRef Prefix, - char ISA, StringRef ParSeq, - StringRef MangledName, bool OutputBecomesInput, - llvm::Function *Fn) { - SmallString<256> Buffer; - llvm::raw_svector_ostream Out(Buffer); - Out << Prefix << ISA << LMask << VLEN; - if (OutputBecomesInput) - Out << "v"; - Out << ParSeq << "_" << MangledName; - Fn->addFnAttr(Out.str()); -} - -// Helper function to generate the Advanced SIMD names depending on -// the value of the NDS when simdlen is not present. -static void addAArch64AdvSIMDNDSNames(unsigned NDS, StringRef Mask, - StringRef Prefix, char ISA, - StringRef ParSeq, StringRef MangledName, - bool OutputBecomesInput, - llvm::Function *Fn) { - switch (NDS) { - case 8: - addAArch64VectorName(8, Mask, Prefix, ISA, ParSeq, MangledName, - OutputBecomesInput, Fn); - addAArch64VectorName(16, Mask, Prefix, ISA, ParSeq, MangledName, - OutputBecomesInput, Fn); - break; - case 16: - addAArch64VectorName(4, Mask, Prefix, ISA, ParSeq, MangledName, - OutputBecomesInput, Fn); - addAArch64VectorName(8, Mask, Prefix, ISA, ParSeq, MangledName, - OutputBecomesInput, Fn); - break; - case 32: - addAArch64VectorName(2, Mask, Prefix, ISA, ParSeq, MangledName, - OutputBecomesInput, Fn); - addAArch64VectorName(4, Mask, Prefix, ISA, ParSeq, MangledName, - OutputBecomesInput, Fn); - break; - case 64: - case 128: - addAArch64VectorName(2, Mask, Prefix, ISA, ParSeq, MangledName, - OutputBecomesInput, Fn); - break; - default: - llvm_unreachable("Scalar type is too wide."); +static llvm::OpenMPIRBuilder::DeclareSimdBranch +convertDeclareSimdBranch(OMPDeclareSimdDeclAttr::BranchStateTy State) { + switch (State) { + case OMPDeclareSimdDeclAttr::BS_Undefined: + return llvm::OpenMPIRBuilder::DeclareSimdBranch::Undefined; + case OMPDeclareSimdDeclAttr::BS_Inbranch: + return llvm::OpenMPIRBuilder::DeclareSimdBranch::Inbranch; + case OMPDeclareSimdDeclAttr::BS_Notinbranch: + return llvm::OpenMPIRBuilder::DeclareSimdBranch::Notinbranch; } + llvm_unreachable("unexpected declare simd branch state"); } -/// Emit vector function attributes for AArch64, as defined in the AAVFABI. -static void emitAArch64DeclareSimdFunction( - CodeGenModule &CGM, const FunctionDecl *FD, unsigned UserVLEN, - ArrayRef<ParamAttrTy> ParamAttrs, - OMPDeclareSimdDeclAttr::BranchStateTy State, StringRef MangledName, - char ISA, unsigned VecRegSize, llvm::Function *Fn, SourceLocation SLoc) { - - // Get basic data for building the vector signature. - const auto Data = getNDSWDS(FD, ParamAttrs); - const unsigned NDS = std::get<0>(Data); - const unsigned WDS = std::get<1>(Data); - const bool OutputBecomesInput = std::get<2>(Data); - - // Check the values provided via `simdlen` by the user. - // 1. A `simdlen(1)` doesn't produce vector signatures, +// Check the values provided via `simdlen` by the user. +static bool validateAArch64Simdlen(CodeGenModule &CGM, SourceLocation SLoc, + unsigned UserVLEN, unsigned WDS, char ISA) { + // 1. A `simdlen(1)` doesn't produce vector signatures. if (UserVLEN == 1) { CGM.getDiags().Report(SLoc, diag::warn_simdlen_1_no_effect); - return; + return false; } - // 2. Section 3.3.1, item 1: user input must be a power of 2 for - // Advanced SIMD output. + // 2. Section 3.3.1, item 1: user input must be a power of 2 for Advanced + // SIMD. if (ISA == 'n' && UserVLEN && !llvm::isPowerOf2_32(UserVLEN)) { CGM.getDiags().Report(SLoc, diag::warn_simdlen_requires_power_of_2); - return; + return false; } - // 3. Section 3.4.1. SVE fixed lengh must obey the architectural - // limits. - if (ISA == 's' && UserVLEN != 0) { - if ((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0)) { - CGM.getDiags().Report(SLoc, diag::warn_simdlen_must_fit_lanes) << WDS; - return; - } + // 3. Section 3.4.1: SVE fixed length must obey the architectural limits. + if (ISA == 's' && UserVLEN != 0 && + ((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0))) { + CGM.getDiags().Report(SLoc, diag::warn_simdlen_must_fit_lanes) << WDS; + return false; } - // Sort out parameter sequence. - const std::string ParSeq = mangleVectorParameters(ParamAttrs); - StringRef Prefix = "_ZGV"; - // Generate simdlen from user input (if any). - if (UserVLEN) { - if (ISA == 's') { - // SVE generates only a masked function. - addAArch64VectorName(UserVLEN, "M", Prefix, ISA, ParSeq, MangledName, - OutputBecomesInput, Fn); - } else { - assert(ISA == 'n' && "Expected ISA either 's' or 'n'."); - // Advanced SIMD generates one or two functions, depending on - // the `[not]inbranch` clause. - switch (State) { - case OMPDeclareSimdDeclAttr::BS_Undefined: - addAArch64VectorName(UserVLEN, "N", Prefix, ISA, ParSeq, MangledName, - OutputBecomesInput, Fn); - addAArch64VectorName(UserVLEN, "M", Prefix, ISA, ParSeq, MangledName, - OutputBecomesInput, Fn); - break; - case OMPDeclareSimdDeclAttr::BS_Notinbranch: - addAArch64VectorName(UserVLEN, "N", Prefix, ISA, ParSeq, MangledName, - OutputBecomesInput, Fn); - break; - case OMPDeclareSimdDeclAttr::BS_Inbranch: - addAArch64VectorName(UserVLEN, "M", Prefix, ISA, ParSeq, MangledName, - OutputBecomesInput, Fn); - break; - } - } - } else { - // If no user simdlen is provided, follow the AAVFABI rules for - // generating the vector length. - if (ISA == 's') { - // SVE, section 3.4.1, item 1. - addAArch64VectorName("x", "M", Prefix, ISA, ParSeq, MangledName, - OutputBecomesInput, Fn); - } else { - assert(ISA == 'n' && "Expected ISA either 's' or 'n'."); - // Advanced SIMD, Section 3.3.1 of the AAVFABI, generates one or - // two vector names depending on the use of the clause - // `[not]inbranch`. - switch (State) { - case OMPDeclareSimdDeclAttr::BS_Undefined: - addAArch64AdvSIMDNDSNames(NDS, "N", Prefix, ISA, ParSeq, MangledName, - OutputBecomesInput, Fn); - addAArch64AdvSIMDNDSNames(NDS, "M", Prefix, ISA, ParSeq, MangledName, - OutputBecomesInput, Fn); - break; - case OMPDeclareSimdDeclAttr::BS_Notinbranch: - addAArch64AdvSIMDNDSNames(NDS, "N", Prefix, ISA, ParSeq, MangledName, - OutputBecomesInput, Fn); - break; - case OMPDeclareSimdDeclAttr::BS_Inbranch: - addAArch64AdvSIMDNDSNames(NDS, "M", Prefix, ISA, ParSeq, MangledName, - OutputBecomesInput, Fn); - break; - } - } - } + return true; } void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD, @@ -12291,7 +12069,8 @@ void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD, ++ParamPos; } for (const auto *Attr : FD->specific_attrs<OMPDeclareSimdDeclAttr>()) { - llvm::SmallVector<ParamAttrTy, 8> ParamAttrs(ParamPositions.size()); + llvm::SmallVector<llvm::OpenMPIRBuilder::DeclareSimdAttrTy, 8> ParamAttrs( + ParamPositions.size()); // Mark uniform parameters. for (const Expr *E : Attr->uniforms()) { E = E->IgnoreParenImpCasts(); @@ -12305,7 +12084,8 @@ void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD, assert(It != ParamPositions.end() && "Function parameter not found"); Pos = It->second; } - ParamAttrs[Pos].Kind = Uniform; + ParamAttrs[Pos].Kind = + llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform; } // Get alignment info. auto *NI = Attr->alignments_begin(); @@ -12366,15 +12146,15 @@ void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD, .getQuantity(); } } - ParamAttrTy &ParamAttr = ParamAttrs[Pos]; + llvm::OpenMPIRBuilder::DeclareSimdAttrTy &ParamAttr = ParamAttrs[Pos]; if (*MI == OMPC_LINEAR_ref) - ParamAttr.Kind = LinearRef; + ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef; else if (*MI == OMPC_LINEAR_uval) - ParamAttr.Kind = LinearUVal; + ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal; else if (IsReferenceType) - ParamAttr.Kind = LinearVal; + ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal; else - ParamAttr.Kind = Linear; + ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear; // Assuming a stride of 1, for `linear` without modifiers. ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(1); if (*SI) { @@ -12399,7 +12179,10 @@ void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD, // rescale the value of linear_step with the byte size of the // pointee type. if (!ParamAttr.HasVarStride && - (ParamAttr.Kind == Linear || ParamAttr.Kind == LinearRef)) + (ParamAttr.Kind == + llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear || + ParamAttr.Kind == + llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef)) ParamAttr.StrideOrArg = ParamAttr.StrideOrArg * PtrRescalingFactor; ++SI; ++MI; @@ -12411,18 +12194,29 @@ void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD, VLENVal = VLENExpr->EvaluateKnownConstInt(C); ExprLoc = VLENExpr->getExprLoc(); } - OMPDeclareSimdDeclAttr::BranchStateTy State = Attr->getBranchState(); + llvm::OpenMPIRBuilder::DeclareSimdBranch State = + convertDeclareSimdBranch(Attr->getBranchState()); if (CGM.getTriple().isX86()) { - emitX86DeclareSimdFunction(FD, Fn, VLENVal, ParamAttrs, State); + unsigned NumElts = evaluateCDTSize(FD, ParamAttrs); + assert(NumElts && "Non-zero simdlen/cdtsize expected"); + OMPBuilder.emitX86DeclareSimdFunction(Fn, NumElts, VLENVal, ParamAttrs, + State); } else if (CGM.getTriple().getArch() == llvm::Triple::aarch64) { unsigned VLEN = VLENVal.getExtValue(); - StringRef MangledName = Fn->getName(); - if (CGM.getTarget().hasFeature("sve")) - emitAArch64DeclareSimdFunction(CGM, FD, VLEN, ParamAttrs, State, - MangledName, 's', 128, Fn, ExprLoc); - else if (CGM.getTarget().hasFeature("neon")) - emitAArch64DeclareSimdFunction(CGM, FD, VLEN, ParamAttrs, State, - MangledName, 'n', 128, Fn, ExprLoc); + // Get basic data for building the vector signature. + const auto Data = getNDSWDS(FD, ParamAttrs); + const unsigned NDS = std::get<0>(Data); + const unsigned WDS = std::get<1>(Data); + const bool OutputBecomesInput = std::get<2>(Data); + if (CGM.getTarget().hasFeature("sve")) { + if (validateAArch64Simdlen(CGM, ExprLoc, VLEN, WDS, 's')) + OMPBuilder.emitAArch64DeclareSimdFunction( + Fn, VLEN, ParamAttrs, State, 's', NDS, OutputBecomesInput); + } else if (CGM.getTarget().hasFeature("neon")) { + if (validateAArch64Simdlen(CGM, ExprLoc, VLEN, WDS, 'n')) + OMPBuilder.emitAArch64DeclareSimdFunction( + Fn, VLEN, ParamAttrs, State, 'n', NDS, OutputBecomesInput); + } } } FD = FD->getPreviousDecl(); diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h index c36d721b567e5..17ab480d783c6 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h @@ -14,6 +14,7 @@ #ifndef LLVM_FRONTEND_OPENMP_OMPIRBUILDER_H #define LLVM_FRONTEND_OPENMP_OMPIRBUILDER_H +#include "llvm/ADT/APSInt.h" #include "llvm/ADT/SetVector.h" #include "llvm/Frontend/Atomic/Atomic.h" #include "llvm/Frontend/OpenMP/OMPConstants.h" @@ -3973,6 +3974,76 @@ class OpenMPIRBuilder { LLVM_ABI InsertPointOrErrorTy createIteratorLoop( LocationDescription Loc, llvm::Value *TripCount, IteratorBodyGenTy BodyGen, llvm::StringRef Name = "iterator"); + + /// Kind of parameter in a function with 'declare simd' directive. + enum class DeclareSimdKindTy { + Linear, + LinearRef, + LinearUVal, + LinearVal, + Uniform, + Vector, + }; + + /// Attribute set of the `declare simd` parameter. + struct DeclareSimdAttrTy { + DeclareSimdKindTy Kind = DeclareSimdKindTy::Vector; + llvm::APSInt StrideOrArg; + llvm::APSInt Alignment; + bool HasVarStride = false; + }; + + enum class DeclareSimdBranch { + Undefined, + Inbranch, + Notinbranch, + }; + + /// Emit x86 vector-function ABI attributes for a `declare simd` function. + /// + /// Generates and attaches `_ZGV*` vector function ABI attributes to \p Fn + /// following the x86 vector ABI used by OpenMP `declare simd`. For each + /// supported ISA (SSE, AVX, AVX2, AVX512) and masking variant, this + /// constructs the appropriate mangled vector-function name and adds it as a + /// function attribute. + /// + /// \param Fn The scalar function to which vector-function attributes + /// are attached. + /// \param NumElements Number of elements used to derive the vector length + /// when + /// \p VLENVal is not specified. + /// \param VLENVal User provided vector length. + /// \param ParamAttrs Array of attribute set of the `declare simd` parameter. + /// \param Branch `undefined`, `inbranch` or `notinbranch` clause. + LLVM_ABI void emitX86DeclareSimdFunction( + llvm::Function *Fn, unsigned NumElements, const llvm::APSInt &VLENVal, + llvm::ArrayRef<DeclareSimdAttrTy> ParamAttrs, DeclareSimdBranch Branch); + + /// Emit AArch64 vector-function ABI attributes for a `declare simd` function. + /// + /// Generates and attaches `_ZGV*` vector function ABI attributes to \p Fn + /// following the AArch64 vector-function ABI. The emitted names depend on the + /// selected ISA, user-specified vector length, parameter attribute mangling, + /// and the declare simd branch clause. + /// + /// \param Fn The scalar function to which vector-function + /// attributes are attached. + /// \param VLENVal User provided vector length. + /// \param ParamAttrs Array of attribute set of the `declare simd` + /// parameter. + /// \param Branch `undefined`, `inbranch` or `notinbranch` + /// clause. + /// \param ISA `'n'` for Advanced SIMD or `'s'` for SVE. + /// \param NarrowestDataSize Narrowest data size in bits used to infer the + /// default vector length when \p VLENVal is + /// absent. + /// \param OutputBecomesInput Whether result values are represented as input + /// parameters in the emitted vector-function ABI + /// name. + LLVM_ABI void emitAArch64DeclareSimdFunction( + llvm::Function *Fn, unsigned VLENVal, + llvm::ArrayRef<DeclareSimdAttrTy> ParamAttrs, DeclareSimdBranch Branch, + char ISA, unsigned NarrowestDataSize, bool OutputBecomesInput); }; /// Class to represented the control flow structure of an OpenMP canonical loop. diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp index 85ecec046cfdb..8148e113195cc 100644 --- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp @@ -11647,6 +11647,220 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createIteratorLoop( return InsertPointTy{ContBB, ContBB->begin()}; } +/// Mangle the parameter part of the vector function name according to +/// their OpenMP classification. The mangling function is defined in +/// section 4.5 of the AAVFABI(2021Q1). +static std::string mangleVectorParameters( + ArrayRef<llvm::OpenMPIRBuilder::DeclareSimdAttrTy> ParamAttrs) { + SmallString<256> Buffer; + llvm::raw_svector_ostream Out(Buffer); + for (const auto &ParamAttr : ParamAttrs) { + switch (ParamAttr.Kind) { + case llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear: + Out << 'l'; + break; + case llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef: + Out << 'R'; + break; + case llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal: + Out << 'U'; + break; + case llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal: + Out << 'L'; + break; + case llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform: + Out << 'u'; + break; + case llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector: + Out << 'v'; + break; + } + if (ParamAttr.HasVarStride) + Out << "s" << ParamAttr.StrideOrArg; + else if (ParamAttr.Kind == + llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear || + ParamAttr.Kind == + llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef || + ParamAttr.Kind == + llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal || + ParamAttr.Kind == + llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal) { + // Don't print the step value if it is not present or if it is + // equal to 1. + if (ParamAttr.StrideOrArg < 0) + Out << 'n' << -ParamAttr.StrideOrArg; + else if (ParamAttr.StrideOrArg != 1) + Out << ParamAttr.StrideOrArg; + } + + if (!!ParamAttr.Alignment) + Out << 'a' << ParamAttr.Alignment; + } + + return std::string(Out.str()); +} + +void OpenMPIRBuilder::emitX86DeclareSimdFunction( + llvm::Function *Fn, unsigned NumElts, const llvm::APSInt &VLENVal, + llvm::ArrayRef<DeclareSimdAttrTy> ParamAttrs, DeclareSimdBranch Branch) { + struct ISADataTy { + char ISA; + unsigned VecRegSize; + }; + ISADataTy ISAData[] = { + {'b', 128}, // SSE + {'c', 256}, // AVX + {'d', 256}, // AVX2 + {'e', 512}, // AVX512 + }; + llvm::SmallVector<char, 2> Masked; + switch (Branch) { + case DeclareSimdBranch::Undefined: + Masked.push_back('N'); + Masked.push_back('M'); + break; + case DeclareSimdBranch::Notinbranch: + Masked.push_back('N'); + break; + case DeclareSimdBranch::Inbranch: + Masked.push_back('M'); + break; + } + for (char Mask : Masked) { + for (const ISADataTy &Data : ISAData) { + llvm::SmallString<256> Buffer; + llvm::raw_svector_ostream Out(Buffer); + Out << "_ZGV" << Data.ISA << Mask; + if (!VLENVal) { + assert(NumElts && "Non-zero simdlen/cdtsize expected"); + Out << llvm::APSInt::getUnsigned(Data.VecRegSize / NumElts); + } else { + Out << VLENVal; + } + Out << mangleVectorParameters(ParamAttrs); + Out << '_' << Fn->getName(); + Fn->addFnAttr(Out.str()); + } + } +} + +// Function used to add the attribute. The parameter `VLEN` is templated to +// allow the use of `x` when targeting scalable functions for SVE. +template <typename T> +static void addAArch64VectorName(T VLEN, StringRef LMask, StringRef Prefix, + char ISA, StringRef ParSeq, + StringRef MangledName, bool OutputBecomesInput, + llvm::Function *Fn) { + SmallString<256> Buffer; + llvm::raw_svector_ostream Out(Buffer); + Out << Prefix << ISA << LMask << VLEN; + if (OutputBecomesInput) + Out << 'v'; + Out << ParSeq << '_' << MangledName; + Fn->addFnAttr(Out.str()); +} + +// Helper function to generate the Advanced SIMD names depending on the value +// of the NDS when simdlen is not present. +static void addAArch64AdvSIMDNDSNames(unsigned NDS, StringRef Mask, + StringRef Prefix, char ISA, + StringRef ParSeq, StringRef MangledName, + bool OutputBecomesInput, + llvm::Function *Fn) { + switch (NDS) { + case 8: + addAArch64VectorName(8, Mask, Prefix, ISA, ParSeq, MangledName, + OutputBecomesInput, Fn); + addAArch64VectorName(16, Mask, Prefix, ISA, ParSeq, MangledName, + OutputBecomesInput, Fn); + break; + case 16: + addAArch64VectorName(4, Mask, Prefix, ISA, ParSeq, MangledName, + OutputBecomesInput, Fn); + addAArch64VectorName(8, Mask, Prefix, ISA, ParSeq, MangledName, + OutputBecomesInput, Fn); + break; + case 32: + addAArch64VectorName(2, Mask, Prefix, ISA, ParSeq, MangledName, + OutputBecomesInput, Fn); + addAArch64VectorName(4, Mask, Prefix, ISA, ParSeq, MangledName, + OutputBecomesInput, Fn); + break; + case 64: + case 128: + addAArch64VectorName(2, Mask, Prefix, ISA, ParSeq, MangledName, + OutputBecomesInput, Fn); + break; + default: + llvm_unreachable("Scalar type is too wide."); + } +} + +/// Emit vector function attributes for AArch64, as defined in the AAVFABI. +void OpenMPIRBuilder::emitAArch64DeclareSimdFunction( + llvm::Function *Fn, unsigned UserVLEN, + llvm::ArrayRef<DeclareSimdAttrTy> ParamAttrs, DeclareSimdBranch Branch, + char ISA, unsigned NarrowestDataSize, bool OutputBecomesInput) { + assert((ISA == 'n' || ISA == 's') && "Expected ISA either 's' or 'n'."); + + // Sort out parameter sequence. + const std::string ParSeq = mangleVectorParameters(ParamAttrs); + StringRef Prefix = "_ZGV"; + StringRef MangledName = Fn->getName(); + + // Generate simdlen from user input (if any). + if (UserVLEN) { + if (ISA == 's') { + // SVE generates only a masked function. + addAArch64VectorName(UserVLEN, "M", Prefix, ISA, ParSeq, MangledName, + OutputBecomesInput, Fn); + return; + } + + switch (Branch) { + case DeclareSimdBranch::Undefined: + addAArch64VectorName(UserVLEN, "N", Prefix, ISA, ParSeq, MangledName, + OutputBecomesInput, Fn); + addAArch64VectorName(UserVLEN, "M", Prefix, ISA, ParSeq, MangledName, + OutputBecomesInput, Fn); + break; + case DeclareSimdBranch::Inbranch: + addAArch64VectorName(UserVLEN, "M", Prefix, ISA, ParSeq, MangledName, + OutputBecomesInput, Fn); + break; + case DeclareSimdBranch::Notinbranch: + addAArch64VectorName(UserVLEN, "N", Prefix, ISA, ParSeq, MangledName, + OutputBecomesInput, Fn); + break; + } + return; + } + + if (ISA == 's') { + // SVE, section 3.4.1, item 1. + addAArch64VectorName("x", "M", Prefix, ISA, ParSeq, MangledName, + OutputBecomesInput, Fn); + return; + } + + switch (Branch) { + case DeclareSimdBranch::Undefined: + addAArch64AdvSIMDNDSNames(NarrowestDataSize, "N", Prefix, ISA, ParSeq, + MangledName, OutputBecomesInput, Fn); + addAArch64AdvSIMDNDSNames(NarrowestDataSize, "M", Prefix, ISA, ParSeq, + MangledName, OutputBecomesInput, Fn); + break; + case DeclareSimdBranch::Inbranch: + addAArch64AdvSIMDNDSNames(NarrowestDataSize, "M", Prefix, ISA, ParSeq, + MangledName, OutputBecomesInput, Fn); + break; + case DeclareSimdBranch::Notinbranch: + addAArch64AdvSIMDNDSNames(NarrowestDataSize, "N", Prefix, ISA, ParSeq, + MangledName, OutputBecomesInput, Fn); + break; + } +} + //===----------------------------------------------------------------------===// // OffloadEntriesInfoManager //===----------------------------------------------------------------------===// _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
