https://github.com/kasuga-fj created
https://github.com/llvm/llvm-project/pull/112501
Previously `#pragma clang loop pipeline` only accepted `disable`. This patch
adds `enable` as a valid argument for this pragma. This allows Software
Pipelining optimization to be applied to some loops instead of all loops.
This is clang part of the fix.
>From 04f0f22178272dbf2ebe8a74569245f97a2f644b Mon Sep 17 00:00:00 2001
From: Ryotaro Kasuga <kasuga.ryot...@fujitsu.com>
Date: Thu, 10 Oct 2024 09:08:00 +0000
Subject: [PATCH] [clang] Support `#pragma clang loop pipeline(enable)`
Previously `#pragma clang loop pipeline` only accepted `disable`. This
patch adds `enable` as a valid argument for this pragma. This allows
Software Pipelining optimization to be applied to some loops instead of
all loops.
This is clang part of the fix.
---
clang/include/clang/Basic/Attr.td | 6 ++--
clang/include/clang/Basic/AttrDocs.td | 12 ++++++-
.../clang/Basic/DiagnosticParseKinds.td | 2 --
clang/lib/CodeGen/CGLoopInfo.cpp | 36 ++++++++++++-------
clang/lib/CodeGen/CGLoopInfo.h | 11 +++---
clang/lib/Parse/ParsePragma.cpp | 34 ++++++++----------
clang/lib/Sema/SemaStmtAttr.cpp | 8 ++---
clang/test/CodeGenCXX/pragma-pipeline.cpp | 12 +++++++
clang/test/Parser/pragma-loop.cpp | 2 +-
clang/test/Parser/pragma-pipeline.cpp | 8 +++--
10 files changed, 82 insertions(+), 49 deletions(-)
diff --git a/clang/include/clang/Basic/Attr.td
b/clang/include/clang/Basic/Attr.td
index ec3d6e0079f630..3d5d5f6ca99f1e 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -4190,7 +4190,7 @@ def LoopHint : Attr {
/// unroll_and_jam: attempt to unroll and jam loop if State == Enable.
/// unroll_and_jam_count: unroll and jams loop 'Value' times.
/// distribute: attempt to distribute loop if State == Enable.
- /// pipeline: disable pipelining loop if State == Disable.
+ /// pipeline: enable pipelining loop if State == Enable.
/// pipeline_initiation_interval: create loop schedule with initiation
interval equal to 'Value'.
/// #pragma unroll <argument> directive
@@ -4210,7 +4210,7 @@ def LoopHint : Attr {
"vectorize_predicate"],
["Vectorize", "VectorizeWidth", "Interleave",
"InterleaveCount",
"Unroll", "UnrollCount", "UnrollAndJam",
"UnrollAndJamCount",
- "PipelineDisabled", "PipelineInitiationInterval",
"Distribute",
+ "Pipeline", "PipelineInitiationInterval",
"Distribute",
"VectorizePredicate"]>,
EnumArgument<"State", "LoopHintState", /*is_string=*/false,
["enable", "disable", "numeric", "fixed_width",
@@ -4230,7 +4230,7 @@ def LoopHint : Attr {
case UnrollCount: return "unroll_count";
case UnrollAndJam: return "unroll_and_jam";
case UnrollAndJamCount: return "unroll_and_jam_count";
- case PipelineDisabled: return "pipeline";
+ case Pipeline: return "pipeline";
case PipelineInitiationInterval: return "pipeline_initiation_interval";
case Distribute: return "distribute";
case VectorizePredicate: return "vectorize_predicate";
diff --git a/clang/include/clang/Basic/AttrDocs.td
b/clang/include/clang/Basic/AttrDocs.td
index b1512e22ee2dd4..e2591c7be7905a 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -3968,8 +3968,18 @@ def PipelineHintDocs : Documentation {
placed immediately before a for, while, do-while, or a C++11 range-based for
loop.
+ Using ``#pragma clang loop pipeline(enable)`` applies the software pipelining
+ optimization if possible:
+
+ .. code-block:: c++
+
+ #pragma clang loop pipeline(enable)
+ for (...) {
+ ...
+ }
+
Using ``#pragma clang loop pipeline(disable)`` avoids the software pipelining
- optimization. The disable state can only be specified:
+ optimization:
.. code-block:: c++
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td
b/clang/include/clang/Basic/DiagnosticParseKinds.td
index 78510e61a639fa..1bf82923aa26bd 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -1676,8 +1676,6 @@ def err_pragma_fp_invalid_argument : Error<
def err_pragma_invalid_keyword : Error<
"invalid argument; expected 'enable'%select{|, 'full'}0%select{|,
'assume_safety'}1 or 'disable'">;
-def err_pragma_pipeline_invalid_keyword : Error<
- "invalid argument; expected 'disable'">;
// API notes.
def err_type_unparsed : Error<"unparsed tokens following type">;
diff --git a/clang/lib/CodeGen/CGLoopInfo.cpp b/clang/lib/CodeGen/CGLoopInfo.cpp
index 6b886bd6b6d2cf..04b229da2013e3 100644
--- a/clang/lib/CodeGen/CGLoopInfo.cpp
+++ b/clang/lib/CodeGen/CGLoopInfo.cpp
@@ -39,9 +39,10 @@ MDNode *LoopInfo::createPipeliningMetadata(const
LoopAttributes &Attrs,
LLVMContext &Ctx = Header->getContext();
std::optional<bool> Enabled;
- if (Attrs.PipelineDisabled)
+ if (Attrs.Pipeline == LoopAttributes::Disable)
Enabled = false;
- else if (Attrs.PipelineInitiationInterval != 0)
+ else if (Attrs.Pipeline == LoopAttributes::Enable ||
+ Attrs.PipelineInitiationInterval != 0)
Enabled = true;
if (Enabled != true) {
@@ -69,6 +70,11 @@ MDNode *LoopInfo::createPipeliningMetadata(const
LoopAttributes &Attrs,
Args.push_back(MDNode::get(Ctx, Vals));
}
+ if (Attrs.Pipeline == LoopAttributes::Enable) {
+ Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.pipeline.enable")};
+ Args.push_back(MDNode::get(Ctx, Vals));
+ }
+
// No follow-up: This is the last transformation.
MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
@@ -460,8 +466,9 @@ LoopAttributes::LoopAttributes(bool IsParallel)
VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
VectorizeScalable(LoopAttributes::Unspecified), InterleaveCount(0),
UnrollCount(0), UnrollAndJamCount(0),
- DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
- PipelineInitiationInterval(0), CodeAlign(0), MustProgress(false) {}
+ DistributeEnable(LoopAttributes::Unspecified),
+ Pipeline(LoopAttributes::Unspecified), PipelineInitiationInterval(0),
+ CodeAlign(0), MustProgress(false) {}
void LoopAttributes::clear() {
IsParallel = false;
@@ -475,7 +482,7 @@ void LoopAttributes::clear() {
UnrollAndJamEnable = LoopAttributes::Unspecified;
VectorizePredicateEnable = LoopAttributes::Unspecified;
DistributeEnable = LoopAttributes::Unspecified;
- PipelineDisabled = false;
+ Pipeline = LoopAttributes::Unspecified;
PipelineInitiationInterval = 0;
CodeAlign = 0;
MustProgress = false;
@@ -496,7 +503,8 @@ LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes
&Attrs,
if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
Attrs.VectorizeScalable == LoopAttributes::Unspecified &&
Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
- Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
+ Attrs.UnrollAndJamCount == 0 &&
+ Attrs.Pipeline == LoopAttributes::Unspecified &&
Attrs.PipelineInitiationInterval == 0 &&
Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
@@ -552,7 +560,7 @@ void LoopInfo::finish() {
AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
AfterJam.UnrollCount = Attrs.UnrollCount;
- AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
+ AfterJam.Pipeline = Attrs.Pipeline;
AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
// If this loop is subject of an unroll-and-jam by the parent loop, and has
@@ -680,8 +688,8 @@ void LoopInfoStack::push(BasicBlock *Header,
clang::ASTContext &Ctx,
case LoopHintAttr::Distribute:
setDistributeState(false);
break;
- case LoopHintAttr::PipelineDisabled:
- setPipelineDisabled(true);
+ case LoopHintAttr::Pipeline:
+ setPipelineEnable(false);
break;
case LoopHintAttr::UnrollCount:
case LoopHintAttr::UnrollAndJamCount:
@@ -710,11 +718,13 @@ void LoopInfoStack::push(BasicBlock *Header,
clang::ASTContext &Ctx,
case LoopHintAttr::Distribute:
setDistributeState(true);
break;
+ case LoopHintAttr::Pipeline:
+ setPipelineEnable(true);
+ break;
case LoopHintAttr::UnrollCount:
case LoopHintAttr::UnrollAndJamCount:
case LoopHintAttr::VectorizeWidth:
case LoopHintAttr::InterleaveCount:
- case LoopHintAttr::PipelineDisabled:
case LoopHintAttr::PipelineInitiationInterval:
llvm_unreachable("Options cannot enabled.");
break;
@@ -736,7 +746,7 @@ void LoopInfoStack::push(BasicBlock *Header,
clang::ASTContext &Ctx,
case LoopHintAttr::VectorizeWidth:
case LoopHintAttr::InterleaveCount:
case LoopHintAttr::Distribute:
- case LoopHintAttr::PipelineDisabled:
+ case LoopHintAttr::Pipeline:
case LoopHintAttr::PipelineInitiationInterval:
llvm_unreachable("Options cannot be used to assume mem safety.");
break;
@@ -757,7 +767,7 @@ void LoopInfoStack::push(BasicBlock *Header,
clang::ASTContext &Ctx,
case LoopHintAttr::VectorizeWidth:
case LoopHintAttr::InterleaveCount:
case LoopHintAttr::Distribute:
- case LoopHintAttr::PipelineDisabled:
+ case LoopHintAttr::Pipeline:
case LoopHintAttr::PipelineInitiationInterval:
case LoopHintAttr::VectorizePredicate:
llvm_unreachable("Options cannot be used with 'full' hint.");
@@ -800,7 +810,7 @@ void LoopInfoStack::push(BasicBlock *Header,
clang::ASTContext &Ctx,
case LoopHintAttr::VectorizeWidth:
case LoopHintAttr::Interleave:
case LoopHintAttr::Distribute:
- case LoopHintAttr::PipelineDisabled:
+ case LoopHintAttr::Pipeline:
llvm_unreachable("Options cannot be assigned a value.");
break;
}
diff --git a/clang/lib/CodeGen/CGLoopInfo.h b/clang/lib/CodeGen/CGLoopInfo.h
index 0fe33b28913063..58d4ea6c45d2cb 100644
--- a/clang/lib/CodeGen/CGLoopInfo.h
+++ b/clang/lib/CodeGen/CGLoopInfo.h
@@ -73,8 +73,8 @@ struct LoopAttributes {
/// Value for llvm.loop.distribute.enable metadata.
LVEnableState DistributeEnable;
- /// Value for llvm.loop.pipeline.disable metadata.
- bool PipelineDisabled;
+ /// Value for llvm.loop.pipeline metadata.
+ LVEnableState Pipeline;
/// Value for llvm.loop.pipeline.iicount metadata.
unsigned PipelineInitiationInterval;
@@ -281,8 +281,11 @@ class LoopInfoStack {
/// \brief Set the unroll count for the next loop pushed.
void setUnrollAndJamCount(unsigned C) { StagedAttrs.UnrollAndJamCount = C; }
- /// Set the pipeline disabled state.
- void setPipelineDisabled(bool S) { StagedAttrs.PipelineDisabled = S; }
+ /// Set the next pushed loop as a pipeline candidate.
+ void setPipelineEnable(bool Enable = true) {
+ StagedAttrs.Pipeline =
+ Enable ? LoopAttributes::Enable : LoopAttributes::Disable;
+ }
/// Set the pipeline initiation interval.
void setPipelineInitiationInterval(unsigned C) {
diff --git a/clang/lib/Parse/ParsePragma.cpp b/clang/lib/Parse/ParsePragma.cpp
index 12fed448d477c0..7e9b80cd0b0a04 100644
--- a/clang/lib/Parse/ParsePragma.cpp
+++ b/clang/lib/Parse/ParsePragma.cpp
@@ -1454,24 +1454,24 @@ bool Parser::HandlePragmaLoopHint(LoopHint &Hint) {
bool OptionUnroll = false;
bool OptionUnrollAndJam = false;
bool OptionDistribute = false;
- bool OptionPipelineDisabled = false;
+ bool OptionPipeline = false;
bool StateOption = false;
if (OptionInfo) { // Pragma Unroll does not specify an option.
OptionUnroll = OptionInfo->isStr("unroll");
OptionUnrollAndJam = OptionInfo->isStr("unroll_and_jam");
OptionDistribute = OptionInfo->isStr("distribute");
- OptionPipelineDisabled = OptionInfo->isStr("pipeline");
+ OptionPipeline = OptionInfo->isStr("pipeline");
StateOption = llvm::StringSwitch<bool>(OptionInfo->getName())
.Case("vectorize", true)
.Case("interleave", true)
.Case("vectorize_predicate", true)
.Default(false) ||
OptionUnroll || OptionUnrollAndJam || OptionDistribute ||
- OptionPipelineDisabled;
+ OptionPipeline;
}
bool AssumeSafetyArg = !OptionUnroll && !OptionUnrollAndJam &&
- !OptionDistribute && !OptionPipelineDisabled;
+ !OptionDistribute && !OptionPipeline;
// Verify loop hint has an argument.
if (Toks[0].is(tok::eof)) {
ConsumeAnnotationToken();
@@ -1488,21 +1488,17 @@ bool Parser::HandlePragmaLoopHint(LoopHint &Hint) {
SourceLocation StateLoc = Toks[0].getLocation();
IdentifierInfo *StateInfo = Toks[0].getIdentifierInfo();
- bool Valid = StateInfo &&
- llvm::StringSwitch<bool>(StateInfo->getName())
- .Case("disable", true)
- .Case("enable", !OptionPipelineDisabled)
- .Case("full", OptionUnroll || OptionUnrollAndJam)
- .Case("assume_safety", AssumeSafetyArg)
- .Default(false);
+ bool Valid =
+ StateInfo && llvm::StringSwitch<bool>(StateInfo->getName())
+ .Case("disable", true)
+ .Case("enable", true)
+ .Case("full", OptionUnroll || OptionUnrollAndJam)
+ .Case("assume_safety", AssumeSafetyArg)
+ .Default(false);
if (!Valid) {
- if (OptionPipelineDisabled) {
- Diag(Toks[0].getLocation(), diag::err_pragma_pipeline_invalid_keyword);
- } else {
- Diag(Toks[0].getLocation(), diag::err_pragma_invalid_keyword)
- << /*FullKeyword=*/(OptionUnroll || OptionUnrollAndJam)
- << /*AssumeSafetyKeyword=*/AssumeSafetyArg;
- }
+ Diag(Toks[0].getLocation(), diag::err_pragma_invalid_keyword)
+ << /*FullKeyword=*/(OptionUnroll || OptionUnrollAndJam)
+ << /*AssumeSafetyKeyword=*/AssumeSafetyArg;
return false;
}
if (Toks.size() > 2)
@@ -3591,7 +3587,7 @@ static bool ParseLoopHintValue(Preprocessor &PP, Token
&Tok, Token PragmaName,
/// 'vectorize_width' '(' loop-hint-value ')'
/// 'interleave_count' '(' loop-hint-value ')'
/// 'unroll_count' '(' loop-hint-value ')'
-/// 'pipeline' '(' disable ')'
+/// 'pipeline' '(' loop-hint-keyword ')'
/// 'pipeline_initiation_interval' '(' loop-hint-value ')'
///
/// loop-hint-keyword:
diff --git a/clang/lib/Sema/SemaStmtAttr.cpp b/clang/lib/Sema/SemaStmtAttr.cpp
index f801455596fe6f..9d2989cd12e021 100644
--- a/clang/lib/Sema/SemaStmtAttr.cpp
+++ b/clang/lib/Sema/SemaStmtAttr.cpp
@@ -142,7 +142,7 @@ static Attr *handleLoopHintAttr(Sema &S, Stmt *St, const
ParsedAttr &A,
.Case("interleave_count", LoopHintAttr::InterleaveCount)
.Case("unroll", LoopHintAttr::Unroll)
.Case("unroll_count", LoopHintAttr::UnrollCount)
- .Case("pipeline", LoopHintAttr::PipelineDisabled)
+ .Case("pipeline", LoopHintAttr::Pipeline)
.Case("pipeline_initiation_interval",
LoopHintAttr::PipelineInitiationInterval)
.Case("distribute", LoopHintAttr::Distribute)
@@ -170,7 +170,7 @@ static Attr *handleLoopHintAttr(Sema &S, Stmt *St, const
ParsedAttr &A,
Option == LoopHintAttr::VectorizePredicate ||
Option == LoopHintAttr::Unroll ||
Option == LoopHintAttr::Distribute ||
- Option == LoopHintAttr::PipelineDisabled) {
+ Option == LoopHintAttr::Pipeline) {
assert(StateLoc && StateLoc->Ident && "Loop hint must have an argument");
if (StateLoc->Ident->isStr("disable"))
State = LoopHintAttr::Disable;
@@ -516,7 +516,7 @@ CheckForIncompatibleAttributes(Sema &S,
// Perform the check for duplicated 'distribute' hints.
Category = Distribute;
break;
- case LoopHintAttr::PipelineDisabled:
+ case LoopHintAttr::Pipeline:
case LoopHintAttr::PipelineInitiationInterval:
Category = Pipeline;
break;
@@ -532,7 +532,7 @@ CheckForIncompatibleAttributes(Sema &S,
Option == LoopHintAttr::Interleave || Option == LoopHintAttr::Unroll ||
Option == LoopHintAttr::UnrollAndJam ||
Option == LoopHintAttr::VectorizePredicate ||
- Option == LoopHintAttr::PipelineDisabled ||
+ Option == LoopHintAttr::Pipeline ||
Option == LoopHintAttr::Distribute) {
// Enable|Disable|AssumeSafety hint. For example, vectorize(enable).
PrevAttr = CategoryState.StateAttr;
diff --git a/clang/test/CodeGenCXX/pragma-pipeline.cpp
b/clang/test/CodeGenCXX/pragma-pipeline.cpp
index b7d2136745c7d9..f8948adf1a9d8a 100644
--- a/clang/test/CodeGenCXX/pragma-pipeline.cpp
+++ b/clang/test/CodeGenCXX/pragma-pipeline.cpp
@@ -36,6 +36,15 @@ void pipeline_disabled_on_nested_loop(int *List, int Length,
int Value) {
}
}
+void pipeline_enabled(int *List, int Length, int Value) {
+// CHECK-LABEL: define {{.*}} @_Z16pipeline_enabled
+#pragma clang loop pipeline(enable)
+ for (int i = 0; i < Length; i++) {
+ // CHECK: br label {{.*}}, !llvm.loop ![[LOOP_5:.*]]
+ List[i] = Value;
+ }
+}
+
// CHECK: ![[LOOP_1]] = distinct !{![[LOOP_1]], [[MP:![0-9]+]],
![[PIPELINE_DISABLE:.*]]}
// CHECK: ![[PIPELINE_DISABLE]] = !{!"llvm.loop.pipeline.disable", i1 true}
@@ -45,3 +54,6 @@ void pipeline_disabled_on_nested_loop(int *List, int Length,
int Value) {
// CHECK: ![[PIPELINE_II_10]] = !{!"llvm.loop.pipeline.initiationinterval",
i32 10}
// CHECK: ![[LOOP_4]] = distinct !{![[LOOP_4]], [[MP]], ![[PIPELINE_DISABLE]]}
+
+// CHECK: ![[LOOP_5]] = distinct !{![[LOOP_5]], [[MP]],
![[PIPELINE_ENABLE:.*]]}
+// CHECK: ![[PIPELINE_ENABLE]] = !{!"llvm.loop.pipeline.enable"}
diff --git a/clang/test/Parser/pragma-loop.cpp
b/clang/test/Parser/pragma-loop.cpp
index 4078210f96e1b9..bb068b1d6c22b9 100644
--- a/clang/test/Parser/pragma-loop.cpp
+++ b/clang/test/Parser/pragma-loop.cpp
@@ -325,7 +325,7 @@ void foo(int *List, int Length) {
List[i] = i;
}
-#pragma clang loop pipeline(disable, extra)
+#pragma clang loop pipeline(enable, extra)
/* expected-warning {{extra tokens at end of '#pragma clang loop pipeline' -
ignored}}*/ while (i-6 < Length) {
List[i] = i;
}
diff --git a/clang/test/Parser/pragma-pipeline.cpp
b/clang/test/Parser/pragma-pipeline.cpp
index e500d4d2d5fb25..7d7a8ebab12f07 100644
--- a/clang/test/Parser/pragma-pipeline.cpp
+++ b/clang/test/Parser/pragma-pipeline.cpp
@@ -6,6 +6,11 @@
void test(int *List, int Length, int Value) {
int i = 0;
+#pragma clang loop pipeline(enable)
+ for (int i = 0; i < Length; i++) {
+ List[i] = Value;
+ }
+
#pragma clang loop pipeline(disable)
for (int i = 0; i < Length; i++) {
List[i] = Value;
@@ -17,8 +22,7 @@ void test(int *List, int Length, int Value) {
}
/* expected-error {{expected ')'}} */ #pragma clang loop pipeline(disable
-/* expected-error {{invalid argument; expected 'disable'}} */ #pragma clang
loop pipeline(enable)
-/* expected-error {{invalid argument; expected 'disable'}} */ #pragma clang
loop pipeline(error)
+/* expected-error {{invalid argument; expected 'enable' or 'disable'}} */
#pragma clang loop pipeline(error)
/* expected-error {{expected '('}} */ #pragma clang loop pipeline disable
/* expected-error {{missing argument; expected an integer value}} */ #pragma
clang loop pipeline_initiation_interval()
/* expected-error {{use of undeclared identifier 'error'}} */ #pragma clang
loop pipeline_initiation_interval(error)
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
