sammccall created this revision.
sammccall added a reviewer: hokein.
Herald added a project: All.
sammccall requested review of this revision.
Herald added subscribers: cfe-commits, alextsao1999.
Herald added a project: clang-tools-extra.
This includes only the taken branch of conditional sections.
The API allows for producing a stream for a particular PP branch, which
will be used later for the secondary GLR parses of not-taken branches.
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D123243
Files:
clang-tools-extra/pseudo/include/clang-pseudo/DirectiveTree.h
clang-tools-extra/pseudo/lib/DirectiveTree.cpp
clang-tools-extra/pseudo/test/preprocess.c
clang-tools-extra/pseudo/tool/ClangPseudo.cpp
clang-tools-extra/pseudo/unittests/DirectiveTreeTest.cpp
Index: clang-tools-extra/pseudo/unittests/DirectiveTreeTest.cpp
===================================================================
--- clang-tools-extra/pseudo/unittests/DirectiveTreeTest.cpp
+++ clang-tools-extra/pseudo/unittests/DirectiveTreeTest.cpp
@@ -27,14 +27,23 @@
using testing::StrEq;
using Chunk = DirectiveTree::Chunk;
-MATCHER_P2(tokensAre, TS, Tokens, "tokens are " + std::string(Tokens)) {
+// Matches text of a list of tokens against a string (joined with spaces).
+// e.g. EXPECT_THAT(Stream.tokens(), tokens("int main ( ) { }"));
+MATCHER_P(tokens, Tokens, "") {
std::vector<llvm::StringRef> Texts;
- for (const Token &Tok : TS.tokens(arg.Tokens))
+ for (const Token &Tok : arg)
Texts.push_back(Tok.text());
return Matcher<std::string>(StrEq(Tokens))
.MatchAndExplain(llvm::join(Texts, " "), result_listener);
}
+// Matches tokens covered a directive chunk (with a Tokens property) against a
+// string, similar to tokens() above.
+// e.g. EXPECT_THAT(SomeDirective, tokensAre(Stream, "# include < vector >"));
+MATCHER_P2(tokensAre, TS, Tokens, "tokens are " + std::string(Tokens)) {
+ return testing::Matches(tokens(Tokens))(TS.tokens(arg.Tokens));
+}
+
MATCHER_P(chunkKind, K, "") { return arg.kind() == K; }
TEST(DirectiveTree, Parse) {
@@ -301,6 +310,44 @@
}
}
+TEST(DirectiveTree, StripDirectives) {
+ LangOptions Opts;
+ std::string Code = R"cpp(
+ a a a
+ #warning AAA
+ b b b
+ #if 1
+ c c c
+ #warning BBB
+ #if 0
+ d d d
+ #warning CC
+ #else
+ e e e
+ #endif
+ f f f
+ #if 0
+ g g g
+ #endif
+ h h h
+ #else
+ i i i
+ #endif
+ j j j
+ )cpp";
+ TokenStream S = cook(lex(Code, Opts), Opts);
+
+ DirectiveTree Tree = DirectiveTree::parse(S);
+ chooseConditionalBranches(Tree, S);
+ EXPECT_THAT(Tree.stripDirectives(S).tokens(),
+ tokens("a a a b b b c c c e e e f f f h h h j j j"));
+
+ const DirectiveTree &Part =
+ ((const DirectiveTree::Conditional &)Tree.Chunks[3]).Branches[0].second;
+ EXPECT_THAT(Part.stripDirectives(S).tokens(),
+ tokens("c c c e e e f f f h h h"));
+}
+
} // namespace
} // namespace pseudo
} // namespace clang
Index: clang-tools-extra/pseudo/tool/ClangPseudo.cpp
===================================================================
--- clang-tools-extra/pseudo/tool/ClangPseudo.cpp
+++ clang-tools-extra/pseudo/tool/ClangPseudo.cpp
@@ -18,6 +18,7 @@
#include "llvm/Support/MemoryBuffer.h"
using clang::pseudo::Grammar;
+using clang::pseudo::TokenStream;
using llvm::cl::desc;
using llvm::cl::init;
using llvm::cl::opt;
@@ -35,6 +36,9 @@
static opt<bool>
PrintDirectiveTree("print-directive-tree",
desc("Print directive structure of source code"));
+static opt<bool>
+ Preprocess("preprocess",
+ desc("Strip directives and select conditional sections"));
static std::string readOrDie(llvm::StringRef Path) {
llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Text =
@@ -73,16 +77,23 @@
if (Source.getNumOccurrences()) {
std::string Text = readOrDie(Source);
clang::LangOptions LangOpts; // FIXME: use real options.
- auto Stream = clang::pseudo::lex(Text, LangOpts);
- auto Structure = clang::pseudo::DirectiveTree::parse(Stream);
- clang::pseudo::chooseConditionalBranches(Structure, Stream);
+ TokenStream Raw = clang::pseudo::lex(Text, LangOpts);
+ TokenStream *Stream = &Raw;
+ auto Structure = clang::pseudo::DirectiveTree::parse(*Stream);
+ clang::pseudo::chooseConditionalBranches(Structure, *Stream);
if (PrintDirectiveTree)
llvm::outs() << Structure;
+ llvm::Optional<TokenStream> Preprocessed;
+ if (Preprocess) {
+ Preprocessed = Structure.stripDirectives(*Stream);
+ Stream = Preprocessed.getPointer();
+ }
+
if (PrintSource)
- Stream.print(llvm::outs());
+ Stream->print(llvm::outs());
if (PrintTokens)
- llvm::outs() << Stream;
+ llvm::outs() << *Stream;
}
return 0;
Index: clang-tools-extra/pseudo/test/preprocess.c
===================================================================
--- /dev/null
+++ clang-tools-extra/pseudo/test/preprocess.c
@@ -0,0 +1,21 @@
+#include <stdio.h>
+int main() {
+#error This was inevitable...
+#if HELLO
+ printf("hello, world\n");
+ return 0;
+#else
+ abort();
+#endif
+}
+
+/* This comment gets lexed along with the input above! We just don't CHECK it.
+
+RUN: clang-pseudo -source %s -preprocess -print-source | FileCheck %s --strict-whitespace
+ CHECK: int main() {
+CHECK-NEXT: printf("hello, world\n");
+CHECK-NEXT: return 0;
+CHECK-NEXT: }
+
+*******************************************************************************/
+
Index: clang-tools-extra/pseudo/lib/DirectiveTree.cpp
===================================================================
--- clang-tools-extra/pseudo/lib/DirectiveTree.cpp
+++ clang-tools-extra/pseudo/lib/DirectiveTree.cpp
@@ -347,5 +347,53 @@
BranchChooser{Code}.choose(Tree);
}
+namespace {
+class Preprocessor {
+ const TokenStream &In;
+ TokenStream &Out;
+
+public:
+ Preprocessor(const TokenStream &In, TokenStream &Out) : In(In), Out(Out) {}
+ ~Preprocessor() { Out.finalize(); }
+
+ void walk(const DirectiveTree &T) {
+ for (const auto &C : T.Chunks)
+ walk(C);
+ }
+
+ void walk(const DirectiveTree::Chunk &C) {
+ switch (C.kind()) {
+ case DirectiveTree::Chunk::K_Code:
+ return walk((const DirectiveTree::Code &)C);
+ case DirectiveTree::Chunk::K_Directive:
+ return walk((const DirectiveTree::Directive &)C);
+ case DirectiveTree::Chunk::K_Conditional:
+ return walk((const DirectiveTree::Conditional &)C);
+ case DirectiveTree::Chunk::K_Empty:
+ break;
+ }
+ llvm_unreachable("bad chunk kind");
+ }
+
+ void walk(const DirectiveTree::Code &C) {
+ for (const auto &Tok : In.tokens(C.Tokens))
+ Out.push(Tok);
+ }
+
+ void walk(const DirectiveTree::Directive &) {}
+
+ void walk(const DirectiveTree::Conditional &C) {
+ if (C.Taken)
+ walk(C.Branches[*C.Taken].second);
+ }
+};
+} // namespace
+
+TokenStream DirectiveTree::stripDirectives(const TokenStream &In) const {
+ TokenStream Out;
+ Preprocessor(In, Out).walk(*this);
+ return Out;
+}
+
} // namespace pseudo
} // namespace clang
Index: clang-tools-extra/pseudo/include/clang-pseudo/DirectiveTree.h
===================================================================
--- clang-tools-extra/pseudo/include/clang-pseudo/DirectiveTree.h
+++ clang-tools-extra/pseudo/include/clang-pseudo/DirectiveTree.h
@@ -92,7 +92,11 @@
/// Extract preprocessor structure by examining the raw tokens.
static DirectiveTree parse(const TokenStream &);
- // FIXME: allow deriving a preprocessed stream
+ /// Produce a parseable token stream by stripping all directive tokens.
+ ///
+ /// Conditional sections are replaced by the taken branch, if any.
+ /// This tree must describe the provided token stream.
+ TokenStream stripDirectives(const TokenStream &) const;
};
llvm::raw_ostream &operator<<(llvm::raw_ostream &, const DirectiveTree &);
llvm::raw_ostream &operator<<(llvm::raw_ostream &, const DirectiveTree::Chunk &);
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
