hokein created this revision.
hokein added a reviewer: sammccall.
Herald added a project: All.
hokein requested review of this revision.
Herald added a subscriber: alextsao1999.
Herald added a project: clang.
Add a utility function to strip comments from a "raw" tokenstream. The
derived stream will be fed to the GLR parser (for early testing).
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D121092
Files:
clang/include/clang/Tooling/Syntax/Pseudo/Preprocess.h
clang/lib/Tooling/Syntax/Pseudo/Preprocess.cpp
clang/unittests/Tooling/Syntax/Pseudo/PreprocessTest.cpp
Index: clang/unittests/Tooling/Syntax/Pseudo/PreprocessTest.cpp
===================================================================
--- clang/unittests/Tooling/Syntax/Pseudo/PreprocessTest.cpp
+++ clang/unittests/Tooling/Syntax/Pseudo/PreprocessTest.cpp
@@ -23,11 +23,16 @@
using testing::_;
using testing::ElementsAre;
+using testing::ElementsAreArray;
using testing::Matcher;
using testing::Pair;
using testing::StrEq;
using Chunk = PPStructure::Chunk;
+MATCHER_P2(token, Text, Kind, "") {
+ return arg.Kind == Kind && arg.text() == Text;
+}
+
MATCHER_P2(tokensAre, TS, Tokens, "tokens are " + std::string(Tokens)) {
std::vector<llvm::StringRef> Texts;
for (const Token &Tok : TS.tokens(arg.Tokens))
@@ -146,6 +151,22 @@
EXPECT_EQ(0u, X.End.Tokens.size());
}
+TEST(Preprocessor, StripComments) {
+ LangOptions Opts;
+ std::string Code = R"cpp(
+ // comment
+ int;
+)cpp";
+ TokenStream Raw = cook(lex(Code, Opts), Opts);
+ TokenStream Stripped = stripComments(Raw);
+ EXPECT_THAT(Raw.tokens(), ElementsAreArray({token("// comment",
tok::comment),
+ token("int", tok::kw_int),
+ token(";", tok::semi)}));
+
+ EXPECT_THAT(Stripped.tokens(), ElementsAreArray({token("int", tok::kw_int),
+ token(";", tok::semi)}));
+}
+
} // namespace
} // namespace pseudo
} // namespace syntax
Index: clang/lib/Tooling/Syntax/Pseudo/Preprocess.cpp
===================================================================
--- clang/lib/Tooling/Syntax/Pseudo/Preprocess.cpp
+++ clang/lib/Tooling/Syntax/Pseudo/Preprocess.cpp
@@ -201,6 +201,17 @@
OSTREAM_DUMP(PPStructure::Code)
#undef OSTREAM_DUMP
+TokenStream stripComments(const TokenStream &Input) {
+ TokenStream Out;
+ for (const Token &T : Input.tokens()) {
+ if (T.Kind == tok::comment)
+ continue;
+ Out.push(T);
+ }
+ Out.finalize();
+ return Out;
+}
+
} // namespace pseudo
} // namespace syntax
} // namespace clang
Index: clang/include/clang/Tooling/Syntax/Pseudo/Preprocess.h
===================================================================
--- clang/include/clang/Tooling/Syntax/Pseudo/Preprocess.h
+++ clang/include/clang/Tooling/Syntax/Pseudo/Preprocess.h
@@ -141,6 +141,9 @@
llvm::Optional<Conditional> ConditionalVariant;
};
+/// Drops comment tokens.
+TokenStream stripComments(const TokenStream &);
+
} // namespace pseudo
} // namespace syntax
} // namespace clang
Index: clang/unittests/Tooling/Syntax/Pseudo/PreprocessTest.cpp
===================================================================
--- clang/unittests/Tooling/Syntax/Pseudo/PreprocessTest.cpp
+++ clang/unittests/Tooling/Syntax/Pseudo/PreprocessTest.cpp
@@ -23,11 +23,16 @@
using testing::_;
using testing::ElementsAre;
+using testing::ElementsAreArray;
using testing::Matcher;
using testing::Pair;
using testing::StrEq;
using Chunk = PPStructure::Chunk;
+MATCHER_P2(token, Text, Kind, "") {
+ return arg.Kind == Kind && arg.text() == Text;
+}
+
MATCHER_P2(tokensAre, TS, Tokens, "tokens are " + std::string(Tokens)) {
std::vector<llvm::StringRef> Texts;
for (const Token &Tok : TS.tokens(arg.Tokens))
@@ -146,6 +151,22 @@
EXPECT_EQ(0u, X.End.Tokens.size());
}
+TEST(Preprocessor, StripComments) {
+ LangOptions Opts;
+ std::string Code = R"cpp(
+ // comment
+ int;
+)cpp";
+ TokenStream Raw = cook(lex(Code, Opts), Opts);
+ TokenStream Stripped = stripComments(Raw);
+ EXPECT_THAT(Raw.tokens(), ElementsAreArray({token("// comment", tok::comment),
+ token("int", tok::kw_int),
+ token(";", tok::semi)}));
+
+ EXPECT_THAT(Stripped.tokens(), ElementsAreArray({token("int", tok::kw_int),
+ token(";", tok::semi)}));
+}
+
} // namespace
} // namespace pseudo
} // namespace syntax
Index: clang/lib/Tooling/Syntax/Pseudo/Preprocess.cpp
===================================================================
--- clang/lib/Tooling/Syntax/Pseudo/Preprocess.cpp
+++ clang/lib/Tooling/Syntax/Pseudo/Preprocess.cpp
@@ -201,6 +201,17 @@
OSTREAM_DUMP(PPStructure::Code)
#undef OSTREAM_DUMP
+TokenStream stripComments(const TokenStream &Input) {
+ TokenStream Out;
+ for (const Token &T : Input.tokens()) {
+ if (T.Kind == tok::comment)
+ continue;
+ Out.push(T);
+ }
+ Out.finalize();
+ return Out;
+}
+
} // namespace pseudo
} // namespace syntax
} // namespace clang
Index: clang/include/clang/Tooling/Syntax/Pseudo/Preprocess.h
===================================================================
--- clang/include/clang/Tooling/Syntax/Pseudo/Preprocess.h
+++ clang/include/clang/Tooling/Syntax/Pseudo/Preprocess.h
@@ -141,6 +141,9 @@
llvm::Optional<Conditional> ConditionalVariant;
};
+/// Drops comment tokens.
+TokenStream stripComments(const TokenStream &);
+
} // namespace pseudo
} // namespace syntax
} // namespace clang
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
