hokein updated this revision to Diff 413446.
hokein marked an inline comment as done.
hokein added a comment.
address comment.
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D121092/new/
https://reviews.llvm.org/D121092
Files:
clang/include/clang/Tooling/Syntax/Pseudo/Token.h
clang/lib/Tooling/Syntax/Pseudo/Token.cpp
clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp
Index: clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp
===================================================================
--- clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp
+++ clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp
@@ -172,6 +172,23 @@
}));
}
+TEST(TokenTest, DropComments) {
+ LangOptions Opts;
+ std::string Code = R"cpp(
+ // comment
+ int /*abc*/;
+)cpp";
+ TokenStream Raw = cook(lex(Code, Opts), Opts);
+ TokenStream Stripped = stripComments(Raw);
+ EXPECT_THAT(Raw.tokens(),
+ ElementsAreArray(
+ {token("// comment", tok::comment), token("int",
tok::kw_int),
+ token("/*abc*/", tok::comment), token(";", tok::semi)}));
+
+ EXPECT_THAT(Stripped.tokens(), ElementsAreArray({token("int", tok::kw_int),
+ token(";", tok::semi)}));
+}
+
} // namespace
} // namespace pseudo
} // namespace syntax
Index: clang/lib/Tooling/Syntax/Pseudo/Token.cpp
===================================================================
--- clang/lib/Tooling/Syntax/Pseudo/Token.cpp
+++ clang/lib/Tooling/Syntax/Pseudo/Token.cpp
@@ -93,6 +93,17 @@
OS << '\n';
}
+TokenStream stripComments(const TokenStream &Input) {
+ TokenStream Out;
+ for (const Token &T : Input.tokens()) {
+ if (T.Kind == tok::comment)
+ continue;
+ Out.push(T);
+ }
+ Out.finalize();
+ return Out;
+}
+
} // namespace pseudo
} // namespace syntax
} // namespace clang
Index: clang/include/clang/Tooling/Syntax/Pseudo/Token.h
===================================================================
--- clang/include/clang/Tooling/Syntax/Pseudo/Token.h
+++ clang/include/clang/Tooling/Syntax/Pseudo/Token.h
@@ -195,6 +195,9 @@
/// (And having cooked token kinds in PP-disabled sections is useful for us).
TokenStream cook(const TokenStream &, const clang::LangOptions &);
+/// Drops comment tokens.
+TokenStream stripComments(const TokenStream &);
+
} // namespace pseudo
} // namespace syntax
} // namespace clang
Index: clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp
===================================================================
--- clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp
+++ clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp
@@ -172,6 +172,23 @@
}));
}
+TEST(TokenTest, DropComments) {
+ LangOptions Opts;
+ std::string Code = R"cpp(
+ // comment
+ int /*abc*/;
+)cpp";
+ TokenStream Raw = cook(lex(Code, Opts), Opts);
+ TokenStream Stripped = stripComments(Raw);
+ EXPECT_THAT(Raw.tokens(),
+ ElementsAreArray(
+ {token("// comment", tok::comment), token("int", tok::kw_int),
+ token("/*abc*/", tok::comment), token(";", tok::semi)}));
+
+ EXPECT_THAT(Stripped.tokens(), ElementsAreArray({token("int", tok::kw_int),
+ token(";", tok::semi)}));
+}
+
} // namespace
} // namespace pseudo
} // namespace syntax
Index: clang/lib/Tooling/Syntax/Pseudo/Token.cpp
===================================================================
--- clang/lib/Tooling/Syntax/Pseudo/Token.cpp
+++ clang/lib/Tooling/Syntax/Pseudo/Token.cpp
@@ -93,6 +93,17 @@
OS << '\n';
}
+TokenStream stripComments(const TokenStream &Input) {
+ TokenStream Out;
+ for (const Token &T : Input.tokens()) {
+ if (T.Kind == tok::comment)
+ continue;
+ Out.push(T);
+ }
+ Out.finalize();
+ return Out;
+}
+
} // namespace pseudo
} // namespace syntax
} // namespace clang
Index: clang/include/clang/Tooling/Syntax/Pseudo/Token.h
===================================================================
--- clang/include/clang/Tooling/Syntax/Pseudo/Token.h
+++ clang/include/clang/Tooling/Syntax/Pseudo/Token.h
@@ -195,6 +195,9 @@
/// (And having cooked token kinds in PP-disabled sections is useful for us).
TokenStream cook(const TokenStream &, const clang::LangOptions &);
+/// Drops comment tokens.
+TokenStream stripComments(const TokenStream &);
+
} // namespace pseudo
} // namespace syntax
} // namespace clang
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
