[PATCH] D88952: [clang-format][tests] Fix MacroExpander lexer not parsing C++ keywords

[Alexander Richardson via Phabricator via cfe-commits]

arichardson created this revision.
arichardson added reviewers: klimek, sammccall.
Herald added a project: clang.
Herald added a subscriber: cfe-commits.
arichardson requested review of this revision.

While debugging a different clang-format failure, I tried to reuse the
MacroExpander lexer, but was surprised to see that it marks all C++
keywords (e.g. const, decltype) as being of type identifier. After stepping
through the ::format() code, I noticed that the difference between these
two is that the identifier table was not being initialized based on the
FormatStyle, so only basic tokens such as tok::semi, tok::plus, etc. were
being handled.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D88952

Files:
  clang/unittests/Format/MacroExpanderTest.cpp
  clang/unittests/Format/TestLexer.h


Index: clang/unittests/Format/TestLexer.h
===================================================================
--- clang/unittests/Format/TestLexer.h
+++ clang/unittests/Format/TestLexer.h
@@ -55,7 +55,9 @@
 
 class TestLexer {
 public:
-  TestLexer() : SourceMgr("test.cpp", "") {}
+  TestLexer(FormatStyle Style = getLLVMStyle())
+      : Style(Style), SourceMgr("test.cpp", ""),
+        IdentTable(getFormattingLangOpts(Style)) {}
 
   TokenList lex(llvm::StringRef Code) {
     Buffers.push_back(
@@ -74,7 +76,7 @@
     return Result[0];
   }
 
-  FormatStyle Style = getLLVMStyle();
+  FormatStyle Style;
   encoding::Encoding Encoding = encoding::Encoding_UTF8;
   std::vector<std::unique_ptr<llvm::MemoryBuffer>> Buffers;
   clang::SourceManagerForFile SourceMgr;
Index: clang/unittests/Format/MacroExpanderTest.cpp
===================================================================
--- clang/unittests/Format/MacroExpanderTest.cpp
+++ clang/unittests/Format/MacroExpanderTest.cpp
@@ -182,6 +182,22 @@
   EXPECT_ATTRIBUTES(Result, Attributes);
 }
 
+TEST_F(MacroExpanderTest, UnderstandsCppTokens) {
+  auto Macros = create({"A(T,name)=T name = 0;"});
+  auto *A = Lex.id("A");
+  auto Args = lexArgs({"const int", "x"});
+  auto Result = uneof(Macros->expand(A, Args));
+  std::vector<MacroAttributes> Attributes = {
+      {tok::kw_const, MR_ExpandedArg, 1, 0, {A}},
+      {tok::kw_int, MR_ExpandedArg, 0, 0, {A}},
+      {tok::identifier, MR_ExpandedArg, 0, 0, {A}},
+      {tok::equal, MR_Hidden, 0, 0, {A}},
+      {tok::numeric_constant, MR_Hidden, 0, 0, {A}},
+      {tok::semi, MR_Hidden, 0, 1, {A}},
+  };
+  EXPECT_ATTRIBUTES(Result, Attributes);
+}
+
 } // namespace
 } // namespace format
 } // namespace clang



Index: clang/unittests/Format/TestLexer.h
===================================================================
--- clang/unittests/Format/TestLexer.h
+++ clang/unittests/Format/TestLexer.h
@@ -55,7 +55,9 @@
 
 class TestLexer {
 public:
-  TestLexer() : SourceMgr("test.cpp", "") {}
+  TestLexer(FormatStyle Style = getLLVMStyle())
+      : Style(Style), SourceMgr("test.cpp", ""),
+        IdentTable(getFormattingLangOpts(Style)) {}
 
   TokenList lex(llvm::StringRef Code) {
     Buffers.push_back(
@@ -74,7 +76,7 @@
     return Result[0];
   }
 
-  FormatStyle Style = getLLVMStyle();
+  FormatStyle Style;
   encoding::Encoding Encoding = encoding::Encoding_UTF8;
   std::vector<std::unique_ptr<llvm::MemoryBuffer>> Buffers;
   clang::SourceManagerForFile SourceMgr;
Index: clang/unittests/Format/MacroExpanderTest.cpp
===================================================================
--- clang/unittests/Format/MacroExpanderTest.cpp
+++ clang/unittests/Format/MacroExpanderTest.cpp
@@ -182,6 +182,22 @@
   EXPECT_ATTRIBUTES(Result, Attributes);
 }
 
+TEST_F(MacroExpanderTest, UnderstandsCppTokens) {
+  auto Macros = create({"A(T,name)=T name = 0;"});
+  auto *A = Lex.id("A");
+  auto Args = lexArgs({"const int", "x"});
+  auto Result = uneof(Macros->expand(A, Args));
+  std::vector<MacroAttributes> Attributes = {
+      {tok::kw_const, MR_ExpandedArg, 1, 0, {A}},
+      {tok::kw_int, MR_ExpandedArg, 0, 0, {A}},
+      {tok::identifier, MR_ExpandedArg, 0, 0, {A}},
+      {tok::equal, MR_Hidden, 0, 0, {A}},
+      {tok::numeric_constant, MR_Hidden, 0, 0, {A}},
+      {tok::semi, MR_Hidden, 0, 1, {A}},
+  };
+  EXPECT_ATTRIBUTES(Result, Attributes);
+}
+
 } // namespace
 } // namespace format
 } // namespace clang

_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits