This revision was landed with ongoing or failed builds.
This revision was automatically updated to reflect the committed changes.
Closed by commit rG3116d60494f2: [Lex] Introduce
Preprocessor::LexTokensUntilEOF() (authored by Hahnfeld).
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D158413/new/
https://reviews.llvm.org/D158413
Files:
clang/include/clang/Lex/Preprocessor.h
clang/lib/Lex/Preprocessor.cpp
clang/unittests/Analysis/MacroExpansionContextTest.cpp
clang/unittests/Basic/SourceManagerTest.cpp
clang/unittests/Lex/LexerTest.cpp
clang/unittests/Lex/ModuleDeclStateTest.cpp
clang/unittests/Lex/PPCallbacksTest.cpp
clang/unittests/Lex/PPConditionalDirectiveRecordTest.cpp
clang/unittests/Lex/PPDependencyDirectivesTest.cpp
clang/unittests/Lex/PPMemoryAllocationsTest.cpp
Index: clang/unittests/Lex/PPMemoryAllocationsTest.cpp
===================================================================
--- clang/unittests/Lex/PPMemoryAllocationsTest.cpp
+++ clang/unittests/Lex/PPMemoryAllocationsTest.cpp
@@ -75,12 +75,7 @@
PP.Initialize(*Target);
PP.EnterMainSourceFile();
- while (1) {
- Token tok;
- PP.Lex(tok);
- if (tok.is(tok::eof))
- break;
- }
+ PP.LexTokensUntilEOF();
size_t NumAllocated = PP.getPreprocessorAllocator().getBytesAllocated();
float BytesPerDefine = float(NumAllocated) / float(NumMacros);
Index: clang/unittests/Lex/PPDependencyDirectivesTest.cpp
===================================================================
--- clang/unittests/Lex/PPDependencyDirectivesTest.cpp
+++ clang/unittests/Lex/PPDependencyDirectivesTest.cpp
@@ -133,12 +133,7 @@
SmallVector<StringRef> IncludedFiles;
PP.addPPCallbacks(std::make_unique<IncludeCollector>(PP, IncludedFiles));
PP.EnterMainSourceFile();
- while (true) {
- Token tok;
- PP.Lex(tok);
- if (tok.is(tok::eof))
- break;
- }
+ PP.LexTokensUntilEOF();
SmallVector<std::string> IncludedFilesSlash;
for (StringRef IncludedFile : IncludedFiles)
Index: clang/unittests/Lex/PPConditionalDirectiveRecordTest.cpp
===================================================================
--- clang/unittests/Lex/PPConditionalDirectiveRecordTest.cpp
+++ clang/unittests/Lex/PPConditionalDirectiveRecordTest.cpp
@@ -87,13 +87,7 @@
PP.EnterMainSourceFile();
std::vector<Token> toks;
- while (1) {
- Token tok;
- PP.Lex(tok);
- if (tok.is(tok::eof))
- break;
- toks.push_back(tok);
- }
+ PP.LexTokensUntilEOF(&toks);
// Make sure we got the tokens that we expected.
ASSERT_EQ(10U, toks.size());
Index: clang/unittests/Lex/PPCallbacksTest.cpp
===================================================================
--- clang/unittests/Lex/PPCallbacksTest.cpp
+++ clang/unittests/Lex/PPCallbacksTest.cpp
@@ -229,13 +229,7 @@
// Lex source text.
PP.EnterMainSourceFile();
-
- while (true) {
- Token Tok;
- PP.Lex(Tok);
- if (Tok.is(tok::eof))
- break;
- }
+ PP.LexTokensUntilEOF();
// Callbacks have been executed at this point -- return filename range.
return Callbacks;
@@ -259,13 +253,7 @@
// Lex source text.
PP.EnterMainSourceFile();
-
- while (true) {
- Token Tok;
- PP.Lex(Tok);
- if (Tok.is(tok::eof))
- break;
- }
+ PP.LexTokensUntilEOF();
return Callbacks->Results;
}
@@ -290,12 +278,7 @@
// Lex source text.
PP.EnterMainSourceFile();
- while (true) {
- Token Tok;
- PP.Lex(Tok);
- if (Tok.is(tok::eof))
- break;
- }
+ PP.LexTokensUntilEOF();
return Callbacks->Marks;
}
@@ -334,12 +317,7 @@
// Lex source text.
PP.EnterMainSourceFile();
- while (true) {
- Token Tok;
- PP.Lex(Tok);
- if (Tok.is(tok::eof))
- break;
- }
+ PP.LexTokensUntilEOF();
PragmaOpenCLExtensionCallbacks::CallbackParameters RetVal = {
Callbacks->Name,
@@ -477,12 +455,7 @@
// Lex source text.
PP.EnterMainSourceFile();
- while (true) {
- Token Tok;
- PP.Lex(Tok);
- if (Tok.is(tok::eof))
- break;
- }
+ PP.LexTokensUntilEOF();
ASSERT_EQ(1u, Callbacks->NumCalls);
ASSERT_EQ(0u, DiagConsumer->getNumErrors());
Index: clang/unittests/Lex/ModuleDeclStateTest.cpp
===================================================================
--- clang/unittests/Lex/ModuleDeclStateTest.cpp
+++ clang/unittests/Lex/ModuleDeclStateTest.cpp
@@ -90,12 +90,7 @@
PP.addPPCallbacks(std::move(C));
PP.EnterMainSourceFile();
- while (1) {
- Token tok;
- PP.Lex(tok);
- if (tok.is(tok::eof))
- break;
- }
+ PP.LexTokensUntilEOF();
}
FileSystemOptions FileMgrOpts;
Index: clang/unittests/Lex/LexerTest.cpp
===================================================================
--- clang/unittests/Lex/LexerTest.cpp
+++ clang/unittests/Lex/LexerTest.cpp
@@ -74,13 +74,7 @@
PP = CreatePP(Source, ModLoader);
std::vector<Token> toks;
- while (1) {
- Token tok;
- PP->Lex(tok);
- if (tok.is(tok::eof))
- break;
- toks.push_back(tok);
- }
+ PP->LexTokensUntilEOF(&toks);
return toks;
}
@@ -628,12 +622,7 @@
TEST_F(LexerTest, CreatedFIDCountForPredefinedBuffer) {
TrivialModuleLoader ModLoader;
auto PP = CreatePP("", ModLoader);
- while (1) {
- Token tok;
- PP->Lex(tok);
- if (tok.is(tok::eof))
- break;
- }
+ PP->LexTokensUntilEOF();
EXPECT_EQ(SourceMgr.getNumCreatedFIDsForFileID(PP->getPredefinesFileID()),
1U);
}
Index: clang/unittests/Basic/SourceManagerTest.cpp
===================================================================
--- clang/unittests/Basic/SourceManagerTest.cpp
+++ clang/unittests/Basic/SourceManagerTest.cpp
@@ -138,13 +138,7 @@
PP.EnterMainSourceFile();
std::vector<Token> toks;
- while (1) {
- Token tok;
- PP.Lex(tok);
- if (tok.is(tok::eof))
- break;
- toks.push_back(tok);
- }
+ PP.LexTokensUntilEOF(&toks);
// Make sure we got the tokens that we expected.
ASSERT_EQ(3U, toks.size());
@@ -195,13 +189,7 @@
llvm::SmallString<8> Scratch;
std::vector<Token> toks;
- while (1) {
- Token tok;
- PP.Lex(tok);
- if (tok.is(tok::eof))
- break;
- toks.push_back(tok);
- }
+ PP.LexTokensUntilEOF(&toks);
// Make sure we got the tokens that we expected.
ASSERT_EQ(4U, toks.size()) << "a >> b c";
@@ -452,13 +440,7 @@
PP.EnterMainSourceFile();
std::vector<Token> toks;
- while (1) {
- Token tok;
- PP.Lex(tok);
- if (tok.is(tok::eof))
- break;
- toks.push_back(tok);
- }
+ PP.LexTokensUntilEOF(&toks);
// Make sure we got the tokens that we expected.
ASSERT_EQ(4U, toks.size());
@@ -574,13 +556,7 @@
PP.EnterMainSourceFile();
std::vector<Token> toks;
- while (1) {
- Token tok;
- PP.Lex(tok);
- if (tok.is(tok::eof))
- break;
- toks.push_back(tok);
- }
+ PP.LexTokensUntilEOF(&toks);
// Make sure we got the tokens that we expected.
ASSERT_EQ(0U, toks.size());
Index: clang/unittests/Analysis/MacroExpansionContextTest.cpp
===================================================================
--- clang/unittests/Analysis/MacroExpansionContextTest.cpp
+++ clang/unittests/Analysis/MacroExpansionContextTest.cpp
@@ -73,12 +73,7 @@
// Lex source text.
PP.EnterMainSourceFile();
- while (true) {
- Token Tok;
- PP.Lex(Tok);
- if (Tok.is(tok::eof))
- break;
- }
+ PP.LexTokensUntilEOF();
// Callbacks have been executed at this point.
return Ctx;
Index: clang/lib/Lex/Preprocessor.cpp
===================================================================
--- clang/lib/Lex/Preprocessor.cpp
+++ clang/lib/Lex/Preprocessor.cpp
@@ -998,6 +998,17 @@
}
}
+void Preprocessor::LexTokensUntilEOF(std::vector<Token> *Tokens) {
+ while (1) {
+ Token Tok;
+ Lex(Tok);
+ if (Tok.isOneOf(tok::unknown, tok::eof, tok::eod))
+ break;
+ if (Tokens != nullptr)
+ Tokens->push_back(Tok);
+ }
+}
+
/// Lex a header-name token (including one formed from header-name-tokens if
/// \p AllowConcatenation is \c true).
///
Index: clang/include/clang/Lex/Preprocessor.h
===================================================================
--- clang/include/clang/Lex/Preprocessor.h
+++ clang/include/clang/Lex/Preprocessor.h
@@ -1722,6 +1722,9 @@
/// Lex the next token for this preprocessor.
void Lex(Token &Result);
+ /// Lex all tokens for this preprocessor until (and excluding) end of file.
+ void LexTokensUntilEOF(std::vector<Token> *Tokens = nullptr);
+
/// Lex a token, forming a header-name token if possible.
bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
_______________________________________________
cfe-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
