ilya-biryukov updated this revision to Diff 197778.
ilya-biryukov added a comment.
- Simplify the interface, get only the expanded token stream
- An attempt to not report tokens from delayed parsing twice, almost works
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D59885/new/
https://reviews.llvm.org/D59885
Files:
clang/include/clang/Lex/Preprocessor.h
clang/include/clang/Lex/TokenLexer.h
clang/lib/Lex/Preprocessor.cpp
Index: clang/lib/Lex/Preprocessor.cpp
===================================================================
--- clang/lib/Lex/Preprocessor.cpp
+++ clang/lib/Lex/Preprocessor.cpp
@@ -867,20 +867,27 @@
// We loop here until a lex function returns a token; this avoids recursion.
bool ReturnedToken;
bool IsNewToken = true;
+ // Whether to call the OnToken callback on this token.
+ bool Report;
do {
switch (CurLexerKind) {
case CLK_Lexer:
+ Report = true;
ReturnedToken = CurLexer->Lex(Result);
break;
case CLK_TokenLexer:
+ Report = CurTokenLexer->isMacroExpansion();
ReturnedToken = CurTokenLexer->Lex(Result);
break;
case CLK_CachingLexer:
CachingLex(Result, IsNewToken);
+ Report = IsNewToken;
ReturnedToken = true;
break;
case CLK_LexAfterModuleImport:
- ReturnedToken = LexAfterModuleImport(Result);
+ LexAfterModuleImport(Result);
+ Report = true;
+ ReturnedToken = true;
break;
}
} while (!ReturnedToken);
@@ -937,6 +944,8 @@
LastTokenWasAt = Result.is(tok::at);
--LexLevel;
+ if (OnToken && LexLevel == 0 && Report)
+ OnToken(Result);
}
/// Lex a header-name token (including one formed from header-name-tokens if
Index: clang/include/clang/Lex/TokenLexer.h
===================================================================
--- clang/include/clang/Lex/TokenLexer.h
+++ clang/include/clang/Lex/TokenLexer.h
@@ -147,6 +147,10 @@
/// preprocessor directive.
bool isParsingPreprocessorDirective() const;
+ /// Returns true iff the TokenLexer is expanding a macro and not replaying a
+ /// stream of tokens.
+ bool isMacroExpansion() const { return Macro != nullptr; }
+
private:
void destroy();
Index: clang/include/clang/Lex/Preprocessor.h
===================================================================
--- clang/include/clang/Lex/Preprocessor.h
+++ clang/include/clang/Lex/Preprocessor.h
@@ -33,6 +33,7 @@
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/FoldingSet.h"
+#include "llvm/ADT/FunctionExtras.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/PointerUnion.h"
@@ -48,8 +49,8 @@
#include <cassert>
#include <cstddef>
#include <cstdint>
-#include <memory>
#include <map>
+#include <memory>
#include <string>
#include <utility>
#include <vector>
@@ -124,6 +125,7 @@
friend class VAOptDefinitionContext;
friend class VariadicMacroScopeGuard;
+ llvm::unique_function<void(const clang::Token &)> OnToken;
std::shared_ptr<PreprocessorOptions> PPOpts;
DiagnosticsEngine *Diags;
LangOptions &LangOpts;
@@ -997,6 +999,12 @@
}
/// \}
+ /// Register a function that would be called on each token in the final
+ /// expanded token stream.
+ void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) {
+ OnToken = std::move(F);
+ }
+
bool isMacroDefined(StringRef Id) {
return isMacroDefined(&Identifiers.get(Id));
}
Index: clang/lib/Lex/Preprocessor.cpp
===================================================================
--- clang/lib/Lex/Preprocessor.cpp
+++ clang/lib/Lex/Preprocessor.cpp
@@ -867,20 +867,27 @@
// We loop here until a lex function returns a token; this avoids recursion.
bool ReturnedToken;
bool IsNewToken = true;
+ // Whether to call the OnToken callback on this token.
+ bool Report;
do {
switch (CurLexerKind) {
case CLK_Lexer:
+ Report = true;
ReturnedToken = CurLexer->Lex(Result);
break;
case CLK_TokenLexer:
+ Report = CurTokenLexer->isMacroExpansion();
ReturnedToken = CurTokenLexer->Lex(Result);
break;
case CLK_CachingLexer:
CachingLex(Result, IsNewToken);
+ Report = IsNewToken;
ReturnedToken = true;
break;
case CLK_LexAfterModuleImport:
- ReturnedToken = LexAfterModuleImport(Result);
+ LexAfterModuleImport(Result);
+ Report = true;
+ ReturnedToken = true;
break;
}
} while (!ReturnedToken);
@@ -937,6 +944,8 @@
LastTokenWasAt = Result.is(tok::at);
--LexLevel;
+ if (OnToken && LexLevel == 0 && Report)
+ OnToken(Result);
}
/// Lex a header-name token (including one formed from header-name-tokens if
Index: clang/include/clang/Lex/TokenLexer.h
===================================================================
--- clang/include/clang/Lex/TokenLexer.h
+++ clang/include/clang/Lex/TokenLexer.h
@@ -147,6 +147,10 @@
/// preprocessor directive.
bool isParsingPreprocessorDirective() const;
+ /// Returns true iff the TokenLexer is expanding a macro and not replaying a
+ /// stream of tokens.
+ bool isMacroExpansion() const { return Macro != nullptr; }
+
private:
void destroy();
Index: clang/include/clang/Lex/Preprocessor.h
===================================================================
--- clang/include/clang/Lex/Preprocessor.h
+++ clang/include/clang/Lex/Preprocessor.h
@@ -33,6 +33,7 @@
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/FoldingSet.h"
+#include "llvm/ADT/FunctionExtras.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/PointerUnion.h"
@@ -48,8 +49,8 @@
#include <cassert>
#include <cstddef>
#include <cstdint>
-#include <memory>
#include <map>
+#include <memory>
#include <string>
#include <utility>
#include <vector>
@@ -124,6 +125,7 @@
friend class VAOptDefinitionContext;
friend class VariadicMacroScopeGuard;
+ llvm::unique_function<void(const clang::Token &)> OnToken;
std::shared_ptr<PreprocessorOptions> PPOpts;
DiagnosticsEngine *Diags;
LangOptions &LangOpts;
@@ -997,6 +999,12 @@
}
/// \}
+ /// Register a function that would be called on each token in the final
+ /// expanded token stream.
+ void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) {
+ OnToken = std::move(F);
+ }
+
bool isMacroDefined(StringRef Id) {
return isMacroDefined(&Identifiers.get(Id));
}
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
