This revision was automatically updated to reflect the committed changes.
Closed by commit rL363803: [clangd] Collect tokens of main files when building
the AST (authored by ibiryukov, committed by ).
Herald added a project: LLVM.
Herald added a subscriber: llvm-commits.
Changed prior to commit:
https://reviews.llvm.org/D62956?vs=205387&id=205579#toc
Repository:
rL LLVM
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D62956/new/
https://reviews.llvm.org/D62956
Files:
clang-tools-extra/trunk/clangd/CMakeLists.txt
clang-tools-extra/trunk/clangd/ClangdUnit.cpp
clang-tools-extra/trunk/clangd/ClangdUnit.h
clang-tools-extra/trunk/clangd/tool/CMakeLists.txt
clang-tools-extra/trunk/clangd/unittests/CMakeLists.txt
clang-tools-extra/trunk/clangd/unittests/ClangdUnitTests.cpp
Index: clang-tools-extra/trunk/clangd/ClangdUnit.cpp
===================================================================
--- clang-tools-extra/trunk/clangd/ClangdUnit.cpp
+++ clang-tools-extra/trunk/clangd/ClangdUnit.cpp
@@ -36,6 +36,7 @@
#include "clang/Serialization/ASTWriter.h"
#include "clang/Serialization/PCHContainerOperations.h"
#include "clang/Tooling/CompilationDatabase.h"
+#include "clang/Tooling/Syntax/Tokens.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallString.h"
@@ -418,6 +419,9 @@
collectIWYUHeaderMaps(&CanonIncludes);
Clang->getPreprocessor().addCommentHandler(IWYUHandler.get());
+ // Collect tokens of the main file.
+ syntax::TokenCollector Tokens(Clang->getPreprocessor());
+
if (!Action->Execute())
log("Execute() failed when building AST for {0}", MainInput.getFile());
@@ -446,8 +450,9 @@
if (Preamble)
Diags.insert(Diags.begin(), Preamble->Diags.begin(), Preamble->Diags.end());
return ParsedAST(std::move(Preamble), std::move(Clang), std::move(Action),
- std::move(ParsedDecls), std::move(Diags),
- std::move(Includes), std::move(CanonIncludes));
+ std::move(Tokens).consume(), std::move(ParsedDecls),
+ std::move(Diags), std::move(Includes),
+ std::move(CanonIncludes));
}
ParsedAST::ParsedAST(ParsedAST &&Other) = default;
@@ -540,11 +545,13 @@
ParsedAST::ParsedAST(std::shared_ptr<const PreambleData> Preamble,
std::unique_ptr<CompilerInstance> Clang,
std::unique_ptr<FrontendAction> Action,
+ syntax::TokenBuffer Tokens,
std::vector<Decl *> LocalTopLevelDecls,
std::vector<Diag> Diags, IncludeStructure Includes,
CanonicalIncludes CanonIncludes)
: Preamble(std::move(Preamble)), Clang(std::move(Clang)),
- Action(std::move(Action)), Diags(std::move(Diags)),
+ Action(std::move(Action)), Tokens(std::move(Tokens)),
+ Diags(std::move(Diags)),
LocalTopLevelDecls(std::move(LocalTopLevelDecls)),
Includes(std::move(Includes)), CanonIncludes(std::move(CanonIncludes)) {
assert(this->Clang);
Index: clang-tools-extra/trunk/clangd/tool/CMakeLists.txt
===================================================================
--- clang-tools-extra/trunk/clangd/tool/CMakeLists.txt
+++ clang-tools-extra/trunk/clangd/tool/CMakeLists.txt
@@ -26,5 +26,6 @@
clangSema
clangTooling
clangToolingCore
+ clangToolingSyntax
${CLANGD_XPC_LIBS}
)
Index: clang-tools-extra/trunk/clangd/CMakeLists.txt
===================================================================
--- clang-tools-extra/trunk/clangd/CMakeLists.txt
+++ clang-tools-extra/trunk/clangd/CMakeLists.txt
@@ -128,6 +128,7 @@
clangToolingCore
clangToolingInclusions
clangToolingRefactoring
+ clangToolingSyntax
${LLVM_PTHREAD_LIB}
${CLANGD_ATOMIC_LIB}
)
Index: clang-tools-extra/trunk/clangd/ClangdUnit.h
===================================================================
--- clang-tools-extra/trunk/clangd/ClangdUnit.h
+++ clang-tools-extra/trunk/clangd/ClangdUnit.h
@@ -24,6 +24,7 @@
#include "clang/Serialization/ASTBitCodes.h"
#include "clang/Tooling/CompilationDatabase.h"
#include "clang/Tooling/Core/Replacement.h"
+#include "clang/Tooling/Syntax/Tokens.h"
#include <memory>
#include <string>
#include <vector>
@@ -115,10 +116,14 @@
const IncludeStructure &getIncludeStructure() const;
const CanonicalIncludes &getCanonicalIncludes() const;
+ /// Tokens recorded while parsing the main file.
+ /// (!) does not have tokens from the preamble.
+ const syntax::TokenBuffer &getTokens() const { return Tokens; }
+
private:
ParsedAST(std::shared_ptr<const PreambleData> Preamble,
std::unique_ptr<CompilerInstance> Clang,
- std::unique_ptr<FrontendAction> Action,
+ std::unique_ptr<FrontendAction> Action, syntax::TokenBuffer Tokens,
std::vector<Decl *> LocalTopLevelDecls, std::vector<Diag> Diags,
IncludeStructure Includes, CanonicalIncludes CanonIncludes);
@@ -132,6 +137,11 @@
// FrontendAction.EndSourceFile).
std::unique_ptr<CompilerInstance> Clang;
std::unique_ptr<FrontendAction> Action;
+ /// Tokens recorded after the preamble finished.
+ /// - Includes all spelled tokens for the main file.
+ /// - Includes expanded tokens produced **after** preabmle.
+ /// - Does not have spelled or expanded tokens for files from preamble.
+ syntax::TokenBuffer Tokens;
// Data, stored after parsing.
std::vector<Diag> Diags;
Index: clang-tools-extra/trunk/clangd/unittests/CMakeLists.txt
===================================================================
--- clang-tools-extra/trunk/clangd/unittests/CMakeLists.txt
+++ clang-tools-extra/trunk/clangd/unittests/CMakeLists.txt
@@ -87,6 +87,7 @@
clangTooling
clangToolingCore
clangToolingInclusions
+ clangToolingSyntax
LLVMSupport
LLVMTestingSupport
)
Index: clang-tools-extra/trunk/clangd/unittests/ClangdUnitTests.cpp
===================================================================
--- clang-tools-extra/trunk/clangd/unittests/ClangdUnitTests.cpp
+++ clang-tools-extra/trunk/clangd/unittests/ClangdUnitTests.cpp
@@ -10,6 +10,8 @@
#include "ClangdUnit.h"
#include "SourceCode.h"
#include "TestTU.h"
+#include "clang/Basic/TokenKinds.h"
+#include "clang/Tooling/Syntax/Tokens.h"
#include "llvm/Support/ScopedPrinter.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
@@ -81,6 +83,37 @@
EXPECT_THAT(AST.getLocalTopLevelDecls(), ElementsAre(DeclNamed("main")));
}
+TEST(ClangdUnitTest, TokensAfterPreamble) {
+ TestTU TU;
+ TU.AdditionalFiles["foo.h"] = R"(
+ int foo();
+ )";
+ TU.Code = R"cpp(
+ #include "foo.h"
+ first_token;
+ void test() {
+ }
+ last_token
+)cpp";
+ auto AST = TU.build();
+ const syntax::TokenBuffer &T = AST.getTokens();
+ const auto &SM = AST.getSourceManager();
+
+ ASSERT_GT(T.expandedTokens().size(), 2u);
+ // Check first token after the preamble.
+ EXPECT_EQ(T.expandedTokens().front().text(SM), "first_token");
+ // Last token is always 'eof'.
+ EXPECT_EQ(T.expandedTokens().back().kind(), tok::eof);
+ // Check the token before 'eof'.
+ EXPECT_EQ(T.expandedTokens().drop_back().back().text(SM), "last_token");
+
+ // The spelled tokens for the main file should have everything.
+ auto Spelled = T.spelledTokens(SM.getMainFileID());
+ ASSERT_FALSE(Spelled.empty());
+ EXPECT_EQ(Spelled.front().kind(), tok::hash);
+ EXPECT_EQ(Spelled.back().text(SM), "last_token");
+}
+
} // namespace
} // namespace clangd
} // namespace clang
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
