Author: ibiryukov Date: Wed Jun 19 07:03:19 2019 New Revision: 363803 URL: http://llvm.org/viewvc/llvm-project?rev=363803&view=rev Log: [clangd] Collect tokens of main files when building the AST
Summary: The first use of this is a code tweak to expand macro calls. Will later be used to build syntax trees. The memory overhead is small as we only store tokens of the main file. Reviewers: sammccall Reviewed By: sammccall Subscribers: mgorny, MaskRay, jkorous, arphaman, kadircet, cfe-commits Tags: #clang Differential Revision: https://reviews.llvm.org/D62956 Modified: clang-tools-extra/trunk/clangd/CMakeLists.txt clang-tools-extra/trunk/clangd/ClangdUnit.cpp clang-tools-extra/trunk/clangd/ClangdUnit.h clang-tools-extra/trunk/clangd/tool/CMakeLists.txt clang-tools-extra/trunk/clangd/unittests/CMakeLists.txt clang-tools-extra/trunk/clangd/unittests/ClangdUnitTests.cpp Modified: clang-tools-extra/trunk/clangd/CMakeLists.txt URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CMakeLists.txt?rev=363803&r1=363802&r2=363803&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/CMakeLists.txt (original) +++ clang-tools-extra/trunk/clangd/CMakeLists.txt Wed Jun 19 07:03:19 2019 @@ -128,6 +128,7 @@ add_clang_library(clangDaemon clangToolingCore clangToolingInclusions clangToolingRefactoring + clangToolingSyntax ${LLVM_PTHREAD_LIB} ${CLANGD_ATOMIC_LIB} ) Modified: clang-tools-extra/trunk/clangd/ClangdUnit.cpp URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdUnit.cpp?rev=363803&r1=363802&r2=363803&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/ClangdUnit.cpp (original) +++ clang-tools-extra/trunk/clangd/ClangdUnit.cpp Wed Jun 19 07:03:19 2019 @@ -36,6 +36,7 @@ #include "clang/Serialization/ASTWriter.h" #include "clang/Serialization/PCHContainerOperations.h" #include "clang/Tooling/CompilationDatabase.h" +#include "clang/Tooling/Syntax/Tokens.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" @@ -418,6 +419,9 @@ ParsedAST::build(std::unique_ptr<Compile collectIWYUHeaderMaps(&CanonIncludes); Clang->getPreprocessor().addCommentHandler(IWYUHandler.get()); + // Collect tokens of the main file. + syntax::TokenCollector Tokens(Clang->getPreprocessor()); + if (!Action->Execute()) log("Execute() failed when building AST for {0}", MainInput.getFile()); @@ -446,8 +450,9 @@ ParsedAST::build(std::unique_ptr<Compile if (Preamble) Diags.insert(Diags.begin(), Preamble->Diags.begin(), Preamble->Diags.end()); return ParsedAST(std::move(Preamble), std::move(Clang), std::move(Action), - std::move(ParsedDecls), std::move(Diags), - std::move(Includes), std::move(CanonIncludes)); + std::move(Tokens).consume(), std::move(ParsedDecls), + std::move(Diags), std::move(Includes), + std::move(CanonIncludes)); } ParsedAST::ParsedAST(ParsedAST &&Other) = default; @@ -540,11 +545,13 @@ PreambleData::PreambleData(PrecompiledPr ParsedAST::ParsedAST(std::shared_ptr<const PreambleData> Preamble, std::unique_ptr<CompilerInstance> Clang, std::unique_ptr<FrontendAction> Action, + syntax::TokenBuffer Tokens, std::vector<Decl *> LocalTopLevelDecls, std::vector<Diag> Diags, IncludeStructure Includes, CanonicalIncludes CanonIncludes) : Preamble(std::move(Preamble)), Clang(std::move(Clang)), - Action(std::move(Action)), Diags(std::move(Diags)), + Action(std::move(Action)), Tokens(std::move(Tokens)), + Diags(std::move(Diags)), LocalTopLevelDecls(std::move(LocalTopLevelDecls)), Includes(std::move(Includes)), CanonIncludes(std::move(CanonIncludes)) { assert(this->Clang); Modified: clang-tools-extra/trunk/clangd/ClangdUnit.h URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdUnit.h?rev=363803&r1=363802&r2=363803&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/ClangdUnit.h (original) +++ clang-tools-extra/trunk/clangd/ClangdUnit.h Wed Jun 19 07:03:19 2019 @@ -24,6 +24,7 @@ #include "clang/Serialization/ASTBitCodes.h" #include "clang/Tooling/CompilationDatabase.h" #include "clang/Tooling/Core/Replacement.h" +#include "clang/Tooling/Syntax/Tokens.h" #include <memory> #include <string> #include <vector> @@ -115,10 +116,14 @@ public: const IncludeStructure &getIncludeStructure() const; const CanonicalIncludes &getCanonicalIncludes() const; + /// Tokens recorded while parsing the main file. + /// (!) does not have tokens from the preamble. + const syntax::TokenBuffer &getTokens() const { return Tokens; } + private: ParsedAST(std::shared_ptr<const PreambleData> Preamble, std::unique_ptr<CompilerInstance> Clang, - std::unique_ptr<FrontendAction> Action, + std::unique_ptr<FrontendAction> Action, syntax::TokenBuffer Tokens, std::vector<Decl *> LocalTopLevelDecls, std::vector<Diag> Diags, IncludeStructure Includes, CanonicalIncludes CanonIncludes); @@ -132,6 +137,11 @@ private: // FrontendAction.EndSourceFile). std::unique_ptr<CompilerInstance> Clang; std::unique_ptr<FrontendAction> Action; + /// Tokens recorded after the preamble finished. + /// - Includes all spelled tokens for the main file. + /// - Includes expanded tokens produced **after** preabmle. + /// - Does not have spelled or expanded tokens for files from preamble. + syntax::TokenBuffer Tokens; // Data, stored after parsing. std::vector<Diag> Diags; Modified: clang-tools-extra/trunk/clangd/tool/CMakeLists.txt URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/tool/CMakeLists.txt?rev=363803&r1=363802&r2=363803&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/tool/CMakeLists.txt (original) +++ clang-tools-extra/trunk/clangd/tool/CMakeLists.txt Wed Jun 19 07:03:19 2019 @@ -26,5 +26,6 @@ target_link_libraries(clangd clangSema clangTooling clangToolingCore + clangToolingSyntax ${CLANGD_XPC_LIBS} ) Modified: clang-tools-extra/trunk/clangd/unittests/CMakeLists.txt URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/unittests/CMakeLists.txt?rev=363803&r1=363802&r2=363803&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/unittests/CMakeLists.txt (original) +++ clang-tools-extra/trunk/clangd/unittests/CMakeLists.txt Wed Jun 19 07:03:19 2019 @@ -87,6 +87,7 @@ target_link_libraries(ClangdTests clangTooling clangToolingCore clangToolingInclusions + clangToolingSyntax LLVMSupport LLVMTestingSupport ) Modified: clang-tools-extra/trunk/clangd/unittests/ClangdUnitTests.cpp URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/unittests/ClangdUnitTests.cpp?rev=363803&r1=363802&r2=363803&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/unittests/ClangdUnitTests.cpp (original) +++ clang-tools-extra/trunk/clangd/unittests/ClangdUnitTests.cpp Wed Jun 19 07:03:19 2019 @@ -10,6 +10,8 @@ #include "ClangdUnit.h" #include "SourceCode.h" #include "TestTU.h" +#include "clang/Basic/TokenKinds.h" +#include "clang/Tooling/Syntax/Tokens.h" #include "llvm/Support/ScopedPrinter.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -81,6 +83,37 @@ TEST(ClangdUnitTest, TopLevelDecls) { EXPECT_THAT(AST.getLocalTopLevelDecls(), ElementsAre(DeclNamed("main"))); } +TEST(ClangdUnitTest, TokensAfterPreamble) { + TestTU TU; + TU.AdditionalFiles["foo.h"] = R"( + int foo(); + )"; + TU.Code = R"cpp( + #include "foo.h" + first_token; + void test() { + } + last_token +)cpp"; + auto AST = TU.build(); + const syntax::TokenBuffer &T = AST.getTokens(); + const auto &SM = AST.getSourceManager(); + + ASSERT_GT(T.expandedTokens().size(), 2u); + // Check first token after the preamble. + EXPECT_EQ(T.expandedTokens().front().text(SM), "first_token"); + // Last token is always 'eof'. + EXPECT_EQ(T.expandedTokens().back().kind(), tok::eof); + // Check the token before 'eof'. + EXPECT_EQ(T.expandedTokens().drop_back().back().text(SM), "last_token"); + + // The spelled tokens for the main file should have everything. + auto Spelled = T.spelledTokens(SM.getMainFileID()); + ASSERT_FALSE(Spelled.empty()); + EXPECT_EQ(Spelled.front().kind(), tok::hash); + EXPECT_EQ(Spelled.back().text(SM), "last_token"); +} + } // namespace } // namespace clangd } // namespace clang _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
