ilya-biryukov created this revision.
ilya-biryukov added reviewers: kadircet, sammccall.
Herald added subscribers: arphaman, jkorous, MaskRay.
Herald added a project: clang.
This significantly improves performance of background indexing.
We do not collect references and declarations inside the processed
files, so this does not affect the final indexing results.
The idea is borrowed from libclang, which has a similar optimization in
its indexing functionality.
Measurements show a nice decrease in indexing time, up to ~40% for
building the whole index. These are not proper benchmarks, so one should
not rely on these results too much.
1. Rebuilding the whole index:
- Before. Total time: 14m58s. ./bin/clangd -pch-storage=memory <
./clangd.input 23917.67s user 515.86s system 2718% cpu 14:58.68 total
- After. Total time: 8m40s. ./bin/clangd -pch-storage=memory < ./clangd.input
13627.29s user 288.10s system 2672% cpu 8:40.67 total
2. Rebuilding index after removing shards matching '*clangd*'
(case-insensitively):
- Before. Total time: 30s. ./bin/clangd -pch-storage=memory < ./clangd.input
130.94s user 6.82s system 452% cpu 30.423 total
- After. Total time: 26s. ./bin/clangd -pch-storage=memory < ./clangd.input
80.51s user 5.40s system 333% cpu 25.777 total
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D66226
Files:
clang-tools-extra/clangd/index/IndexAction.cpp
clang-tools-extra/clangd/index/SymbolCollector.cpp
clang-tools-extra/clangd/index/SymbolCollector.h
Index: clang-tools-extra/clangd/index/SymbolCollector.h
===================================================================
--- clang-tools-extra/clangd/index/SymbolCollector.h
+++ clang-tools-extra/clangd/index/SymbolCollector.h
@@ -112,6 +112,11 @@
RefSlab takeRefs() { return std::move(Refs).build(); }
RelationSlab takeRelations() { return std::move(Relations).build(); }
+ /// Returns true if we are interested in references and declarations from \p
+ /// FID. If this function return false, bodies of functions inside those files
+ /// will be skipped to decrease indexing time.
+ bool shouldProcessFile(FileID FID);
+
void finish() override;
private:
Index: clang-tools-extra/clangd/index/SymbolCollector.cpp
===================================================================
--- clang-tools-extra/clangd/index/SymbolCollector.cpp
+++ clang-tools-extra/clangd/index/SymbolCollector.cpp
@@ -742,5 +742,12 @@
return false;
}
+bool SymbolCollector::shouldProcessFile(FileID FID) {
+ assert(ASTCtx);
+ if (!Opts.FileFilter)
+ return true;
+ return Opts.FileFilter(ASTCtx->getSourceManager(), FID);
+}
+
} // namespace clangd
} // namespace clang
Index: clang-tools-extra/clangd/index/IndexAction.cpp
===================================================================
--- clang-tools-extra/clangd/index/IndexAction.cpp
+++ clang-tools-extra/clangd/index/IndexAction.cpp
@@ -11,9 +11,17 @@
#include "Logger.h"
#include "index/Relation.h"
#include "index/SymbolOrigin.h"
+#include "clang/AST/ASTConsumer.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/SourceManager.h"
#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/MultiplexConsumer.h"
#include "clang/Index/IndexingAction.h"
#include "clang/Tooling/Tooling.h"
+#include "llvm/ADT/STLExtras.h"
+#include <functional>
+#include <memory>
#include <utility>
namespace clang {
@@ -113,6 +121,40 @@
IncludeGraph &IG;
};
+/// Returns an ASTConsumer that wraps \p Inner and additionally instructs the
+/// parser to skip bodies of functions in the files that should not be
+/// processed.
+static std::unique_ptr<ASTConsumer>
+skipProcessedFunctions(std::unique_ptr<ASTConsumer> Inner,
+ std::function<bool(FileID)> ShouldProcessFile) {
+ class SkipProcessedFunctions : public ASTConsumer {
+ public:
+ SkipProcessedFunctions(std::function<bool(FileID)> FileFilter)
+ : ShouldProcessFile(std::move(FileFilter)), Context(nullptr) {
+ assert(this->ShouldProcessFile);
+ }
+
+ void Initialize(ASTContext &Context) override { this->Context = &Context; }
+ bool shouldSkipFunctionBody(Decl *D) override {
+ assert(Context && "Initialize() was never called.");
+ auto &SM = Context->getSourceManager();
+ auto FID = SM.getFileID(SM.getExpansionLoc(D->getLocation()));
+ if (!FID.isValid())
+ return false;
+ return !ShouldProcessFile(FID);
+ }
+
+ private:
+ std::function<bool(FileID)> ShouldProcessFile;
+ const ASTContext *Context;
+ };
+ std::vector<std::unique_ptr<ASTConsumer>> Consumers;
+ Consumers.push_back(
+ llvm::make_unique<SkipProcessedFunctions>(ShouldProcessFile));
+ Consumers.push_back(std::move(Inner));
+ return llvm::make_unique<MultiplexConsumer>(std::move(Consumers));
+}
+
// Wraps the index action and reports index data after each translation unit.
class IndexAction : public WrapperFrontendAction {
public:
@@ -137,7 +179,9 @@
if (IncludeGraphCallback != nullptr)
CI.getPreprocessor().addPPCallbacks(
llvm::make_unique<IncludeGraphCollector>(CI.getSourceManager(), IG));
- return WrapperFrontendAction::CreateASTConsumer(CI, InFile);
+ return skipProcessedFunctions(
+ WrapperFrontendAction::CreateASTConsumer(CI, InFile),
+ [this](FileID FID) { return Collector->shouldProcessFile(FID); });
}
bool BeginInvocation(CompilerInstance &CI) override {
@@ -147,6 +191,10 @@
// Avoids some analyses too. Set in two places as we're late to the party.
CI.getDiagnosticOpts().IgnoreWarnings = true;
CI.getDiagnostics().setIgnoreAllWarnings(true);
+ /// Instruct the parser to ask our ASTConsumer if it should skip function
+ /// bodies. The ASTConsumer will take care of skipping only functions inside
+ /// the files that we have already processed.
+ CI.getFrontendOpts().SkipFunctionBodies = true;
return WrapperFrontendAction::BeginInvocation(CI);
}
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
