kadircet updated this revision to Diff 191684.
kadircet marked 11 inline comments as done.
kadircet added a comment.
Herald added a subscriber: jfb.
- Address comments
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D59605/new/
https://reviews.llvm.org/D59605
Files:
clang-tools-extra/clangd/CMakeLists.txt
clang-tools-extra/clangd/background-indexer/BackgroundIndexer.cpp
clang-tools-extra/clangd/background-indexer/CMakeLists.txt
clang-tools-extra/clangd/index/Background.cpp
clang-tools-extra/clangd/index/Background.h
Index: clang-tools-extra/clangd/index/Background.h
===================================================================
--- clang-tools-extra/clangd/index/Background.h
+++ clang-tools-extra/clangd/index/Background.h
@@ -144,6 +144,10 @@
std::deque<std::pair<Task, ThreadPriority>> Queue;
std::vector<std::thread> ThreadPool; // FIXME: Abstract this away.
GlobalCompilationDatabase::CommandChanged::Subscription CommandsChanged;
+
+ // For logging
+ std::atomic<size_t> EnqueuedTUs{0};
+ std::atomic<size_t> IndexedTUs{0};
};
} // namespace clangd
Index: clang-tools-extra/clangd/index/Background.cpp
===================================================================
--- clang-tools-extra/clangd/index/Background.cpp
+++ clang-tools-extra/clangd/index/Background.cpp
@@ -29,6 +29,7 @@
#include <chrono>
#include <memory>
+#include <mutex>
#include <numeric>
#include <queue>
#include <random>
@@ -215,6 +216,7 @@
// We're doing this asynchronously, because we'll read shards here too.
log("Enqueueing {0} commands for indexing", ChangedFiles.size());
SPAN_ATTACH(Tracer, "files", int64_t(ChangedFiles.size()));
+ EnqueuedTUs += ChangedFiles.size();
auto NeedsReIndexing = loadShards(std::move(ChangedFiles));
// Run indexing for files that need to be updated.
@@ -447,8 +449,9 @@
assert(Index.Symbols && Index.Refs && Index.Sources &&
"Symbols, Refs and Sources must be set.");
- log("Indexed {0} ({1} symbols, {2} refs, {3} files)",
- Inputs.CompileCommand.Filename, Index.Symbols->size(),
+ ++IndexedTUs;
+ log("[{0}/{1}] Indexed {2} ({3} symbols, {4} refs, {5} files)", IndexedTUs,
+ EnqueuedTUs, Inputs.CompileCommand.Filename, Index.Symbols->size(),
Index.Refs->numRefs(), Index.Sources->size());
SPAN_ATTACH(Tracer, "symbols", int(Index.Symbols->size()));
SPAN_ATTACH(Tracer, "refs", int(Index.Refs->numRefs()));
@@ -581,12 +584,14 @@
// Keeps track of the loaded shards to make sure we don't perform redundant
// disk IO. Keys are absolute paths.
llvm::StringSet<> LoadedShards;
+ size_t UpToDateTUs = 0;
for (const auto &File : ChangedFiles) {
ProjectInfo PI;
auto Cmd = CDB.getCompileCommand(File, &PI);
if (!Cmd)
continue;
BackgroundIndexStorage *IndexStorage = IndexStorageFactory(PI.SourceRoot);
+ ++UpToDateTUs;
auto Dependencies = loadShard(*Cmd, IndexStorage, LoadedShards);
for (const auto &Dependency : Dependencies) {
if (!Dependency.NeedsReIndexing || FilesToIndex.count(Dependency.Path))
@@ -594,17 +599,19 @@
// FIXME: Currently, we simply schedule indexing on a TU whenever any of
// its dependencies needs re-indexing. We might do it smarter by figuring
// out a minimal set of TUs that will cover all the stale dependencies.
- vlog("Enqueueing TU {0} because its dependency {1} needs re-indexing.",
+ dlog("Enqueueing TU {0} because its dependency {1} needs re-indexing.",
Cmd->Filename, Dependency.Path);
NeedsReIndexing.push_back({std::move(*Cmd), IndexStorage});
// Mark all of this TU's dependencies as to-be-indexed so that we won't
// try to re-index those.
for (const auto &Dependency : Dependencies)
FilesToIndex.insert(Dependency.Path);
+ --UpToDateTUs;
break;
}
}
- vlog("Loaded all shards");
+ IndexedTUs += UpToDateTUs;
+ log("[{0}/{1}] Loaded shards from storage", IndexedTUs, EnqueuedTUs);
reset(IndexedSymbols.buildIndex(IndexType::Heavy, DuplicateHandling::Merge));
vlog("BackgroundIndex: built symbol index with estimated memory {0} "
"bytes.",
Index: clang-tools-extra/clangd/background-indexer/CMakeLists.txt
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/background-indexer/CMakeLists.txt
@@ -0,0 +1,20 @@
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../)
+
+set(LLVM_LINK_COMPONENTS
+ Support
+ )
+
+add_clang_executable(clangd-background-indexer
+ BackgroundIndexer.cpp
+)
+
+target_link_libraries(clangd-background-indexer
+ PRIVATE
+ clangAST
+ clangIndex
+ clangDaemon
+ clangBasic
+ clangFrontend
+ clangLex
+ clangTooling
+)
Index: clang-tools-extra/clangd/background-indexer/BackgroundIndexer.cpp
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/background-indexer/BackgroundIndexer.cpp
@@ -0,0 +1,70 @@
+//===--- BackgroundIndexer.cpp -----------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// A standalone binary to run clangd's background indexer.
+//
+//===----------------------------------------------------------------------===//
+
+#include "Context.h"
+#include "FSProvider.h"
+#include "GlobalCompilationDatabase.h"
+#include "Logger.h"
+#include "index/Background.h"
+#include "clang/Tooling/CompilationDatabase.h"
+#include "llvm/ADT/None.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Signals.h"
+#include "llvm/Support/raw_ostream.h"
+#include <memory>
+
+static llvm::cl::opt<std::string> CompileCommandsDir(
+ llvm::cl::Positional,
+ llvm::cl::desc("Directory containing compilation database"),
+ llvm::cl::init("."));
+
+int main(int Argc, const char **Argv) {
+ llvm::sys::PrintStackTraceOnErrorSignal(Argv[0]);
+ llvm::cl::ParseCommandLineOptions(
+ Argc, Argv, "A standalone binary to run clangd's background indexer.");
+ clang::clangd::RealFileSystemProvider FSProvider;
+
+ llvm::errs().SetBuffered();
+ clang::clangd::StreamLogger Logger(llvm::errs(), clang::clangd::Logger::Info);
+ clang::clangd::LoggingSession LoggingSession(Logger);
+
+ clang::clangd::DirectoryBasedGlobalCompilationDatabase DirectoryBasedCDB(
+ /*CompileCommandsDir=*/llvm::None);
+ clang::clangd::OverlayCDB CDB(&DirectoryBasedCDB);
+ // FIXME: Maybe add a way to not lower thread priority?
+ clang::clangd::BackgroundIndex BackgroundIdx(
+ clang::clangd::Context::empty(), FSProvider, CDB,
+ clang::clangd::BackgroundIndexStorage::createDiskBackedStorageFactory(),
+ // We don't want to build dex/mem index until every file has been
+ // indexed/loaded.
+ // FIXME: Change BackgroundIndex to only built after all files have been
+ // read if build period is zero, which should only be used by
+ // non-interactive tools like this one.
+ 24 * 60 * 60 * 1000);
+
+ // To trigger background indexing we need to discover a compilation database,
+ // which can be done by simply querying compile commands for a cpp file under
+ // the directory containing compilation_database.jon.
+ llvm::SmallString<128> DummyFile(CompileCommandsDir);
+ llvm::sys::fs::make_absolute(DummyFile);
+ llvm::sys::path::remove_dots(DummyFile, true);
+ llvm::sys::path::append(DummyFile, "dummy.cpp");
+ CDB.getCompileCommand(DummyFile);
+
+ // Wait untill indexing finishes.
+ BackgroundIdx.blockUntilIdleForTest(llvm::None);
+ return 0;
+}
Index: clang-tools-extra/clangd/CMakeLists.txt
===================================================================
--- clang-tools-extra/clangd/CMakeLists.txt
+++ clang-tools-extra/clangd/CMakeLists.txt
@@ -121,6 +121,7 @@
endif()
add_subdirectory(tool)
add_subdirectory(indexer)
+add_subdirectory(background-indexer)
add_subdirectory(index/dex/dexp)
if (LLVM_INCLUDE_BENCHMARKS)
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
