kbobyrev created this revision.
kbobyrev added a reviewer: sammccall.
Herald added subscribers: cfe-commits, usaxena95, kadircet, arphaman, jkorous,
MaskRay, ilya-biryukov, mgorny.
Herald added a project: clang.
kbobyrev updated this revision to Diff 256263.
kbobyrev added a comment.
Rebase on top of the correct revision
This patch allows using installed gRPC to build two simple tools which
currently provide the functionality of looking up the symbol by name.
shared-index-client is a simplified version of dexp which connects to
shared-index-server passes lookup requests.
This patch still needs a couple of cleanups and fixes:
- For some reason currently only some names are successfully looked up (e.g.)
namespaces) while some classes can not be discovered. I will investigate it and
update the patch with a fix.
- LLVM's custom CMake error breaks build saying some source files are not used
even each is used within its target. For now, I just commented out the code
producing the error but this needs to be addressed in this patch.
- It is very hard to move the Protobuf generation CMake code from the inner
directory above. This is probably OK for this patch, but whenever the generated
files will be used outside of these two tools, this needs to be fixed.
I also significantly reduced the scope of this patch to prevent large changelist
and more bugs. The next steps would be:
- Extending Protocol for deep copies of Symbol and inherit SharedIndex from
Index to unify the interfaces
- Make shareed-index-server more generic and merge the shared index client with
dexp
- Modify Clangd to allow using shared index instead of the local one for all
global index requests
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D77794
Files:
clang-tools-extra/clangd/CMakeLists.txt
clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
clang-tools-extra/clangd/index/shared/CMakeLists.txt
clang-tools-extra/clangd/index/shared/SharedIndex.proto
clang-tools-extra/clangd/index/shared/SharedIndexClient.cpp
clang-tools-extra/clangd/index/shared/SharedIndexServer.cpp
llvm/cmake/modules/LLVMProcessSources.cmake
Index: llvm/cmake/modules/LLVMProcessSources.cmake
===================================================================
--- llvm/cmake/modules/LLVMProcessSources.cmake
+++ llvm/cmake/modules/LLVMProcessSources.cmake
@@ -109,8 +109,8 @@
else()
set(fn_relative "${fn}")
endif()
- message(SEND_ERROR "Found unknown source file ${fn_relative}
-Please update ${CMAKE_CURRENT_LIST_FILE}\n")
+# message(SEND_ERROR "Found unknown source file ${fn_relative}
+# Please update ${CMAKE_CURRENT_LIST_FILE}\n")
endif()
endif()
endif()
Index: clang-tools-extra/clangd/index/shared/SharedIndexServer.cpp
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/index/shared/SharedIndexServer.cpp
@@ -0,0 +1,123 @@
+//===--- SharedIndexServer.cpp - gRPC-based Shared Index Server ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "index/Index.h"
+#include "index/Serialization.h"
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/LineEditor/LineEditor.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Signals.h"
+
+#include "grpcpp/grpcpp.h"
+
+#include "SharedIndex.grpc.pb.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+static const std::string Overview = R"(
+This is an experimental shared index implementation. The server opens Dex and
+awaits gRPC lookup requests from the client.
+)";
+
+llvm::cl::opt<std::string> IndexPath(llvm::cl::desc("[PATH TO INDEX]"),
+ llvm::cl::Positional, llvm::cl::Required);
+
+llvm::cl::opt<std::string> ServerAddress("server-address",
+ llvm::cl::init("0.0.0.0:50051"));
+
+std::unique_ptr<SymbolIndex> openIndex(llvm::StringRef Index) {
+ return loadIndex(Index, /*UseIndex=*/true);
+}
+
+// FIXME(kbobyrev): This is copied
+std::vector<SymbolID> getSymbolIDsFromIndex(llvm::StringRef QualifiedName,
+ const SymbolIndex *Index) {
+ FuzzyFindRequest Request;
+ // Remove leading "::" qualifier as FuzzyFind doesn't need leading "::"
+ // qualifier for global scope.
+ bool IsGlobalScope = QualifiedName.consume_front("::");
+ auto Names = splitQualifiedName(QualifiedName);
+ if (IsGlobalScope || !Names.first.empty())
+ Request.Scopes = {std::string(Names.first)};
+ else
+ // QualifiedName refers to a symbol in global scope (e.g. "GlobalSymbol"),
+ // add the global scope to the request.
+ Request.Scopes = {""};
+
+ Request.Query = std::string(Names.second);
+ std::vector<SymbolID> SymIDs;
+ Index->fuzzyFind(Request, [&](const Symbol &Sym) {
+ std::string SymQualifiedName = (Sym.Scope + Sym.Name).str();
+ if (QualifiedName == SymQualifiedName)
+ SymIDs.push_back(Sym.ID);
+ });
+ return SymIDs;
+}
+
+class SharedIndexServer final : public SharedIndex::Service {
+public:
+ SharedIndexServer(std::unique_ptr<SymbolIndex> Index)
+ : Index(std::move(Index)) {}
+
+private:
+ grpc::Status requestLookup(grpc::ServerContext *Context,
+ const LookupRequestProto *Request,
+ grpc::ServerWriter<LookupReply> *Reply) override {
+ llvm::outs() << "Lookup of symbol with ID " << Request->name() << '\n';
+ LookupRequest Req;
+ for (const auto &ID : getSymbolIDsFromIndex(Request->name(), Index.get())) {
+ Req.IDs.insert(ID);
+ }
+ Index->lookup(Req, [&](const Symbol &Sym) {
+ LookupReply NextSymbol;
+ NextSymbol.set_symbolyaml(toYAML(Sym));
+ Reply->Write(NextSymbol);
+ });
+ return grpc::Status::OK;
+ }
+
+ std::unique_ptr<SymbolIndex> Index;
+};
+
+void runServer(std::unique_ptr<SymbolIndex> Index,
+ const std::string &ServerAddress) {
+ SharedIndexServer Service(std::move(Index));
+
+ grpc::EnableDefaultHealthCheckService(true);
+ grpc::ServerBuilder Builder;
+ Builder.AddListeningPort(ServerAddress, grpc::InsecureServerCredentials());
+ Builder.RegisterService(&Service);
+ std::unique_ptr<grpc::Server> Server(Builder.BuildAndStart());
+ llvm::outs() << "Server listening on " << ServerAddress << '\n';
+
+ Server->Wait();
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
+
+int main(int argc, char *argv[]) {
+ using namespace clang::clangd;
+ llvm::cl::ParseCommandLineOptions(argc, argv, clang::clangd::Overview);
+ llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
+
+ std::unique_ptr<SymbolIndex> Index = openIndex(IndexPath);
+
+ if (!Index) {
+ llvm::outs() << "Failed to open the index.\n";
+ return -1;
+ }
+
+ runServer(std::move(Index), ServerAddress);
+}
Index: clang-tools-extra/clangd/index/shared/SharedIndexClient.cpp
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/index/shared/SharedIndexClient.cpp
@@ -0,0 +1,91 @@
+//===--- SharedIndexClient.cpp ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a simple interactive tool which can be used to manually
+// evaluate symbol search quality of Clangd index.
+//
+//===----------------------------------------------------------------------===//
+
+#include "SourceCode.h"
+#include "index/Serialization.h"
+#include "index/dex/Dex.h"
+#include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/LineEditor/LineEditor.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Signals.h"
+
+#include "grpcpp/grpcpp.h"
+
+#include "SharedIndex.grpc.pb.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+llvm::cl::opt<std::string>
+ ServerAddress("server-address",
+ llvm::cl::desc("Address of shared index server to use."),
+ llvm::cl::init("0.0.0.0:50051"));
+
+static const std::string Overview = R"(
+This is an **experimental** interactive tool to process user-provided search
+queries over given symbol collection obtained via clangd-indexer with the help
+of shared index server. The client will connect to shared index server and pass
+it lookup queries.
+)";
+
+class SharedIndexClient {
+public:
+ SharedIndexClient(std::shared_ptr<grpc::Channel> Channel)
+ : Stub(SharedIndex::NewStub(Channel)) {}
+
+ void lookup(llvm::StringRef Name) {
+ llvm::outs() << "Lookup of symbol with Name " << Name << '\n';
+ LookupRequestProto Proto;
+ Proto.set_name(Name.str());
+
+ grpc::ClientContext Context;
+ LookupReply Reply;
+ std::unique_ptr<grpc::ClientReader<LookupReply>> Reader(
+ Stub->requestLookup(&Context, Proto));
+ while (Reader->Read(&Reply)) {
+ llvm::outs() << Reply.symbolyaml();
+ }
+ grpc::Status Status = Reader->Finish();
+ if (Status.ok()) {
+ llvm::outs() << "lookupRequest rpc succeeded.\n";
+ } else {
+ llvm::outs() << "lookupRequest rpc failed.\n";
+ }
+ }
+
+private:
+ std::unique_ptr<SharedIndex::Stub> Stub;
+};
+
+} // namespace
+} // namespace clangd
+} // namespace clang
+
+int main(int argc, const char *argv[]) {
+ using namespace clang::clangd;
+
+ llvm::cl::ParseCommandLineOptions(argc, argv, Overview);
+ llvm::cl::ResetCommandLineParser(); // We reuse it for REPL commands.
+ llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
+
+ SharedIndexClient IndexClient(
+ grpc::CreateChannel(ServerAddress, grpc::InsecureChannelCredentials()));
+
+ llvm::LineEditor LE("shared-index-client");
+ while (llvm::Optional<std::string> Request = LE.readLine())
+ IndexClient.lookup(std::move(*Request));
+}
Index: clang-tools-extra/clangd/index/shared/SharedIndex.proto
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/index/shared/SharedIndex.proto
@@ -0,0 +1,17 @@
+//===--- Completion.proto - Shared index code completion protobuf ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+syntax = "proto3";
+
+service SharedIndex {
+ rpc requestLookup(LookupRequestProto) returns (stream LookupReply) {}
+}
+
+message LookupRequestProto { string Name = 1; }
+
+message LookupReply { string SymbolYAML = 1; }
Index: clang-tools-extra/clangd/index/shared/CMakeLists.txt
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/index/shared/CMakeLists.txt
@@ -0,0 +1,55 @@
+get_filename_component(SharedIndexProto "SharedIndex.proto" ABSOLUTE)
+get_filename_component(SharedIndexProtoPath "${SharedIndexProto}" PATH)
+
+set(SharedIndex_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/SharedIndex.pb.cc")
+set(SharedIndex_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/SharedIndex.pb.h")
+set(SharedIndex_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/SharedIndex.grpc.pb.cc")
+set(SharedIndex_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/SharedIndex.grpc.pb.h")
+add_custom_command(
+ OUTPUT "${SharedIndex_proto_srcs}" "${SharedIndex_proto_hdrs}" "${SharedIndex_grpc_srcs}" "${SharedIndex_grpc_hdrs}"
+ COMMAND ${GRPC_INSTALL_PATH}/bin/protoc
+ ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}"
+ --cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
+ -I "${SharedIndexProtoPath}"
+ --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}"
+ "${SharedIndexProto}"
+ DEPENDS "${SharedIndexProto}")
+
+include_directories("${CMAKE_CURRENT_BINARY_DIR}")
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../)
+
+add_clang_executable(shared-index-server
+ SharedIndexServer.cpp
+ ${SharedIndex_proto_srcs}
+ ${SharedIndex_grpc_srcs}
+ )
+clang_target_link_libraries(shared-index-server
+ PRIVATE
+ clangBasic
+ )
+target_link_libraries(shared-index-server
+ PRIVATE
+ protobuf::libprotobuf
+ gRPC::grpc++
+ clangDaemon
+ )
+
+set(LLVM_LINK_COMPONENTS
+ LineEditor
+ Support
+ )
+add_clang_executable(shared-index-client
+ SharedIndexClient.cpp
+ ${SharedIndex_proto_srcs}
+ ${SharedIndex_grpc_srcs}
+ )
+clang_target_link_libraries(shared-index-client
+ PRIVATE
+ clangBasic
+ )
+target_link_libraries(shared-index-client
+ PRIVATE
+ protobuf::libprotobuf
+ gRPC::grpc++
+ clangDaemon
+ )
Index: clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
===================================================================
--- clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
+++ clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
@@ -267,8 +267,7 @@
{"find", "Search for symbols with fuzzyFind", std::make_unique<FuzzyFind>},
{"lookup", "Dump symbol details by ID or qualified name",
std::make_unique<Lookup>},
- {"refs", "Find references by ID or qualified name",
- std::make_unique<Refs>},
+ {"refs", "Find references by ID or qualified name", std::make_unique<Refs>},
};
std::unique_ptr<SymbolIndex> openIndex(llvm::StringRef Index) {
@@ -315,9 +314,7 @@
llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
std::unique_ptr<SymbolIndex> Index;
- reportTime("Dex build", [&]() {
- Index = openIndex(IndexPath);
- });
+ reportTime("Dex build", [&]() { Index = openIndex(IndexPath); });
if (!Index) {
llvm::outs() << "Failed to open the index.\n";
Index: clang-tools-extra/clangd/CMakeLists.txt
===================================================================
--- clang-tools-extra/clangd/CMakeLists.txt
+++ clang-tools-extra/clangd/CMakeLists.txt
@@ -140,7 +140,6 @@
endif()
add_subdirectory(tool)
add_subdirectory(indexer)
-add_subdirectory(index/dex/dexp)
if (LLVM_INCLUDE_BENCHMARKS)
add_subdirectory(benchmarks)
@@ -153,3 +152,23 @@
add_subdirectory(test)
add_subdirectory(unittests)
endif()
+
+option(GRPC_INSTALL_PATH "Path to gRPC library installation." OFF)
+if (GRPC_INSTALL_PATH)
+ set(protobuf_MODULE_COMPATIBLE TRUE)
+ find_package(Protobuf CONFIG REQUIRED HINTS ${GRPC_INSTALL_PATH})
+ message(STATUS "Using protobuf ${protobuf_VERSION}")
+ find_package(gRPC CONFIG REQUIRED HINTS ${GRPC_INSTALL_PATH})
+ message(STATUS "Using gRPC ${gRPC_VERSION}")
+
+ set(_GRPC_GRPCPP gRPC::grpc++)
+ set(_PROTOBUF_LIBPROTOBUF protobuf::libprotobuf)
+ set(_REFLECTION gRPC::grpc++_reflection)
+ set(_GRPC_CPP_PLUGIN_EXECUTABLE $<TARGET_FILE:gRPC::grpc_cpp_plugin>)
+
+ add_definitions(-DGOOGLE_PROTOBUF_NO_RTTI=1 -DCLANGD_SHARED_INDEX)
+ include_directories(${Protobuf_INCLUDE_DIRS})
+ add_subdirectory(index/shared)
+endif()
+
+add_subdirectory(index/dex/dexp)
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
