kbobyrev updated this revision to Diff 257658.
kbobyrev marked 14 inline comments as done.
kbobyrev added a comment.
Address the current round of comments.
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D77794/new/
https://reviews.llvm.org/D77794
Files:
clang-tools-extra/clangd/CMakeLists.txt
clang-tools-extra/clangd/index/remote/CMakeLists.txt
clang-tools-extra/clangd/index/remote/Index.proto
clang-tools-extra/clangd/index/remote/README.md
clang-tools-extra/clangd/index/remote/client/CMakeLists.txt
clang-tools-extra/clangd/index/remote/client/Client.cpp
clang-tools-extra/clangd/index/remote/server/CMakeLists.txt
clang-tools-extra/clangd/index/remote/server/Server.cpp
llvm/CMakeLists.txt
llvm/cmake/modules/FindGRPC.cmake
Index: llvm/cmake/modules/FindGRPC.cmake
===================================================================
--- /dev/null
+++ llvm/cmake/modules/FindGRPC.cmake
@@ -0,0 +1,49 @@
+# FIXME(kirillbobyrev): Allow using system installed gRPC.
+if (GRPC_INSTALL_PATH)
+ set(protobuf_MODULE_COMPATIBLE TRUE)
+ find_package(Protobuf CONFIG REQUIRED HINTS ${GRPC_INSTALL_PATH})
+ message(STATUS "Using protobuf ${protobuf_VERSION}")
+ find_package(gRPC CONFIG REQUIRED HINTS ${GRPC_INSTALL_PATH})
+ message(STATUS "Using gRPC ${gRPC_VERSION}")
+
+ include_directories(${Protobuf_INCLUDE_DIRS})
+
+ # gRPC CMake CONFIG gives the libraries slightly odd names, make them match
+ # the conventional system-installed names.
+ set_target_properties(protobuf::libprotobuf PROPERTIES IMPORTED_GLOBAL TRUE)
+ add_library(protobuf ALIAS protobuf::libprotobuf)
+ set_target_properties(gRPC::grpc++ PROPERTIES IMPORTED_GLOBAL TRUE)
+ add_library(grpc++ ALIAS gRPC::grpc++)
+
+ set(GRPC_CPP_PLUGIN $<TARGET_FILE:gRPC::grpc_cpp_plugin>)
+ set(PROTOC ${Protobuf_PROTOC_EXECUTABLE})
+else()
+ get_filename_component(GRPC_CPP_PLUGIN grpc_cpp_plugin ABSOLUTE)
+ get_filename_component(PROTOC protoc ABSOLUTE)
+endif()
+
+# Proto headers are generated in ${CMAKE_CURRENT_BINARY_DIR}.
+# Libraries that use these headers should adjust the include path.
+# FIXME(kirillbobyrev): Allow optional generation of prptos only and give
+# callers control over it via additional parameters.
+function(generate_grpc_protos ProtoFile LibraryName)
+ get_filename_component(ProtoSourceAbsolutePath "${CMAKE_CURRENT_SOURCE_DIR}/${ProtoFile}" ABSOLUTE)
+ get_filename_component(ProtoSourcePath ${ProtoSourceAbsolutePath} PATH)
+
+ set(GeneratedProtoSource "${CMAKE_CURRENT_BINARY_DIR}/Index.pb.cc")
+ set(GeneratedProtoHeader "${CMAKE_CURRENT_BINARY_DIR}/Index.pb.h")
+ set(GeneratedGRPCSource "${CMAKE_CURRENT_BINARY_DIR}/Index.grpc.pb.cc")
+ set(GeneratedGRPCHeader "${CMAKE_CURRENT_BINARY_DIR}/Index.grpc.pb.h")
+ add_custom_command(
+ OUTPUT "${GeneratedProtoSource}" "${GeneratedProtoHeader}" "${GeneratedGRPCSource}" "${GeneratedGRPCHeader}"
+ COMMAND ${PROTOC}
+ ARGS --grpc_out="${CMAKE_CURRENT_BINARY_DIR}"
+ --cpp_out="${CMAKE_CURRENT_BINARY_DIR}"
+ --proto_path="${ProtoSourcePath}"
+ --plugin=protoc-gen-grpc="${GRPC_CPP_PLUGIN}"
+ "${ProtoSourceAbsolutePath}"
+ DEPENDS "${ProtoSourceAbsolutePath}")
+
+ add_library(${LibraryName} ${GeneratedProtoSource} ${GeneratedProtoHeader} ${GeneratedGRPCSource} ${GeneratedGRPCHeader})
+ target_link_libraries(${LibraryName} grpc++ protobuf)
+endfunction()
Index: llvm/CMakeLists.txt
===================================================================
--- llvm/CMakeLists.txt
+++ llvm/CMakeLists.txt
@@ -375,6 +375,15 @@
set(LLVM_WITH_Z3 1)
endif()
+# FIXME(kirillbobyrev): Document this in the LLVM docs.
+option(CLANGD_ENABLE_REMOTE "Use gRPC library to enable remote index support for Clangd" OFF)
+# FIXME(kirillbobyrev): Check if it works with optimized tablegen CMake option.
+set(GRPC_INSTALL_PATH "" CACHE PATH "Path to gRPC library installation.")
+
+if (CLANGD_ENABLE_REMOTE)
+ include(FindGRPC)
+endif()
+
if( LLVM_TARGETS_TO_BUILD STREQUAL "all" )
set( LLVM_TARGETS_TO_BUILD ${LLVM_ALL_TARGETS} )
endif()
Index: clang-tools-extra/clangd/index/remote/server/Server.cpp
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/index/remote/server/Server.cpp
@@ -0,0 +1,103 @@
+//===--- Server.cpp - gRPC-based Remote Index Server ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "index/Index.h"
+#include "index/Serialization.h"
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/LineEditor/LineEditor.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Signals.h"
+
+#include "grpcpp/grpcpp.h"
+#include "grpcpp/health_check_service_interface.h"
+
+#include "Index.grpc.pb.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+static const std::string Overview = R"(
+This is an experimental remote index implementation. The server opens Dex and
+awaits gRPC lookup requests from the client.
+)";
+
+llvm::cl::opt<std::string> IndexPath(llvm::cl::desc("<INDEX FILE>"),
+ llvm::cl::Positional, llvm::cl::Required);
+
+llvm::cl::opt<std::string> ServerAddress("server-address",
+ llvm::cl::init("0.0.0.0:50051"));
+
+std::unique_ptr<SymbolIndex> openIndex(llvm::StringRef Index) {
+ return loadIndex(Index, /*UseIndex=*/true);
+}
+
+class RemoteIndexServer final : public remote::Index::Service {
+public:
+ RemoteIndexServer(std::unique_ptr<SymbolIndex> Index)
+ : Index(std::move(Index)) {}
+
+private:
+ grpc::Status
+ requestLookup(grpc::ServerContext *Context,
+ const remote::LookupRequest *Request,
+ grpc::ServerWriter<remote::LookupReply> *Reply) override {
+ llvm::outs() << "Lookup of symbol with ID " << Request->id() << '\n';
+ LookupRequest Req;
+ auto SID = SymbolID::fromStr(Request->id());
+ if (!SID) {
+ llvm::outs() << llvm::toString(SID.takeError()) << "\n";
+ return grpc::Status::CANCELLED;
+ }
+ Req.IDs.insert(*SID);
+ Index->lookup(Req, [&](const Symbol &Sym) {
+ remote::LookupReply NextSymbol;
+ NextSymbol.set_symbol_yaml(toYAML(Sym));
+ Reply->Write(NextSymbol);
+ });
+ return grpc::Status::OK;
+ }
+
+ std::unique_ptr<SymbolIndex> Index;
+};
+
+void runServer(std::unique_ptr<SymbolIndex> Index,
+ const std::string &ServerAddress) {
+ RemoteIndexServer Service(std::move(Index));
+
+ grpc::EnableDefaultHealthCheckService(true);
+ grpc::ServerBuilder Builder;
+ Builder.AddListeningPort(ServerAddress, grpc::InsecureServerCredentials());
+ Builder.RegisterService(&Service);
+ std::unique_ptr<grpc::Server> Server(Builder.BuildAndStart());
+ llvm::outs() << "Server listening on " << ServerAddress << '\n';
+
+ Server->Wait();
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
+
+int main(int argc, char *argv[]) {
+ using namespace clang::clangd;
+ llvm::cl::ParseCommandLineOptions(argc, argv, clang::clangd::Overview);
+ llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
+
+ std::unique_ptr<SymbolIndex> Index = openIndex(IndexPath);
+
+ if (!Index) {
+ llvm::outs() << "Failed to open the index.\n";
+ return -1;
+ }
+
+ runServer(std::move(Index), ServerAddress);
+}
Index: clang-tools-extra/clangd/index/remote/server/CMakeLists.txt
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/index/remote/server/CMakeLists.txt
@@ -0,0 +1,16 @@
+add_clang_executable(clangd-index-server
+ Server.cpp
+ )
+target_compile_definitions(clangd-index-server PRIVATE -DGOOGLE_PROTOBUF_NO_RTTI=1)
+clang_target_link_libraries(clangd-index-server
+ PRIVATE
+ clangBasic
+ )
+target_link_libraries(clangd-index-server
+ PRIVATE
+ RemoteIndexProtos
+
+ protobuf
+ grpc++
+ clangDaemon
+ )
Index: clang-tools-extra/clangd/index/remote/client/Client.cpp
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/index/remote/client/Client.cpp
@@ -0,0 +1,91 @@
+//===--- Client.cpp - Remote Index Client -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a simple interactive tool which can be used to manually
+// evaluate symbol search quality of Clangd index.
+//
+//===----------------------------------------------------------------------===//
+
+#include "SourceCode.h"
+#include "index/Serialization.h"
+#include "index/dex/Dex.h"
+#include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/LineEditor/LineEditor.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Signals.h"
+
+#include "grpcpp/grpcpp.h"
+
+#include "Index.grpc.pb.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+llvm::cl::opt<std::string>
+ ServerAddress("server-address",
+ llvm::cl::desc("Address of remote index server to use."),
+ llvm::cl::init("0.0.0.0:50051"));
+
+static const std::string Overview = R"(
+This is an **experimental** interactive tool to process user-provided search
+queries over given symbol collection obtained via clangd-indexer with the help
+of remote index server. The client will connect to remote index server and pass
+it lookup queries.
+)";
+
+class RemoteIndexClient {
+public:
+ RemoteIndexClient(std::shared_ptr<grpc::Channel> Channel)
+ : Stub(remote::Index::NewStub(Channel)) {}
+
+ void lookup(llvm::StringRef ID) {
+ llvm::outs() << "Lookup of symbol with ID " << ID << '\n';
+ remote::LookupRequest Proto;
+ Proto.set_id(ID.str());
+
+ grpc::ClientContext Context;
+ remote::LookupReply Reply;
+ std::unique_ptr<grpc::ClientReader<remote::LookupReply>> Reader(
+ Stub->requestLookup(&Context, Proto));
+ while (Reader->Read(&Reply)) {
+ llvm::outs() << Reply.symbol_yaml();
+ }
+ grpc::Status Status = Reader->Finish();
+ if (Status.ok()) {
+ llvm::outs() << "lookupRequest rpc succeeded.\n";
+ } else {
+ llvm::outs() << "lookupRequest rpc failed.\n";
+ }
+ }
+
+private:
+ std::unique_ptr<remote::Index::Stub> Stub;
+};
+
+} // namespace
+} // namespace clangd
+} // namespace clang
+
+int main(int argc, const char *argv[]) {
+ using namespace clang::clangd;
+
+ llvm::cl::ParseCommandLineOptions(argc, argv, Overview);
+ llvm::cl::ResetCommandLineParser(); // We reuse it for REPL commands.
+ llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
+
+ RemoteIndexClient IndexClient(
+ grpc::CreateChannel(ServerAddress, grpc::InsecureChannelCredentials()));
+
+ llvm::LineEditor LE("remote-index-client");
+ while (llvm::Optional<std::string> Request = LE.readLine())
+ IndexClient.lookup(std::move(*Request));
+}
Index: clang-tools-extra/clangd/index/remote/client/CMakeLists.txt
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/index/remote/client/CMakeLists.txt
@@ -0,0 +1,15 @@
+add_clang_executable(clangd-index-client
+ Client.cpp
+ )
+target_compile_definitions(clangd-index-client PRIVATE -DGOOGLE_PROTOBUF_NO_RTTI=1)
+clang_target_link_libraries(clangd-index-client
+ PRIVATE
+ clangBasic
+ )
+target_link_libraries(clangd-index-client
+ PRIVATE
+ RemoteIndexProtos
+
+ protobuf
+ grpc++
+ )
Index: clang-tools-extra/clangd/index/remote/README.md
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/index/remote/README.md
@@ -0,0 +1,114 @@
+# Clangd remote index
+
+Clangd global index used for code completion, code navigation and a number of
+useful features takes a long time to build (3-4 hours on powerful machines for
+Chrome-sized projects) and induces a significant memory overhead (up to 2 GB)
+to serve. As a result, developers in large open source projects like LLVM often
+don't get the best experience for editing and navigating source code.
+
+To relieve that burden, we're building remote index — a global index
+served on a different machine. This directory contains code that is used as
+Proof of Concept for the upcoming remote index feature.
+
+## Building
+
+This feature uses gRPC and Protobuf libraries, so you will need to install them.
+There are two ways of doing that.
+
+Regardless of the dependencies installation way, to enable this feature and
+build remote index tools you will need to set this CMake flag —
+`-DCLANGD_ENABLE_REMOTE=On`.
+
+### System-installed libraries
+
+On Debian-like systems gRPC and Protobuf can be installed from apt:
+
+```bash
+apt install libgrpc++-dev libprotobuf-dev protobuf-compiler protobuf-compiler-grpc
+```
+
+### Building from sources
+
+Another way of installing gRPC and Protobuf is building from sources using
+CMake. The easiest way of doing that would be to choose a directory where you
+want to install so that the installation files are not copied to system root and
+you can uninstall gRPC or use different versions of the library.
+
+```bash
+# Get source code.
+$ git clone -b v1.28.1 https://github.com/grpc/grpc
+$ cd grpc
+$ git submodule update --init
+# Choose directory where you want gRPC installation to live.
+$ export GRPC_INSTALL_PATH=/where/you/want/grpc/to/be/installed
+# Build and install gRPC to ${GRPC_INSTALL_PATH}
+$ mkdir build; cd build
+$ cmake -DgRPC_INSTALL=ON -DCMAKE_INSTALL_PREFIX=${GRPC_INSTALL_PATH} -DCMAKE_BUILD_TYPE=Release ..
+$ make install
+```
+
+This [guide](https://github.com/grpc/grpc/blob/master/BUILDING.md) goes into
+more detail on how to build gRPC from sources.
+
+After installation, you need to
+
+By default, CMake will look for system-installed libraries when building remote
+index tools so you will have to adjust LLVM's CMake invocation. The following
+flag will inform build system that you chose this option —
+`-DGRPC_INSTALL_PATH=${GRPC_INSTALL_PATH}`.
+
+## Running
+
+Right now it is only possible to connect to the index service hosted on the
+same machine — `clangd-remote-server`. `clangd-remote-index` is a simple
+`dexp`-like tool that passes lookup requests to the server. The easiest way to
+try it out is running `clangd-remote-index` and `clangd-remote-server` in two
+different processes by opening two terminals:
+
+Terminal one:
+
+```bash
+# Get ID of some symbol from the index.
+$ ./bin/dexp /tmp/llvm.dex -c="lookup -name clang::clangd"
+---
+ID: 43C8113D5731B053
+Name: clangd
+Scope: 'clang::'
+...
+$ ./bin/clangd-index-server /tmp/llvm.dex
+```
+
+Terminal two:
+
+```bash
+$ ./bin/clangd-index-client
+# Paste ID of the symbol from dexp lookup.
+remote-index-client > 43C8113D5731B053
+Lookup of symbol with ID 43C8113D5731B053
+---
+ID: 43C8113D5731B053
+Name: clangd
+Scope: 'clang::'
+SymInfo:
+ Kind: Namespace
+ Lang: Cpp
+CanonicalDeclaration:
+ FileURI: 'file:///usr/local/google/home/kbobyrev/dev/src/llvm-project/clang-tools-extra/clangd/FuzzyMatch.h'
+ Start:
+ Line: 24
+ Column: 10
+ End:
+ Line: 24
+ Column: 16
+References: 25
+Origin: 12
+Flags: 9
+Signature: ''
+TemplateSpecializationArgs: ''
+CompletionSnippetSuffix: ''
+Documentation: ''
+ReturnType: ''
+Type: ''
+...
+lookupRequest rpc succeeded.
+```
Index: clang-tools-extra/clangd/index/remote/Index.proto
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/index/remote/Index.proto
@@ -0,0 +1,19 @@
+//===--- Index.proto - Remote index Protocol Buffers definition -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+syntax = "proto3";
+
+package clang.clangd.remote;
+
+service Index {
+ rpc requestLookup(LookupRequest) returns (stream LookupReply) {}
+}
+
+message LookupRequest { string ID = 1; }
+
+message LookupReply { string Symbol_YAML = 1; }
Index: clang-tools-extra/clangd/index/remote/CMakeLists.txt
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/index/remote/CMakeLists.txt
@@ -0,0 +1,12 @@
+generate_grpc_protos("Index.proto" RemoteIndexProtos)
+
+include_directories("${CMAKE_CURRENT_BINARY_DIR}")
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../)
+
+set(LLVM_LINK_COMPONENTS
+ LineEditor
+ Support
+ )
+
+add_subdirectory(client)
+add_subdirectory(server)
Index: clang-tools-extra/clangd/CMakeLists.txt
===================================================================
--- clang-tools-extra/clangd/CMakeLists.txt
+++ clang-tools-extra/clangd/CMakeLists.txt
@@ -140,7 +140,6 @@
endif()
add_subdirectory(tool)
add_subdirectory(indexer)
-add_subdirectory(index/dex/dexp)
if (LLVM_INCLUDE_BENCHMARKS)
add_subdirectory(benchmarks)
@@ -153,3 +152,11 @@
add_subdirectory(test)
add_subdirectory(unittests)
endif()
+
+# This setup requires gRPC to be built from sources using CMake and installed to
+# ${GRPC_INSTALL_PATH} via -DCMAKE_INSTALL_PREFIX=${GRPC_INSTALL_PATH}.
+if (CLANGD_ENABLE_REMOTE)
+ add_subdirectory(index/remote)
+endif()
+
+add_subdirectory(index/dex/dexp)
_______________________________________________
cfe-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
