kbobyrev updated this revision to Diff 257270. kbobyrev marked 6 inline comments as done. kbobyrev added a comment.
Improve CMake infrastructure. - Move client and server into separate directories and enable LLVM's error logic - Address a bunch of comments regarding flags enabling gRPC & remote index build - Simplify protobuf generation script Repository: rG LLVM Github Monorepo CHANGES SINCE LAST ACTION https://reviews.llvm.org/D77794/new/ https://reviews.llvm.org/D77794 Files: clang-tools-extra/clangd/CMakeLists.txt clang-tools-extra/clangd/index/remote/CMakeLists.txt clang-tools-extra/clangd/index/remote/Index.proto clang-tools-extra/clangd/index/remote/client/CMakeLists.txt clang-tools-extra/clangd/index/remote/client/Client.cpp clang-tools-extra/clangd/index/remote/server/CMakeLists.txt clang-tools-extra/clangd/index/remote/server/Server.cpp llvm/CMakeLists.txt
Index: llvm/CMakeLists.txt =================================================================== --- llvm/CMakeLists.txt +++ llvm/CMakeLists.txt @@ -1107,3 +1107,8 @@ if (LLVM_INCLUDE_UTILS AND LLVM_INCLUDE_TOOLS) add_subdirectory(utils/llvm-locstats) endif() + +# FIXME(kirillbobyrev): Document this in the LLVM docs. +option(LLVM_USE_GRPC "Use gRPC library to enable remote index support for Clangd" OFF) +# FIXME(kbobyrev): Check if it works with optimized tablegen CMake option. +set(GRPC_INSTALL_PATH "" CACHE PATH "Path to gRPC library installation.") Index: clang-tools-extra/clangd/index/remote/server/Server.cpp =================================================================== --- /dev/null +++ clang-tools-extra/clangd/index/remote/server/Server.cpp @@ -0,0 +1,125 @@ +//===--- Server.cpp - gRPC-based Remote Index Server ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "index/Index.h" +#include "index/Serialization.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/LineEditor/LineEditor.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Signals.h" + +#include "grpcpp/grpcpp.h" +#include "grpcpp/health_check_service_interface.h" + +#include "Index.grpc.pb.h" + +namespace clang { +namespace clangd { +namespace { + +static const std::string Overview = R"( +This is an experimental remote index implementation. The server opens Dex and +awaits gRPC lookup requests from the client. +)"; + +llvm::cl::opt<std::string> IndexPath(llvm::cl::desc("[PATH TO INDEX]"), + llvm::cl::Positional, llvm::cl::Required); + +llvm::cl::opt<std::string> ServerAddress("server-address", + llvm::cl::init("0.0.0.0:50051")); + +std::unique_ptr<SymbolIndex> openIndex(llvm::StringRef Index) { + return loadIndex(Index, /*UseIndex=*/true); +} + +// FIXME(kbobyrev): This is copied +std::vector<SymbolID> getSymbolIDsFromIndex(llvm::StringRef QualifiedName, + const SymbolIndex *Index) { + FuzzyFindRequest Request; + // Remove leading "::" qualifier as FuzzyFind doesn't need leading "::" + // qualifier for global scope. + bool IsGlobalScope = QualifiedName.consume_front("::"); + auto Names = splitQualifiedName(QualifiedName); + if (IsGlobalScope || !Names.first.empty()) + Request.Scopes = {std::string(Names.first)}; + else + // QualifiedName refers to a symbol in global scope (e.g. "GlobalSymbol"), + // add the global scope to the request. + Request.Scopes = {""}; + + Request.Query = std::string(Names.second); + std::vector<SymbolID> SymIDs; + Index->fuzzyFind(Request, [&](const Symbol &Sym) { + std::string SymQualifiedName = (Sym.Scope + Sym.Name).str(); + if (QualifiedName == SymQualifiedName) + SymIDs.push_back(Sym.ID); + }); + return SymIDs; +} + +class RemoteIndexServer final : public remote::Index::Service { +public: + RemoteIndexServer(std::unique_ptr<SymbolIndex> Index) + : Index(std::move(Index)) {} + +private: + grpc::Status + requestLookup(grpc::ServerContext *Context, + const remote::LookupRequest *Request, + grpc::ServerWriter<remote::LookupReply> *Reply) override { + llvm::outs() << "Lookup of symbol with ID " << Request->name() << '\n'; + LookupRequest Req; + for (const auto &ID : getSymbolIDsFromIndex(Request->name(), Index.get())) { + Req.IDs.insert(ID); + } + Index->lookup(Req, [&](const Symbol &Sym) { + remote::LookupReply NextSymbol; + NextSymbol.set_symbol_yaml(toYAML(Sym)); + Reply->Write(NextSymbol); + }); + return grpc::Status::OK; + } + + std::unique_ptr<SymbolIndex> Index; +}; + +void runServer(std::unique_ptr<SymbolIndex> Index, + const std::string &ServerAddress) { + RemoteIndexServer Service(std::move(Index)); + + grpc::EnableDefaultHealthCheckService(true); + grpc::ServerBuilder Builder; + Builder.AddListeningPort(ServerAddress, grpc::InsecureServerCredentials()); + Builder.RegisterService(&Service); + std::unique_ptr<grpc::Server> Server(Builder.BuildAndStart()); + llvm::outs() << "Server listening on " << ServerAddress << '\n'; + + Server->Wait(); +} + +} // namespace +} // namespace clangd +} // namespace clang + +int main(int argc, char *argv[]) { + using namespace clang::clangd; + llvm::cl::ParseCommandLineOptions(argc, argv, clang::clangd::Overview); + llvm::sys::PrintStackTraceOnErrorSignal(argv[0]); + + std::unique_ptr<SymbolIndex> Index = openIndex(IndexPath); + + if (!Index) { + llvm::outs() << "Failed to open the index.\n"; + return -1; + } + + runServer(std::move(Index), ServerAddress); +} Index: clang-tools-extra/clangd/index/remote/server/CMakeLists.txt =================================================================== --- /dev/null +++ clang-tools-extra/clangd/index/remote/server/CMakeLists.txt @@ -0,0 +1,15 @@ +add_clang_executable(clangd-index-server + Server.cpp + ) +clang_target_link_libraries(clangd-index-server + PRIVATE + clangBasic + ) +target_link_libraries(clangd-index-server + PRIVATE + gRPCProtos + + protobuf::libprotobuf + gRPC::grpc++ + clangDaemon + ) Index: clang-tools-extra/clangd/index/remote/client/Client.cpp =================================================================== --- /dev/null +++ clang-tools-extra/clangd/index/remote/client/Client.cpp @@ -0,0 +1,91 @@ +//===--- Client.cpp - Remote Index Client -----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements a simple interactive tool which can be used to manually +// evaluate symbol search quality of Clangd index. +// +//===----------------------------------------------------------------------===// + +#include "SourceCode.h" +#include "index/Serialization.h" +#include "index/dex/Dex.h" +#include "llvm/ADT/ScopeExit.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/LineEditor/LineEditor.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Signals.h" + +#include "grpcpp/grpcpp.h" + +#include "Index.grpc.pb.h" + +namespace clang { +namespace clangd { +namespace { + +llvm::cl::opt<std::string> + ServerAddress("server-address", + llvm::cl::desc("Address of remote index server to use."), + llvm::cl::init("0.0.0.0:50051")); + +static const std::string Overview = R"( +This is an **experimental** interactive tool to process user-provided search +queries over given symbol collection obtained via clangd-indexer with the help +of remote index server. The client will connect to remote index server and pass +it lookup queries. +)"; + +class RemoteIndexClient { +public: + RemoteIndexClient(std::shared_ptr<grpc::Channel> Channel) + : Stub(remote::Index::NewStub(Channel)) {} + + void lookup(llvm::StringRef Name) { + llvm::outs() << "Lookup of symbol with Name " << Name << '\n'; + remote::LookupRequest Proto; + Proto.set_name(Name.str()); + + grpc::ClientContext Context; + remote::LookupReply Reply; + std::unique_ptr<grpc::ClientReader<remote::LookupReply>> Reader( + Stub->requestLookup(&Context, Proto)); + while (Reader->Read(&Reply)) { + llvm::outs() << Reply.symbol_yaml(); + } + grpc::Status Status = Reader->Finish(); + if (Status.ok()) { + llvm::outs() << "lookupRequest rpc succeeded.\n"; + } else { + llvm::outs() << "lookupRequest rpc failed.\n"; + } + } + +private: + std::unique_ptr<remote::Index::Stub> Stub; +}; + +} // namespace +} // namespace clangd +} // namespace clang + +int main(int argc, const char *argv[]) { + using namespace clang::clangd; + + llvm::cl::ParseCommandLineOptions(argc, argv, Overview); + llvm::cl::ResetCommandLineParser(); // We reuse it for REPL commands. + llvm::sys::PrintStackTraceOnErrorSignal(argv[0]); + + RemoteIndexClient IndexClient( + grpc::CreateChannel(ServerAddress, grpc::InsecureChannelCredentials())); + + llvm::LineEditor LE("remote-index-client"); + while (llvm::Optional<std::string> Request = LE.readLine()) + IndexClient.lookup(std::move(*Request)); +} Index: clang-tools-extra/clangd/index/remote/client/CMakeLists.txt =================================================================== --- /dev/null +++ clang-tools-extra/clangd/index/remote/client/CMakeLists.txt @@ -0,0 +1,14 @@ +add_clang_executable(clangd-index-client + Client.cpp + ) +clang_target_link_libraries(clangd-index-client + PRIVATE + clangBasic + ) +target_link_libraries(clangd-index-client + PRIVATE + gRPCProtos + + protobuf::libprotobuf + gRPC::grpc++ + ) Index: clang-tools-extra/clangd/index/remote/Index.proto =================================================================== --- /dev/null +++ clang-tools-extra/clangd/index/remote/Index.proto @@ -0,0 +1,19 @@ +//===--- Index.proto - Remote index Protocol Buffers definition -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +syntax = "proto3"; + +package remote; + +service Index { + rpc requestLookup(LookupRequest) returns (stream LookupReply) {} +} + +message LookupRequest { string Name = 1; } + +message LookupReply { string Symbol_YAML = 1; } Index: clang-tools-extra/clangd/index/remote/CMakeLists.txt =================================================================== --- /dev/null +++ clang-tools-extra/clangd/index/remote/CMakeLists.txt @@ -0,0 +1,30 @@ +get_filename_component(IndexProto "Index.proto" ABSOLUTE) +get_filename_component(IndexProtoPath "${IndexProto}" PATH) + +set(Index_proto_source "${CMAKE_CURRENT_BINARY_DIR}/Index.pb.cc") +set(Index_proto_header "${CMAKE_CURRENT_BINARY_DIR}/Index.pb.h") +set(Index_grpc_source "${CMAKE_CURRENT_BINARY_DIR}/Index.grpc.pb.cc") +set(Index_grpc_header "${CMAKE_CURRENT_BINARY_DIR}/Index.grpc.pb.h") +add_custom_command( + OUTPUT "${Index_proto_source}" "${Index_proto_header}" "${Index_grpc_source}" "${Index_grpc_header}" + COMMAND ${GRPC_INSTALL_PATH}/bin/protoc + ARGS --grpc_out="${CMAKE_CURRENT_BINARY_DIR}" + --cpp_out="${CMAKE_CURRENT_BINARY_DIR}" + --proto_path="${IndexProtoPath}" + --plugin=protoc-gen-grpc="${GRPC_CPP_PLUGIN}" + "${IndexProto}" + DEPENDS "${IndexProto}") + +add_library(gRPCProtos ${Index_proto_source} ${Index_grpc_header} ${Index_grpc_source} ${Index_grpc_header}) +target_link_libraries(gRPCProtos gRPC::grpc++ protobuf::libprotobuf) + +include_directories("${CMAKE_CURRENT_BINARY_DIR}") +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../) + +set(LLVM_LINK_COMPONENTS + LineEditor + Support + ) + +add_subdirectory(client) +add_subdirectory(server) Index: clang-tools-extra/clangd/CMakeLists.txt =================================================================== --- clang-tools-extra/clangd/CMakeLists.txt +++ clang-tools-extra/clangd/CMakeLists.txt @@ -140,7 +140,6 @@ endif() add_subdirectory(tool) add_subdirectory(indexer) -add_subdirectory(index/dex/dexp) if (LLVM_INCLUDE_BENCHMARKS) add_subdirectory(benchmarks) @@ -153,3 +152,20 @@ add_subdirectory(test) add_subdirectory(unittests) endif() + +# FIXME(kirillbobyrev): Move the gRPC setup logic to llvm/CMakeLists.txt +if (LLVM_USE_GRPC) + set(protobuf_MODULE_COMPATIBLE TRUE) + find_package(Protobuf CONFIG REQUIRED HINTS ${GRPC_INSTALL_PATH}) + message(STATUS "Using protobuf ${protobuf_VERSION}") + find_package(gRPC CONFIG REQUIRED HINTS ${GRPC_INSTALL_PATH}) + message(STATUS "Using gRPC ${gRPC_VERSION}") + + set(GRPC_CPP_PLUGIN $<TARGET_FILE:gRPC::grpc_cpp_plugin>) + + add_definitions(-DGOOGLE_PROTOBUF_NO_RTTI=1) + include_directories(${Protobuf_INCLUDE_DIRS}) + add_subdirectory(index/remote) +endif() + +add_subdirectory(index/dex/dexp)
_______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
