kbobyrev updated this revision to Diff 275613.
kbobyrev marked 19 inline comments as done.
kbobyrev added a comment.
Store the progress so that I don't lose it. Still WIP.
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D82938/new/
https://reviews.llvm.org/D82938
Files:
clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
clang-tools-extra/clangd/index/remote/Client.cpp
clang-tools-extra/clangd/index/remote/Client.h
clang-tools-extra/clangd/index/remote/Index.proto
clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
clang-tools-extra/clangd/index/remote/server/Server.cpp
clang-tools-extra/clangd/index/remote/unimplemented/UnimplementedClient.cpp
clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
Index: clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
===================================================================
--- clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
+++ clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
@@ -7,8 +7,10 @@
//===----------------------------------------------------------------------===//
#include "../TestTU.h"
+#include "TestFS.h"
#include "index/Serialization.h"
#include "index/remote/marshalling/Marshalling.h"
+#include "llvm/ADT/SmallString.h"
#include "llvm/Support/StringSaver.h"
#include "gtest/gtest.h"
@@ -40,8 +42,9 @@
llvm::BumpPtrAllocator Arena;
llvm::UniqueStringSaver Strings(Arena);
for (auto &Sym : Symbols) {
- const auto ProtobufMeessage = toProtobuf(Sym);
- const auto SymToProtobufAndBack = fromProtobuf(ProtobufMeessage, &Strings);
+ const auto ProtobufMessage = toProtobuf(Sym, "/");
+ const auto SymToProtobufAndBack = fromProtobuf(
+ ProtobufMessage, &Strings, std::string(testRoot()), "unittest");
EXPECT_TRUE(SymToProtobufAndBack.hasValue());
EXPECT_EQ(toYAML(Sym), toYAML(*SymToProtobufAndBack));
}
@@ -80,12 +83,46 @@
EXPECT_GE(References.numRefs(), 5UL);
for (const auto &SymbolWithRefs : References) {
for (const auto &Ref : SymbolWithRefs.second) {
- const auto RefToProtobufAndBack = fromProtobuf(toProtobuf(Ref), &Strings);
+ const auto RefToProtobufAndBack = fromProtobuf(
+ toProtobuf(Ref, "/"), &Strings, std::string(testRoot()), "unittest");
EXPECT_TRUE(RefToProtobufAndBack.hasValue());
EXPECT_EQ(toYAML(Ref), toYAML(*RefToProtobufAndBack));
}
}
-} // namespace
+}
+
+TEST(RemoteMarshallingTest, URITranslation) {
+ llvm::BumpPtrAllocator Arena;
+ llvm::UniqueStringSaver Strings(Arena);
+ clangd::Ref Original;
+ const std::string RemoteIndexPrefix = testPath("remote/machine/project/");
+ std::string RelativePath = "llvm-project/clang-tools-extra/clangd/"
+ "unittests/remote/MarshallingTests.cpp";
+ llvm::SmallString<256> NativePath = StringRef(RelativePath);
+ llvm::sys::path::native(NativePath);
+ RelativePath = std::string(NativePath);
+ const auto URI = URI::createFile(RemoteIndexPrefix + RelativePath);
+ Original.Location.FileURI = Strings.save(URI.toString()).begin();
+ Ref Serialized = toProtobuf(Original, RemoteIndexPrefix);
+ EXPECT_EQ(Serialized.location().file_path(), RelativePath);
+ const std::string LocalIndexPrefix = testPath("local/machine/project/");
+ auto Deserialized = fromProtobuf(Serialized, &Strings, LocalIndexPrefix);
+ EXPECT_TRUE(Deserialized);
+ EXPECT_EQ(Deserialized->Location.FileURI,
+ URI::createFile(LocalIndexPrefix + RelativePath).toString());
+
+ clangd::Ref WithInvalidURI;
+ WithInvalidURI.Location.FileURI = "This is not a URI";
+ Serialized = toProtobuf(WithInvalidURI, RemoteIndexPrefix);
+ // Invalid URI results in empty path.
+ EXPECT_EQ(Serialized.location().file_path(), "");
+
+ Ref WithAbsolutePath;
+ *WithAbsolutePath.mutable_location()->mutable_file_path() = "/usr/local/bin/";
+ Deserialized = fromProtobuf(WithAbsolutePath, &Strings, LocalIndexPrefix);
+ // Can not build URI given absolute path.
+ EXPECT_EQ(std::string(Deserialized->Location.FileURI), "");
+}
} // namespace
} // namespace remote
Index: clang-tools-extra/clangd/index/remote/unimplemented/UnimplementedClient.cpp
===================================================================
--- clang-tools-extra/clangd/index/remote/unimplemented/UnimplementedClient.cpp
+++ clang-tools-extra/clangd/index/remote/unimplemented/UnimplementedClient.cpp
@@ -8,12 +8,14 @@
#include "index/remote/Client.h"
#include "support/Logger.h"
+#include "llvm/ADT/StringRef.h"
namespace clang {
namespace clangd {
namespace remote {
-std::unique_ptr<clangd::SymbolIndex> getClient(llvm::StringRef Address) {
+std::unique_ptr<clangd::SymbolIndex> getClient(llvm::StringRef Address,
+ llvm::StringRef IndexRoot) {
elog("Can't create SymbolIndex client without Remote Index support.");
return nullptr;
}
Index: clang-tools-extra/clangd/index/remote/server/Server.cpp
===================================================================
--- clang-tools-extra/clangd/index/remote/server/Server.cpp
+++ clang-tools-extra/clangd/index/remote/server/Server.cpp
@@ -31,6 +31,9 @@
llvm::cl::opt<std::string> IndexPath(llvm::cl::desc("<INDEX FILE>"),
llvm::cl::Positional, llvm::cl::Required);
+llvm::cl::opt<std::string> IndexRoot(llvm::cl::desc("<PROJECT ROOT>"),
+ llvm::cl::Positional, llvm::cl::Required);
+
llvm::cl::opt<std::string> ServerAddress(
"server-address", llvm::cl::init("0.0.0.0:50051"),
llvm::cl::desc("Address of the invoked server. Defaults to 0.0.0.0:50051"));
@@ -57,7 +60,7 @@
}
Index->lookup(Req, [&](const clangd::Symbol &Sym) {
LookupReply NextMessage;
- *NextMessage.mutable_stream_result() = toProtobuf(Sym);
+ *NextMessage.mutable_stream_result() = toProtobuf(Sym, IndexRoot);
Reply->Write(NextMessage);
});
LookupReply LastMessage;
@@ -72,7 +75,7 @@
const auto Req = fromProtobuf(Request);
bool HasMore = Index->fuzzyFind(Req, [&](const clangd::Symbol &Sym) {
FuzzyFindReply NextMessage;
- *NextMessage.mutable_stream_result() = toProtobuf(Sym);
+ *NextMessage.mutable_stream_result() = toProtobuf(Sym, IndexRoot);
Reply->Write(NextMessage);
});
FuzzyFindReply LastMessage;
@@ -92,7 +95,7 @@
}
bool HasMore = Index->refs(Req, [&](const clangd::Ref &Reference) {
RefsReply NextMessage;
- *NextMessage.mutable_stream_result() = toProtobuf(Reference);
+ *NextMessage.mutable_stream_result() = toProtobuf(Reference, IndexRoot);
Reply->Write(NextMessage);
});
RefsReply LastMessage;
Index: clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
===================================================================
--- clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
+++ clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
@@ -15,6 +15,7 @@
#include "Index.pb.h"
#include "index/Index.h"
+#include "llvm/ADT/StringRef.h"
#include "llvm/Support/StringSaver.h"
namespace clang {
@@ -22,17 +23,28 @@
namespace remote {
clangd::FuzzyFindRequest fromProtobuf(const FuzzyFindRequest *Request);
+/// Deserializes clangd symbols and translates relative paths into
+/// machine-native URIs.
llvm::Optional<clangd::Symbol> fromProtobuf(const Symbol &Message,
- llvm::UniqueStringSaver *Strings);
+ llvm::UniqueStringSaver *Strings,
+ llvm::StringRef IndexRoot,
+ llvm::StringRef Scheme = "file");
llvm::Optional<clangd::Ref> fromProtobuf(const Ref &Message,
- llvm::UniqueStringSaver *Strings);
+ llvm::UniqueStringSaver *Strings,
+ llvm::StringRef IndexRoot,
+ llvm::StringRef Scheme = "file");
LookupRequest toProtobuf(const clangd::LookupRequest &From);
FuzzyFindRequest toProtobuf(const clangd::FuzzyFindRequest &From);
RefsRequest toProtobuf(const clangd::RefsRequest &From);
-Ref toProtobuf(const clangd::Ref &From);
-Symbol toProtobuf(const clangd::Symbol &From);
+/// Serializes Clangd and strips \p IndexedIndexRoot from the file paths
+/// because they are specific to the indexing machine. For example,
+/// "/remote/machine/project/lib/HelloWorld.cpp" will be translated into
+/// "/lib/HelloWorld.cpp" because the absolute path would be different on client
+/// machine.
+Ref toProtobuf(const clangd::Ref &From, llvm::StringRef IndexedIndexRoot);
+Symbol toProtobuf(const clangd::Symbol &From, llvm::StringRef IndexedIndexRoot);
} // namespace remote
} // namespace clangd
Index: clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
===================================================================
--- clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
+++ clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
@@ -16,7 +16,10 @@
#include "index/SymbolOrigin.h"
#include "support/Logger.h"
#include "clang/Index/IndexSymbol.h"
+#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
#include "llvm/Support/StringSaver.h"
namespace clang {
@@ -25,6 +28,54 @@
namespace {
+/// Translates \p RelativePath into the absolute path and builds URI for the
+/// user machine. This translation happens on the client side with the
+/// \p RelativePath received from remote index server and \p IndexRoot is
+/// provided by the client.
+std::string relativePathToURI(llvm::StringRef RelativePath,
+ llvm::StringRef IndexRoot,
+ llvm::StringRef Scheme) {
+ if (RelativePath.empty())
+ return "";
+ if (llvm::sys::path::is_absolute(RelativePath)) {
+ elog("{0} is not relative path", RelativePath);
+ return "";
+ }
+ llvm::SmallString<256> FullPath = IndexRoot;
+ llvm::sys::path::append(FullPath, RelativePath);
+ auto Result = URI::create(FullPath, Scheme);
+ if (!Result) {
+ elog("Can not create URI given absolute path {} with scheme {}: {}",
+ FullPath, Scheme, Result.takeError());
+ return "";
+ }
+ return Result->toString();
+}
+
+/// Translates a URI from the server's backing index to a relative path suitable
+/// to send over the wire to the client.
+std::string uriToRelativePath(llvm::StringRef URI,
+ llvm::StringRef IndexedProjectRoot) {
+ if (URI.empty())
+ return "";
+ auto ParsedURI = URI::parse(URI);
+ if (!ParsedURI) {
+ elog("Can not parse URI {0}: {1}", URI, ParsedURI.takeError());
+ return "";
+ }
+ llvm::SmallString<256> Result = ParsedURI->body();
+ if (IndexedProjectRoot.empty())
+ return static_cast<std::string>(Result);
+ if (!llvm::sys::path::replace_path_prefix(Result, IndexedProjectRoot, "")) {
+ elog("Can not get relative path from the URI {} given the index root {}",
+ URI, IndexedProjectRoot);
+ return "";
+ }
+ // Make sure the result has UNIX slashes.
+ return llvm::sys::path::convert_to_slash(Result,
+ llvm::sys::path::Style::posix);
+}
+
clangd::SymbolLocation::Position fromProtobuf(const Position &Message) {
clangd::SymbolLocation::Position Result;
Result.setColumn(static_cast<uint32_t>(Message.column()));
@@ -59,19 +110,25 @@
}
clangd::SymbolLocation fromProtobuf(const SymbolLocation &Message,
- llvm::UniqueStringSaver *Strings) {
+ llvm::UniqueStringSaver *Strings,
+ llvm::StringRef IndexRoot,
+ llvm::StringRef Scheme) {
clangd::SymbolLocation Location;
Location.Start = fromProtobuf(Message.start());
Location.End = fromProtobuf(Message.end());
- Location.FileURI = Strings->save(Message.file_uri()).begin();
+ Location.FileURI =
+ Strings->save(relativePathToURI(Message.file_path(), IndexRoot, Scheme))
+ .begin();
return Location;
}
-SymbolLocation toProtobuf(const clangd::SymbolLocation &Location) {
+SymbolLocation toProtobuf(const clangd::SymbolLocation &Location,
+ llvm::StringRef IndexedProjectRoot) {
remote::SymbolLocation Result;
*Result.mutable_start() = toProtobuf(Location.Start);
*Result.mutable_end() = toProtobuf(Location.End);
- *Result.mutable_file_uri() = Location.FileURI;
+ *Result.mutable_file_path() =
+ uriToRelativePath(Location.FileURI, IndexedProjectRoot);
return Result;
}
@@ -108,7 +165,9 @@
}
llvm::Optional<clangd::Symbol> fromProtobuf(const Symbol &Message,
- llvm::UniqueStringSaver *Strings) {
+ llvm::UniqueStringSaver *Strings,
+ llvm::StringRef IndexRoot,
+ llvm::StringRef Scheme) {
if (!Message.has_info() || !Message.has_definition() ||
!Message.has_canonical_declarattion()) {
elog("Cannot convert Symbol from Protobuf: {}", Message.ShortDebugString());
@@ -117,7 +176,7 @@
clangd::Symbol Result;
auto ID = SymbolID::fromStr(Message.id());
if (!ID) {
- elog("Cannot convert parse SymbolID {} from Protobuf: {}", ID.takeError(),
+ elog("Cannot parse SymbolID {} given Protobuf: {}", ID.takeError(),
Message.ShortDebugString());
return llvm::None;
}
@@ -125,9 +184,10 @@
Result.SymInfo = fromProtobuf(Message.info());
Result.Name = Message.name();
Result.Scope = Message.scope();
- Result.Definition = fromProtobuf(Message.definition(), Strings);
- Result.CanonicalDeclaration =
- fromProtobuf(Message.canonical_declarattion(), Strings);
+ Result.Definition =
+ fromProtobuf(Message.definition(), Strings, IndexRoot, Scheme);
+ Result.CanonicalDeclaration = fromProtobuf(Message.canonical_declarattion(),
+ Strings, IndexRoot, Scheme);
Result.References = Message.references();
Result.Origin = static_cast<clangd::SymbolOrigin>(Message.origin());
Result.Signature = Message.signature();
@@ -144,13 +204,16 @@
}
llvm::Optional<clangd::Ref> fromProtobuf(const Ref &Message,
- llvm::UniqueStringSaver *Strings) {
+ llvm::UniqueStringSaver *Strings,
+ llvm::StringRef IndexRoot,
+ llvm::StringRef Scheme) {
if (!Message.has_location()) {
elog("Cannot convert Ref from Protobuf: {}", Message.ShortDebugString());
return llvm::None;
}
clangd::Ref Result;
- Result.Location = fromProtobuf(Message.location(), Strings);
+ Result.Location =
+ fromProtobuf(Message.location(), Strings, IndexRoot, Scheme);
Result.Kind = static_cast<clangd::RefKind>(Message.kind());
return Result;
}
@@ -188,15 +251,15 @@
return RPCRequest;
}
-Symbol toProtobuf(const clangd::Symbol &From) {
+Symbol toProtobuf(const clangd::Symbol &From, llvm::StringRef IndexRoot) {
Symbol Result;
Result.set_id(From.ID.str());
*Result.mutable_info() = toProtobuf(From.SymInfo);
Result.set_name(From.Name.str());
- *Result.mutable_definition() = toProtobuf(From.Definition);
+ *Result.mutable_definition() = toProtobuf(From.Definition, IndexRoot);
Result.set_scope(From.Scope.str());
*Result.mutable_canonical_declarattion() =
- toProtobuf(From.CanonicalDeclaration);
+ toProtobuf(From.CanonicalDeclaration, IndexRoot);
Result.set_references(From.References);
Result.set_origin(static_cast<uint32_t>(From.Origin));
Result.set_signature(From.Signature.str());
@@ -214,10 +277,10 @@
return Result;
}
-Ref toProtobuf(const clangd::Ref &From) {
+Ref toProtobuf(const clangd::Ref &From, llvm::StringRef IndexRoot) {
Ref Result;
Result.set_kind(static_cast<uint32_t>(From.Kind));
- *Result.mutable_location() = toProtobuf(From.Location);
+ *Result.mutable_location() = toProtobuf(From.Location, IndexRoot);
return Result;
}
Index: clang-tools-extra/clangd/index/remote/Index.proto
===================================================================
--- clang-tools-extra/clangd/index/remote/Index.proto
+++ clang-tools-extra/clangd/index/remote/Index.proto
@@ -99,7 +99,10 @@
message SymbolLocation {
Position start = 1;
Position end = 2;
- string file_uri = 3;
+ // clangd::SymbolLocation storees FileURI, but the protocol transmits only a
+ // part of its body. Because paths are different on the remote and local
+ // machines they will be translated in the marshalling layer.
+ string file_path = 3;
}
message Position {
Index: clang-tools-extra/clangd/index/remote/Client.h
===================================================================
--- clang-tools-extra/clangd/index/remote/Client.h
+++ clang-tools-extra/clangd/index/remote/Client.h
@@ -10,6 +10,7 @@
#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_REMOTE_INDEX_H
#include "index/Index.h"
+#include "llvm/ADT/StringRef.h"
namespace clang {
namespace clangd {
@@ -17,12 +18,16 @@
/// Returns an SymbolIndex client that passes requests to remote index located
/// at \p Address. The client allows synchronous RPC calls.
+/// \p IndexRoot is an absolute path on the local machine to the source tree
+/// described by the remote index. Paths returned by the index will be treated
+/// as relative to this directory.
///
/// This method attempts to resolve the address and establish the connection.
///
/// \returns nullptr if the address is not resolved during the function call or
/// if the project was compiled without Remote Index support.
-std::unique_ptr<clangd::SymbolIndex> getClient(llvm::StringRef Address);
+std::unique_ptr<clangd::SymbolIndex> getClient(llvm::StringRef Address,
+ llvm::StringRef IndexRoot);
} // namespace remote
} // namespace clangd
Index: clang-tools-extra/clangd/index/remote/Client.cpp
===================================================================
--- clang-tools-extra/clangd/index/remote/Client.cpp
+++ clang-tools-extra/clangd/index/remote/Client.cpp
@@ -14,6 +14,7 @@
#include "marshalling/Marshalling.h"
#include "support/Logger.h"
#include "support/Trace.h"
+#include "llvm/ADT/StringRef.h"
namespace clang {
namespace clangd {
@@ -44,7 +45,7 @@
FinalResult = Reply.final_result();
continue;
}
- auto Sym = fromProtobuf(Reply.stream_result(), &Strings);
+ auto Sym = fromProtobuf(Reply.stream_result(), &Strings, ProjectRoot);
if (!Sym)
elog("Received invalid {0}", ReplyT::descriptor()->name());
Callback(*Sym);
@@ -54,8 +55,9 @@
}
public:
- IndexClient(std::shared_ptr<grpc::Channel> Channel)
- : Stub(remote::SymbolIndex::NewStub(Channel)) {}
+ IndexClient(std::shared_ptr<grpc::Channel> Channel,
+ llvm::StringRef ProjectRoot)
+ : Stub(remote::SymbolIndex::NewStub(Channel)), ProjectRoot(ProjectRoot) {}
void lookup(const clangd::LookupRequest &Request,
llvm::function_ref<void(const clangd::Symbol &)> Callback) const {
@@ -84,15 +86,18 @@
private:
std::unique_ptr<remote::SymbolIndex::Stub> Stub;
+ std::string ProjectRoot;
};
} // namespace
-std::unique_ptr<clangd::SymbolIndex> getClient(llvm::StringRef Address) {
+std::unique_ptr<clangd::SymbolIndex> getClient(llvm::StringRef Address,
+ llvm::StringRef ProjectRoot) {
const auto Channel =
grpc::CreateChannel(Address.str(), grpc::InsecureChannelCredentials());
Channel->GetState(true);
- return std::unique_ptr<clangd::SymbolIndex>(new IndexClient(Channel));
+ return std::unique_ptr<clangd::SymbolIndex>(
+ new IndexClient(Channel, ProjectRoot));
}
} // namespace remote
Index: clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
===================================================================
--- clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
+++ clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
@@ -32,6 +32,9 @@
llvm::cl::opt<std::string>
ExecCommand("c", llvm::cl::desc("Command to execute and then exit"));
+llvm::cl::opt<std::string> ProjectRoot("project-root",
+ llvm::cl::desc("Path to the project"));
+
static constexpr char Overview[] = R"(
This is an **experimental** interactive tool to process user-provided search
queries over given symbol collection obtained via clangd-indexer. The
@@ -326,7 +329,8 @@
std::unique_ptr<SymbolIndex> openIndex(llvm::StringRef Index) {
return Index.startswith("remote:")
- ? remote::getClient(Index.drop_front(strlen("remote:")))
+ ? remote::getClient(Index.drop_front(strlen("remote:")),
+ ProjectRoot)
: loadIndex(Index, /*UseDex=*/true);
}
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
