sammccall created this revision.
sammccall added a reviewer: ioeric.
Herald added subscribers: cfe-commits, kadircet, arphaman, jkorous, MaskRay,
ilya-biryukov.
Repository:
rCTE Clang Tools Extra
https://reviews.llvm.org/D52531
Files:
clangd/index/IndexAction.cpp
clangd/index/IndexAction.h
clangd/index/Serialization.cpp
clangd/index/Serialization.h
clangd/index/YAMLSerialization.cpp
clangd/indexer/IndexerMain.cpp
unittests/clangd/SerializationTests.cpp
Index: unittests/clangd/SerializationTests.cpp
===================================================================
--- unittests/clangd/SerializationTests.cpp
+++ unittests/clangd/SerializationTests.cpp
@@ -13,15 +13,18 @@
#include "gmock/gmock.h"
#include "gtest/gtest.h"
+using testing::_;
using testing::AllOf;
+using testing::Pair;
using testing::UnorderedElementsAre;
using testing::UnorderedElementsAreArray;
namespace clang {
namespace clangd {
namespace {
const char *YAML = R"(
---
+!Symbol
ID: 057557CEBF6E6B2DD437FBF60CC58F352D1DF856
Name: 'Foo1'
Scope: 'clang::'
@@ -47,6 +50,7 @@
References: 3
...
---
+!Symbol
ID: 057557CEBF6E6B2DD437FBF60CC58F352D1DF858
Name: 'Foo2'
Scope: 'clang::'
@@ -65,6 +69,18 @@
Signature: '-sig'
CompletionSnippetSuffix: '-snippet'
...
+!Refs
+ID: 057557CEBF6E6B2DD437FBF60CC58F352D1DF856
+References:
+ - Kind: 4
+ Location:
+ FileURI: file:///path/foo.cc
+ Start:
+ Line: 5
+ Column: 3
+ End:
+ Line: 5
+ Column: 8
)";
MATCHER_P(ID, I, "") { return arg.ID == cantFail(SymbolID::fromStr(I)); }
@@ -108,32 +124,56 @@
EXPECT_EQ(Sym2.CanonicalDeclaration.FileURI, "file:///path/bar.h");
EXPECT_FALSE(Sym2.Flags & Symbol::IndexedForCodeCompletion);
EXPECT_TRUE(Sym2.Flags & Symbol::Deprecated);
+
+ ASSERT_TRUE(bool(ParsedYAML->Refs));
+ EXPECT_THAT(*ParsedYAML->Refs,
+ UnorderedElementsAre(
+ Pair(cantFail(SymbolID::fromStr(
+ "057557CEBF6E6B2DD437FBF60CC58F352D1DF856")),
+ testing::SizeIs(1))));
+ auto Ref1 = ParsedYAML->Refs->begin()->second.front();
+ EXPECT_EQ(Ref1.Kind, RefKind::Reference);
+ EXPECT_EQ(Ref1.Location.FileURI, "file:///path/foo.cc");
}
std::vector<std::string> YAMLFromSymbols(const SymbolSlab &Slab) {
std::vector<std::string> Result;
for (const auto &Sym : Slab)
Result.push_back(toYAML(Sym));
return Result;
}
+std::vector<std::string> YAMLFromRefs(const RefSlab &Slab) {
+ std::vector<std::string> Result;
+ for (const auto &Sym : Slab)
+ Result.push_back(toYAML(Sym));
+ return Result;
+}
+
TEST(SerializationTest, BinaryConversions) {
auto In = readIndexFile(YAML);
EXPECT_TRUE(bool(In)) << In.takeError();
// Write to binary format, and parse again.
- IndexFileOut Out;
- Out.Symbols = In->Symbols.getPointer();
+ IndexFileOut Out(*In);
Out.Format = IndexFileFormat::RIFF;
std::string Serialized = llvm::to_string(Out);
+ {
+ std::error_code EC;
+ llvm::raw_fd_ostream F("/tmp/foo",EC);
+ F << Serialized;
+ }
auto In2 = readIndexFile(Serialized);
ASSERT_TRUE(bool(In2)) << In.takeError();
- ASSERT_TRUE(In->Symbols);
+ ASSERT_TRUE(In2->Symbols);
+ ASSERT_TRUE(In2->Refs);
// Assert the YAML serializations match, for nice comparisons and diffs.
EXPECT_THAT(YAMLFromSymbols(*In2->Symbols),
UnorderedElementsAreArray(YAMLFromSymbols(*In->Symbols)));
+ EXPECT_THAT(YAMLFromRefs(*In2->Refs),
+ UnorderedElementsAreArray(YAMLFromRefs(*In->Refs)));
}
} // namespace
Index: clangd/indexer/IndexerMain.cpp
===================================================================
--- clangd/indexer/IndexerMain.cpp
+++ clangd/indexer/IndexerMain.cpp
@@ -77,9 +77,10 @@
/// Consume a SymbolSlab build for a file.
virtual void consumeSymbols(SymbolSlab Symbols) = 0;
+ virtual void consumeRefs(RefSlab Refs) = 0;
/// Produce a resulting symbol slab, by combining occurrences of the same
/// symbols across translation units.
- virtual SymbolSlab mergeResults() = 0;
+ virtual std::pair<SymbolSlab, RefSlab> mergeResults() = 0;
};
class SymbolIndexActionFactory : public tooling::FrontendActionFactory {
@@ -91,7 +92,8 @@
CollectorOpts.FallbackDir = AssumedHeaderDir;
return createStaticIndexingAction(
CollectorOpts,
- [&](SymbolSlab S) { Consumer.consumeSymbols(std::move(S)); })
+ [&](SymbolSlab S) { Consumer.consumeSymbols(std::move(S)); },
+ [&](RefSlab S) { Consumer.consumeRefs(std::move(S)); })
.release();
}
@@ -109,8 +111,9 @@
for (const auto &Sym : Symbols)
Executor.getExecutionContext()->reportResult(Sym.ID.str(), toYAML(Sym));
}
+ void consumeRefs(RefSlab) override {}
- SymbolSlab mergeResults() override {
+ std::pair<SymbolSlab, RefSlab> mergeResults() override {
SymbolSlab::Builder UniqueSymbols;
Executor.getToolResults()->forEachResult(
[&](llvm::StringRef Key, llvm::StringRef Value) {
@@ -122,7 +125,7 @@
else
UniqueSymbols.insert(Sym);
});
- return std::move(UniqueSymbols).build();
+ return {std::move(UniqueSymbols).build(), RefSlab()};
}
private:
@@ -136,21 +139,29 @@
void consumeSymbols(SymbolSlab Symbols) override {
std::lock_guard<std::mutex> Lock(Mut);
for (auto &&Sym : Symbols) {
- if (const auto *Existing = Result.find(Sym.ID))
- Result.insert(mergeSymbol(*Existing, Sym));
+ if (const auto *Existing = this->Symbols.find(Sym.ID))
+ this->Symbols.insert(mergeSymbol(*Existing, Sym));
else
- Result.insert(Sym);
+ this->Symbols.insert(Sym);
+ }
+ }
+ void consumeRefs(RefSlab Refs) override {
+ std::lock_guard<std::mutex> Lock(Mut);
+ for (const auto &Sym : Refs) {
+ for (const auto& Ref : Sym.second)
+ this->Refs.insert(Sym.first, Ref);
}
}
- SymbolSlab mergeResults() override {
+ std::pair<SymbolSlab, RefSlab> mergeResults() override {
std::lock_guard<std::mutex> Lock(Mut);
- return std::move(Result).build();
+ return {std::move(Symbols).build(), std::move(Refs).build()};
}
private:
std::mutex Mut;
- SymbolSlab::Builder Result;
+ SymbolSlab::Builder Symbols;
+ RefSlab::Builder Refs;
};
} // namespace
@@ -215,7 +226,8 @@
auto UniqueSymbols = Consumer->mergeResults();
// Output phase: emit result symbols.
clang::clangd::IndexFileOut Out;
- Out.Symbols = &UniqueSymbols;
+ Out.Symbols = &UniqueSymbols.first;
+ Out.Refs = &UniqueSymbols.second;
Out.Format = clang::clangd::Format;
llvm::outs() << Out;
return 0;
Index: clangd/index/YAMLSerialization.cpp
===================================================================
--- clangd/index/YAMLSerialization.cpp
+++ clangd/index/YAMLSerialization.cpp
@@ -6,6 +6,12 @@
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
+//
+// A YAML index file is a sequence of tagged entries.
+// Each entry either encodes a Symbol or the list of references to a symbol
+// (a "ref bundle").
+//
+//===----------------------------------------------------------------------===//
#include "Index.h"
#include "Serialization.h"
@@ -20,10 +26,22 @@
#include <cstdint>
LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Symbol::IncludeHeaderWithReferences)
+LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Ref)
+namespace {
+using RefBundle =
+ std::pair<clang::clangd::SymbolID, std::vector<clang::clangd::Ref>>;
+// This is a pale imitation of std::variant<Symbol, RefBundle>
+struct VariantEntry {
+ llvm::Optional<clang::clangd::Symbol> Symbol;
+ llvm::Optional<RefBundle> Refs;
+};
+}
namespace llvm {
namespace yaml {
+using clang::clangd::Ref;
+using clang::clangd::RefKind;
using clang::clangd::Symbol;
using clang::clangd::SymbolID;
using clang::clangd::SymbolLocation;
@@ -179,31 +197,86 @@
}
};
+template <> struct MappingTraits<RefBundle> {
+ static void mapping(IO &IO, RefBundle &Refs) {
+ MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO,
+ Refs.first);
+ IO.mapRequired("ID", NSymbolID->HexString);
+ IO.mapRequired("References", Refs.second);
+ }
+};
+
+struct NormalizedRefKind {
+ NormalizedRefKind(IO &) {}
+ NormalizedRefKind(IO &, RefKind O) { Kind = static_cast<uint8_t>(O); }
+
+ RefKind denormalize(IO &) { return static_cast<RefKind>(Kind); }
+
+ uint8_t Kind = 0;
+};
+
+template <> struct MappingTraits<Ref> {
+ static void mapping(IO &IO, Ref &R) {
+ MappingNormalization<NormalizedRefKind, RefKind> NKind(IO, R.Kind);
+ IO.mapRequired("Kind", NKind->Kind);
+ IO.mapRequired("Location", R.Location);
+ }
+};
+
+template <> struct MappingTraits<VariantEntry> {
+ static void mapping(IO &IO, VariantEntry &Variant) {
+ if (IO.mapTag("!Symbol", Variant.Symbol.hasValue())) {
+ if (!IO.outputting())
+ Variant.Symbol.emplace();
+ MappingTraits<Symbol>::mapping(IO, *Variant.Symbol);
+ } else if (IO.mapTag("!Refs", Variant.Refs.hasValue())) {
+ if (!IO.outputting())
+ Variant.Refs.emplace();
+ MappingTraits<RefBundle>::mapping(IO, *Variant.Refs);
+ }
+ }
+};
+
} // namespace yaml
} // namespace llvm
namespace clang {
namespace clangd {
void writeYAML(const IndexFileOut &O, raw_ostream &OS) {
llvm::yaml::Output Yout(OS);
- for (Symbol Sym : *O.Symbols) // copy: Yout<< requires mutability.
- Yout << Sym;
+ for (const auto& Sym : *O.Symbols) {
+ VariantEntry Entry;
+ Entry.Symbol = Sym;
+ Yout << Entry;
+ }
+ if (O.Refs)
+ for (auto& Sym : *O.Refs) {
+ VariantEntry Entry;
+ Entry.Refs = Sym;
+ Yout << Entry;
+ }
}
Expected<IndexFileIn> readYAML(StringRef Data) {
SymbolSlab::Builder Symbols;
+ RefSlab::Builder Refs;
llvm::yaml::Input Yin(Data);
do {
- Symbol S;
- Yin >> S;
+ VariantEntry Variant;
+ Yin >> Variant;
if (Yin.error())
return llvm::errorCodeToError(Yin.error());
- Symbols.insert(S);
+ if (Variant.Symbol)
+ Symbols.insert(*Variant.Symbol);
+ if (Variant.Refs)
+ for (const auto& Ref : Variant.Refs->second)
+ Refs.insert(Variant.Refs->first, Ref);
} while (Yin.nextDocument());
IndexFileIn Result;
Result.Symbols.emplace(std::move(Symbols).build());
+ Result.Refs.emplace(std::move(Refs).build());
return std::move(Result);
}
@@ -218,6 +291,17 @@
return Buf;
}
+std::string toYAML(const std::pair<SymbolID, ArrayRef<Ref>> &Data) {
+ RefBundle Refs = {Data.first, Data.second};
+ std::string Buf;
+ {
+ llvm::raw_string_ostream OS(Buf);
+ llvm::yaml::Output Yout(OS);
+ Yout << Refs;
+ }
+ return Buf;
+}
+
Expected<Symbol> symbolFromYAML(llvm::yaml::Input &Yin) {
Symbol S;
Yin >> S;
Index: clangd/index/Serialization.h
===================================================================
--- clangd/index/Serialization.h
+++ clangd/index/Serialization.h
@@ -43,25 +43,29 @@
// Holds the contents of an index file that was read.
struct IndexFileIn {
llvm::Optional<SymbolSlab> Symbols;
+ llvm::Optional<RefSlab> Refs;
};
-// Parse an index file. The input must be a RIFF container chunk.
+// Parse an index file. The input must be a RIFF or YAML file.
llvm::Expected<IndexFileIn> readIndexFile(llvm::StringRef);
// Specifies the contents of an index file to be written.
struct IndexFileOut {
- const SymbolSlab *Symbols;
- // TODO: Support serializing symbol occurrences.
+ const SymbolSlab *Symbols = nullptr;
+ const RefSlab *Refs = nullptr;
// TODO: Support serializing Dex posting lists.
IndexFileFormat Format = IndexFileFormat::RIFF;
IndexFileOut() = default;
IndexFileOut(const IndexFileIn &I)
- : Symbols(I.Symbols ? I.Symbols.getPointer() : nullptr) {}
+ : Symbols(I.Symbols ? I.Symbols.getPointer() : nullptr),
+ Refs(I.Refs ? I.Refs.getPointer() : nullptr) {}
};
// Serializes an index file.
llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const IndexFileOut &O);
std::string toYAML(const Symbol &);
+std::string toYAML(const std::pair<SymbolID, ArrayRef<Ref>> &);
+
// Returned symbol is backed by the YAML input.
// FIXME: this is only needed for IndexerMain, find a better solution.
llvm::Expected<Symbol> symbolFromYAML(llvm::yaml::Input &);
Index: clangd/index/Serialization.cpp
===================================================================
--- clangd/index/Serialization.cpp
+++ clangd/index/Serialization.cpp
@@ -296,6 +296,35 @@
return Sym;
}
+// REFS ENCODING
+// A refs section has data grouped by Symbol. Each symbol has:
+// - SymbolID: 20 bytes
+// - NumRefs: varint
+// - Ref[NumRefs]
+// Fields of Ref are encoded in turn, see implementation.
+
+void writeRefs(const SymbolID &ID, ArrayRef<Ref> Refs,
+ const StringTableOut &Strings, raw_ostream &OS) {
+ OS << ID.raw();
+ writeVar(Refs.size(), OS);
+ for (const auto& Ref : Refs) {
+ OS.write(static_cast<unsigned char>(Ref.Kind));
+ writeLocation(Ref.Location, Strings, OS);
+ }
+}
+
+std::pair<SymbolID, std::vector<Ref>> readRefs(Reader &Data,
+ ArrayRef<StringRef> Strings) {
+ std::pair<SymbolID, std::vector<Ref>> Result;
+ Result.first = Data.consumeID();
+ Result.second.resize(Data.consumeVar());
+ for (auto& Ref : Result.second) {
+ Ref.Kind = static_cast<RefKind>(Data.consume8());
+ Ref.Location= readLocation(Data, Strings);
+ }
+ return Result;
+}
+
// FILE ENCODING
// A file is a RIFF chunk with type 'CdIx'.
// It contains the sections:
@@ -306,7 +335,7 @@
// The current versioning scheme is simple - non-current versions are rejected.
// If you make a breaking change, bump this version number to invalidate stored
// data. Later we may want to support some backward compatibility.
-constexpr static uint32_t Version = 4;
+constexpr static uint32_t Version = 5;
Expected<IndexFileIn> readRIFF(StringRef Data) {
auto RIFF = riff::readFile(Data);
@@ -340,6 +369,18 @@
return makeError("malformed or truncated symbol");
Result.Symbols = std::move(Symbols).build();
}
+ if (Chunks.count("refs")) {
+ Reader RefsReader(Chunks.lookup("refs"));
+ RefSlab::Builder Refs;
+ while (!RefsReader.eof()) {
+ auto RefsBundle = readRefs(RefsReader, Strings->Strings);
+ for (const auto &Ref : RefsBundle.second) // FIXME: bulk insert?
+ Refs.insert(RefsBundle.first, Ref);
+ }
+ if (RefsReader.err())
+ return makeError("malformed or truncated refs");
+ Result.Refs = std::move(Refs).build();
+ }
return std::move(Result);
}
@@ -361,6 +402,14 @@
Symbols.emplace_back(Sym);
visitStrings(Symbols.back(), [&](StringRef &S) { Strings.intern(S); });
}
+ std::vector<std::pair<SymbolID, std::vector<Ref>>> Refs;
+ if (Data.Refs) {
+ for (const auto &Sym : *Data.Refs) {
+ Refs.emplace_back(Sym);
+ for (auto &Ref : Refs.back().second)
+ Strings.intern(Ref.Location.FileURI);
+ }
+ }
std::string StringSection;
{
@@ -377,6 +426,16 @@
}
RIFF.Chunks.push_back({riff::fourCC("symb"), SymbolSection});
+ std::string RefsSection;
+ if (Data.Refs) {
+ {
+ raw_string_ostream RefsOS(RefsSection);
+ for (const auto &Sym : Refs)
+ writeRefs(Sym.first, Sym.second, Strings, RefsOS);
+ }
+ RIFF.Chunks.push_back({riff::fourCC("refs"), RefsSection});
+ }
+
OS << RIFF;
}
@@ -426,6 +485,8 @@
if (auto I = readIndexFile(Buffer->get()->getBuffer())) {
if (I->Symbols)
Symbols = std::move(*I->Symbols);
+ if (I->Refs)
+ Refs = std::move(*I->Refs);
} else {
llvm::errs() << "Bad Index: " << llvm::toString(I.takeError()) << "\n";
return nullptr;
Index: clangd/index/IndexAction.h
===================================================================
--- clangd/index/IndexAction.h
+++ clangd/index/IndexAction.h
@@ -21,10 +21,12 @@
// Only a subset of SymbolCollector::Options are respected:
// - include paths are always collected, and canonicalized appropriately
// - references are always counted
+// - all refs are collected (if RefsCallback is non-null)
// - the symbol origin is always Static
std::unique_ptr<FrontendAction>
createStaticIndexingAction(SymbolCollector::Options Opts,
- std::function<void(SymbolSlab)> SymbolsCallback);
+ std::function<void(SymbolSlab)> SymbolsCallback,
+ std::function<void(RefSlab)> RefsCallback);
} // namespace clangd
} // namespace clang
Index: clangd/index/IndexAction.cpp
===================================================================
--- clangd/index/IndexAction.cpp
+++ clangd/index/IndexAction.cpp
@@ -13,10 +13,11 @@
IndexAction(std::shared_ptr<SymbolCollector> C,
std::unique_ptr<CanonicalIncludes> Includes,
const index::IndexingOptions &Opts,
- std::function<void(SymbolSlab)> &SymbolsCallback)
+ std::function<void(SymbolSlab)> SymbolsCallback,
+ std::function<void(RefSlab)> RefsCallback)
: WrapperFrontendAction(index::createIndexingAction(C, Opts, nullptr)),
- SymbolsCallback(SymbolsCallback), Collector(C),
- Includes(std::move(Includes)),
+ SymbolsCallback(SymbolsCallback), RefsCallback(RefsCallback),
+ Collector(C), Includes(std::move(Includes)),
PragmaHandler(collectIWYUHeaderMaps(this->Includes.get())) {}
std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
@@ -41,10 +42,13 @@
return;
}
SymbolsCallback(Collector->takeSymbols());
+ if (RefsCallback != nullptr)
+ RefsCallback(Collector->takeRefs());
}
private:
std::function<void(SymbolSlab)> SymbolsCallback;
+ std::function<void(RefSlab)> RefsCallback;
std::shared_ptr<SymbolCollector> Collector;
std::unique_ptr<CanonicalIncludes> Includes;
std::unique_ptr<CommentHandler> PragmaHandler;
@@ -54,19 +58,22 @@
std::unique_ptr<FrontendAction>
createStaticIndexingAction(SymbolCollector::Options Opts,
- std::function<void(SymbolSlab)> SymbolsCallback) {
+ std::function<void(SymbolSlab)> SymbolsCallback,
+ std::function<void(RefSlab)> RefsCallback) {
index::IndexingOptions IndexOpts;
IndexOpts.SystemSymbolFilter =
index::IndexingOptions::SystemSymbolFilterKind::All;
Opts.CollectIncludePath = true;
Opts.CountReferences = true;
Opts.Origin = SymbolOrigin::Static;
+ if (RefsCallback != nullptr)
+ Opts.RefFilter = RefKind::All;
auto Includes = llvm::make_unique<CanonicalIncludes>();
addSystemHeadersMapping(Includes.get());
Opts.Includes = Includes.get();
return llvm::make_unique<IndexAction>(
std::make_shared<SymbolCollector>(std::move(Opts)), std::move(Includes),
- IndexOpts, SymbolsCallback);
+ IndexOpts, SymbolsCallback, RefsCallback);
};
} // namespace clangd
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
