sammccall updated this revision to Diff 392612.
sammccall added a comment.
Update remote index server and FileIndex
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D115243/new/
https://reviews.llvm.org/D115243
Files:
clang-tools-extra/clangd/index/BackgroundIndexStorage.cpp
clang-tools-extra/clangd/index/FileIndex.cpp
clang-tools-extra/clangd/index/Serialization.cpp
clang-tools-extra/clangd/index/Serialization.h
clang-tools-extra/clangd/index/SymbolOrigin.cpp
clang-tools-extra/clangd/index/SymbolOrigin.h
clang-tools-extra/clangd/index/YAMLSerialization.cpp
clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
clang-tools-extra/clangd/index/remote/Index.proto
clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
clang-tools-extra/clangd/index/remote/server/Server.cpp
clang-tools-extra/clangd/test/index-serialization/Inputs/sample.idx
clang-tools-extra/clangd/tool/ClangdMain.cpp
clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
clang-tools-extra/clangd/unittests/SerializationTests.cpp
Index: clang-tools-extra/clangd/unittests/SerializationTests.cpp
===================================================================
--- clang-tools-extra/clangd/unittests/SerializationTests.cpp
+++ clang-tools-extra/clangd/unittests/SerializationTests.cpp
@@ -49,7 +49,6 @@
End:
Line: 1
Column: 1
-Origin: 128
Flags: 129
Documentation: 'Foo doc'
ReturnType: 'int'
@@ -121,6 +120,10 @@
return (arg.IncludeHeader == IncludeHeader) && (arg.References == References);
}
+auto readIndexFile(llvm::StringRef Text) {
+ return readIndexFile(Text, SymbolOrigin::Static);
+}
+
TEST(SerializationTest, NoCrashOnEmptyYAML) {
EXPECT_TRUE(bool(readIndexFile("")));
}
@@ -143,7 +146,7 @@
EXPECT_EQ(Sym1.Documentation, "Foo doc");
EXPECT_EQ(Sym1.ReturnType, "int");
EXPECT_EQ(StringRef(Sym1.CanonicalDeclaration.FileURI), "file:///path/foo.h");
- EXPECT_EQ(Sym1.Origin, static_cast<SymbolOrigin>(1 << 7));
+ EXPECT_EQ(Sym1.Origin, SymbolOrigin::Static);
EXPECT_EQ(static_cast<uint8_t>(Sym1.Flags), 129);
EXPECT_TRUE(Sym1.Flags & Symbol::IndexedForCodeCompletion);
EXPECT_FALSE(Sym1.Flags & Symbol::Deprecated);
Index: clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
===================================================================
--- clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
+++ clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
@@ -73,7 +73,8 @@
if (Storage.find(ShardIdentifier) == Storage.end()) {
return nullptr;
}
- auto IndexFile = readIndexFile(Storage[ShardIdentifier]);
+ auto IndexFile =
+ readIndexFile(Storage[ShardIdentifier], SymbolOrigin::Background);
if (!IndexFile) {
ADD_FAILURE() << "Error while reading " << ShardIdentifier << ':'
<< IndexFile.takeError();
Index: clang-tools-extra/clangd/tool/ClangdMain.cpp
===================================================================
--- clang-tools-extra/clangd/tool/ClangdMain.cpp
+++ clang-tools-extra/clangd/tool/ClangdMain.cpp
@@ -586,7 +586,7 @@
auto NewIndex = std::make_unique<SwapIndex>(std::make_unique<MemIndex>());
auto IndexLoadTask = [File = External.Location,
PlaceHolder = NewIndex.get()] {
- if (auto Idx = loadIndex(File, /*UseDex=*/true))
+ if (auto Idx = loadIndex(File, SymbolOrigin::Static, /*UseDex=*/true))
PlaceHolder->reset(std::move(Idx));
};
if (Tasks) {
Index: clang-tools-extra/clangd/index/remote/server/Server.cpp
===================================================================
--- clang-tools-extra/clangd/index/remote/server/Server.cpp
+++ clang-tools-extra/clangd/index/remote/server/Server.cpp
@@ -395,7 +395,8 @@
"{0}, new index was modified at {1}. Attempting to reload.",
LastStatus.getLastModificationTime(), Status->getLastModificationTime());
LastStatus = *Status;
- std::unique_ptr<clang::clangd::SymbolIndex> NewIndex = loadIndex(IndexPath);
+ std::unique_ptr<clang::clangd::SymbolIndex> NewIndex =
+ loadIndex(IndexPath, SymbolOrigin::Static);
if (!NewIndex) {
elog("Failed to load new index. Old index will be served.");
return;
Index: clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
===================================================================
--- clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
+++ clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
@@ -269,7 +269,6 @@
return Declaration.takeError();
*Result.mutable_canonical_declaration() = *Declaration;
Result.set_references(From.References);
- Result.set_origin(static_cast<uint32_t>(From.Origin));
Result.set_signature(From.Signature.str());
Result.set_template_specialization_args(
From.TemplateSpecializationArgs.str());
Index: clang-tools-extra/clangd/index/remote/Index.proto
===================================================================
--- clang-tools-extra/clangd/index/remote/Index.proto
+++ clang-tools-extra/clangd/index/remote/Index.proto
@@ -66,7 +66,7 @@
optional string scope = 5;
optional SymbolLocation canonical_declaration = 6;
optional int32 references = 7;
- optional uint32 origin = 8;
+ reserved 8;
optional string signature = 9;
optional string template_specialization_args = 10;
optional string completion_snippet_suffix = 11;
Index: clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
===================================================================
--- clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
+++ clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
@@ -334,7 +334,8 @@
}
// Auto-detects input format when parsing
- auto IndexIn = clang::clangd::readIndexFile(Buffer->get()->getBuffer());
+ auto IndexIn = clang::clangd::readIndexFile(Buffer->get()->getBuffer(),
+ SymbolOrigin::Static);
if (!IndexIn) {
llvm::errs() << llvm::toString(IndexIn.takeError()) << "\n";
return;
@@ -374,7 +375,7 @@
return Index.startswith("remote:")
? remote::getClient(Index.drop_front(strlen("remote:")),
ProjectRoot)
- : loadIndex(Index, /*UseDex=*/true);
+ : loadIndex(Index, SymbolOrigin::Static, /*UseDex=*/true);
}
bool runCommand(std::string Request, const SymbolIndex &Index) {
Index: clang-tools-extra/clangd/index/YAMLSerialization.cpp
===================================================================
--- clang-tools-extra/clangd/index/YAMLSerialization.cpp
+++ clang-tools-extra/clangd/index/YAMLSerialization.cpp
@@ -69,7 +69,6 @@
using clang::clangd::Symbol;
using clang::clangd::SymbolID;
using clang::clangd::SymbolLocation;
-using clang::clangd::SymbolOrigin;
using clang::index::SymbolInfo;
using clang::index::SymbolKind;
using clang::index::SymbolLanguage;
@@ -108,17 +107,6 @@
uint8_t Flag = 0;
};
-struct NormalizedSymbolOrigin {
- NormalizedSymbolOrigin(IO &) {}
- NormalizedSymbolOrigin(IO &, SymbolOrigin O) {
- Origin = static_cast<uint8_t>(O);
- }
-
- SymbolOrigin denormalize(IO &) { return static_cast<SymbolOrigin>(Origin); }
-
- uint8_t Origin = 0;
-};
-
template <> struct MappingTraits<YPosition> {
static void mapping(IO &IO, YPosition &Value) {
IO.mapRequired("Line", Value.Line);
@@ -194,8 +182,6 @@
MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, Sym.ID);
MappingNormalization<NormalizedSymbolFlag, Symbol::SymbolFlag> NSymbolFlag(
IO, Sym.Flags);
- MappingNormalization<NormalizedSymbolOrigin, SymbolOrigin> NSymbolOrigin(
- IO, Sym.Origin);
IO.mapRequired("ID", NSymbolID->HexString);
IO.mapRequired("Name", Sym.Name);
IO.mapRequired("Scope", Sym.Scope);
@@ -204,7 +190,6 @@
SymbolLocation());
IO.mapOptional("Definition", Sym.Definition, SymbolLocation());
IO.mapOptional("References", Sym.References, 0u);
- IO.mapOptional("Origin", NSymbolOrigin->Origin);
IO.mapOptional("Flags", NSymbolFlag->Flag);
IO.mapOptional("Signature", Sym.Signature);
IO.mapOptional("TemplateSpecializationArgs",
@@ -436,7 +421,8 @@
}
}
-llvm::Expected<IndexFileIn> readYAML(llvm::StringRef Data) {
+llvm::Expected<IndexFileIn> readYAML(llvm::StringRef Data,
+ SymbolOrigin Origin) {
SymbolSlab::Builder Symbols;
RefSlab::Builder Refs;
RelationSlab::Builder Relations;
@@ -453,8 +439,10 @@
if (Yin.error())
return llvm::errorCodeToError(Yin.error());
- if (Variant.Symbol)
+ if (Variant.Symbol) {
+ Variant.Symbol->Origin = Origin;
Symbols.insert(*Variant.Symbol);
+ }
if (Variant.Refs)
for (const auto &Ref : Variant.Refs->second)
Refs.insert(Variant.Refs->first, Ref);
@@ -528,25 +516,5 @@
return Buf;
}
-llvm::Expected<clangd::Symbol>
-symbolFromYAML(StringRef YAML, llvm::UniqueStringSaver *Strings) {
- clangd::Symbol Deserialized;
- llvm::yaml::Input YAMLInput(YAML, Strings);
- if (YAMLInput.error())
- return error("Unable to deserialize Symbol from YAML: {0}", YAML);
- YAMLInput >> Deserialized;
- return Deserialized;
-}
-
-llvm::Expected<clangd::Ref> refFromYAML(StringRef YAML,
- llvm::UniqueStringSaver *Strings) {
- clangd::Ref Deserialized;
- llvm::yaml::Input YAMLInput(YAML, Strings);
- if (YAMLInput.error())
- return error("Unable to deserialize Symbol from YAML: {0}", YAML);
- YAMLInput >> Deserialized;
- return Deserialized;
-}
-
} // namespace clangd
} // namespace clang
Index: clang-tools-extra/clangd/index/SymbolOrigin.h
===================================================================
--- clang-tools-extra/clangd/index/SymbolOrigin.h
+++ clang-tools-extra/clangd/index/SymbolOrigin.h
@@ -18,27 +18,29 @@
// Describes the source of information about a symbol.
// Mainly useful for debugging, e.g. understanding code completion results.
// This is a bitfield as information can be combined from several sources.
-enum class SymbolOrigin : uint8_t {
+enum class SymbolOrigin : uint16_t {
Unknown = 0,
AST = 1 << 0, // Directly from the AST (indexes should not set this).
- Dynamic = 1 << 1, // From the dynamic index of opened files.
- Static = 1 << 2, // From the static, externally-built index.
+ Open = 1 << 1, // From the dynamic index of open files.
+ Static = 1 << 2, // From a static, externally-built index.
Merge = 1 << 3, // A non-trivial index merge was performed.
Identifier = 1 << 4, // Raw identifiers in file.
Remote = 1 << 5, // Remote index.
- // Remaining bits reserved for index implementations.
+ Preamble = 1 << 6, // From the dynamic index of preambles.
+ // 7 reserved
+ Background = 1 << 8, // From the automatic project index.
};
inline SymbolOrigin operator|(SymbolOrigin A, SymbolOrigin B) {
- return static_cast<SymbolOrigin>(static_cast<uint8_t>(A) |
- static_cast<uint8_t>(B));
+ return static_cast<SymbolOrigin>(static_cast<uint16_t>(A) |
+ static_cast<uint16_t>(B));
}
inline SymbolOrigin &operator|=(SymbolOrigin &A, SymbolOrigin B) {
return A = A | B;
}
inline SymbolOrigin operator&(SymbolOrigin A, SymbolOrigin B) {
- return static_cast<SymbolOrigin>(static_cast<uint8_t>(A) &
- static_cast<uint8_t>(B));
+ return static_cast<SymbolOrigin>(static_cast<uint16_t>(A) &
+ static_cast<uint16_t>(B));
}
llvm::raw_ostream &operator<<(llvm::raw_ostream &, SymbolOrigin);
Index: clang-tools-extra/clangd/index/SymbolOrigin.cpp
===================================================================
--- clang-tools-extra/clangd/index/SymbolOrigin.cpp
+++ clang-tools-extra/clangd/index/SymbolOrigin.cpp
@@ -14,9 +14,9 @@
llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, SymbolOrigin O) {
if (O == SymbolOrigin::Unknown)
return OS << "unknown";
- constexpr static char Sigils[] = "ADSMIR67";
+ constexpr static char Sigils[] = "AOSMIRP7B9012345";
for (unsigned I = 0; I < sizeof(Sigils); ++I)
- if (static_cast<uint8_t>(O) & 1u << I)
+ if (static_cast<uint16_t>(O) & 1u << I)
OS << Sigils[I];
return OS;
}
Index: clang-tools-extra/clangd/index/Serialization.h
===================================================================
--- clang-tools-extra/clangd/index/Serialization.h
+++ clang-tools-extra/clangd/index/Serialization.h
@@ -49,7 +49,7 @@
llvm::Optional<tooling::CompileCommand> Cmd;
};
// Parse an index file. The input must be a RIFF or YAML file.
-llvm::Expected<IndexFileIn> readIndexFile(llvm::StringRef);
+llvm::Expected<IndexFileIn> readIndexFile(llvm::StringRef, SymbolOrigin);
// Specifies the contents of an index file to be written.
struct IndexFileOut {
@@ -79,16 +79,10 @@
std::string toYAML(const Relation &);
std::string toYAML(const Ref &);
-// Deserialize a single symbol from YAML.
-llvm::Expected<clangd::Symbol> symbolFromYAML(StringRef YAML,
- llvm::UniqueStringSaver *Strings);
-llvm::Expected<clangd::Ref> refFromYAML(StringRef YAML,
- llvm::UniqueStringSaver *Strings);
-
// Build an in-memory static index from an index file.
// The size should be relatively small, so data can be managed in memory.
std::unique_ptr<SymbolIndex> loadIndex(llvm::StringRef Filename,
- bool UseDex = true);
+ SymbolOrigin Origin, bool UseDex = true);
} // namespace clangd
} // namespace clang
Index: clang-tools-extra/clangd/index/Serialization.cpp
===================================================================
--- clang-tools-extra/clangd/index/Serialization.cpp
+++ clang-tools-extra/clangd/index/Serialization.cpp
@@ -320,7 +320,6 @@
writeLocation(Sym.CanonicalDeclaration, Strings, OS);
writeVar(Sym.References, OS);
OS.write(static_cast<uint8_t>(Sym.Flags));
- OS.write(static_cast<uint8_t>(Sym.Origin));
writeVar(Strings.index(Sym.Signature), OS);
writeVar(Strings.index(Sym.CompletionSnippetSuffix), OS);
writeVar(Strings.index(Sym.Documentation), OS);
@@ -336,7 +335,8 @@
WriteInclude(Include);
}
-Symbol readSymbol(Reader &Data, llvm::ArrayRef<llvm::StringRef> Strings) {
+Symbol readSymbol(Reader &Data, llvm::ArrayRef<llvm::StringRef> Strings,
+ SymbolOrigin Origin) {
Symbol Sym;
Sym.ID = Data.consumeID();
Sym.SymInfo.Kind = static_cast<index::SymbolKind>(Data.consume8());
@@ -348,7 +348,7 @@
Sym.CanonicalDeclaration = readLocation(Data, Strings);
Sym.References = Data.consumeVar();
Sym.Flags = static_cast<Symbol::SymbolFlag>(Data.consume8());
- Sym.Origin = static_cast<SymbolOrigin>(Data.consume8());
+ Sym.Origin = Origin;
Sym.Signature = Data.consumeString(Strings);
Sym.CompletionSnippetSuffix = Data.consumeString(Strings);
Sym.Documentation = Data.consumeString(Strings);
@@ -452,9 +452,10 @@
// The current versioning scheme is simple - non-current versions are rejected.
// If you make a breaking change, bump this version number to invalidate stored
// data. Later we may want to support some backward compatibility.
-constexpr static uint32_t Version = 16;
+constexpr static uint32_t Version = 17;
-llvm::Expected<IndexFileIn> readRIFF(llvm::StringRef Data) {
+llvm::Expected<IndexFileIn> readRIFF(llvm::StringRef Data,
+ SymbolOrigin Origin) {
auto RIFF = riff::readFile(Data);
if (!RIFF)
return RIFF.takeError();
@@ -503,7 +504,7 @@
Reader SymbolReader(Chunks.lookup("symb"));
SymbolSlab::Builder Symbols;
while (!SymbolReader.eof())
- Symbols.insert(readSymbol(SymbolReader, Strings->Strings));
+ Symbols.insert(readSymbol(SymbolReader, Strings->Strings, Origin));
if (SymbolReader.err())
return error("malformed or truncated symbol");
Result.Symbols = std::move(Symbols).build();
@@ -670,7 +671,7 @@
// Defined in YAMLSerialization.cpp.
void writeYAML(const IndexFileOut &, llvm::raw_ostream &);
-llvm::Expected<IndexFileIn> readYAML(llvm::StringRef);
+llvm::Expected<IndexFileIn> readYAML(llvm::StringRef, SymbolOrigin Origin);
llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const IndexFileOut &O) {
switch (O.Format) {
@@ -684,11 +685,12 @@
return OS;
}
-llvm::Expected<IndexFileIn> readIndexFile(llvm::StringRef Data) {
+llvm::Expected<IndexFileIn> readIndexFile(llvm::StringRef Data,
+ SymbolOrigin Origin) {
if (Data.startswith("RIFF")) {
- return readRIFF(Data);
+ return readRIFF(Data, Origin);
}
- if (auto YAMLContents = readYAML(Data)) {
+ if (auto YAMLContents = readYAML(Data, Origin)) {
return std::move(*YAMLContents);
} else {
return error("Not a RIFF file and failed to parse as YAML: {0}",
@@ -697,7 +699,7 @@
}
std::unique_ptr<SymbolIndex> loadIndex(llvm::StringRef SymbolFilename,
- bool UseDex) {
+ SymbolOrigin Origin, bool UseDex) {
trace::Span OverallTracer("LoadIndex");
auto Buffer = llvm::MemoryBuffer::getFile(SymbolFilename);
if (!Buffer) {
@@ -710,7 +712,7 @@
RelationSlab Relations;
{
trace::Span Tracer("ParseIndex");
- if (auto I = readIndexFile(Buffer->get()->getBuffer())) {
+ if (auto I = readIndexFile(Buffer->get()->getBuffer(), Origin)) {
if (I->Symbols)
Symbols = std::move(*I->Symbols);
if (I->Refs)
Index: clang-tools-extra/clangd/index/FileIndex.cpp
===================================================================
--- clang-tools-extra/clangd/index/FileIndex.cpp
+++ clang-tools-extra/clangd/index/FileIndex.cpp
@@ -27,14 +27,12 @@
#include "clang/AST/ASTContext.h"
#include "clang/Index/IndexingAction.h"
#include "clang/Index/IndexingOptions.h"
-#include "clang/Lex/MacroInfo.h"
#include "clang/Lex/Preprocessor.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
-#include "llvm/Support/Error.h"
#include <algorithm>
#include <memory>
#include <tuple>
@@ -54,7 +52,8 @@
CollectorOpts.CollectIncludePath = true;
CollectorOpts.Includes = &Includes;
CollectorOpts.CountReferences = false;
- CollectorOpts.Origin = SymbolOrigin::Dynamic;
+ CollectorOpts.Origin =
+ IsIndexMainAST ? SymbolOrigin::Open : SymbolOrigin::Preamble;
CollectorOpts.CollectMainFileRefs = CollectMainFileRefs;
index::IndexingOptions IndexOpts;
Index: clang-tools-extra/clangd/index/BackgroundIndexStorage.cpp
===================================================================
--- clang-tools-extra/clangd/index/BackgroundIndexStorage.cpp
+++ clang-tools-extra/clangd/index/BackgroundIndexStorage.cpp
@@ -58,7 +58,8 @@
auto Buffer = llvm::MemoryBuffer::getFile(ShardPath);
if (!Buffer)
return nullptr;
- if (auto I = readIndexFile(Buffer->get()->getBuffer()))
+ if (auto I =
+ readIndexFile(Buffer->get()->getBuffer(), SymbolOrigin::Background))
return std::make_unique<IndexFileIn>(std::move(*I));
else
elog("Error while reading shard {0}: {1}", ShardIdentifier,
_______________________________________________
cfe-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
