kadircet created this revision.
kadircet added reviewers: sammccall, kbobyrev.
Herald added subscribers: usaxena95, arphaman.
Herald added a project: All.
kadircet requested review of this revision.
Herald added subscribers: cfe-commits, MaskRay, ilya-biryukov.
Herald added a project: clang-tools-extra.
Posting lists were copied into the final inverted index rather than
being moved. Also we weren't shrinking the vectors. This patch should help with
both peak memory usage while building dex (preamble/background-index) and it's
idle memory usage (unclear if copy constructor of vector preserves the capacity
or not).
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D124240
Files:
clang-tools-extra/clangd/index/dex/Dex.cpp
Index: clang-tools-extra/clangd/index/dex/Dex.cpp
===================================================================
--- clang-tools-extra/clangd/index/dex/Dex.cpp
+++ clang-tools-extra/clangd/index/dex/Dex.cpp
@@ -21,6 +21,7 @@
#include "llvm/Support/ScopedPrinter.h"
#include <algorithm>
#include <queue>
+#include <utility>
namespace clang {
namespace clangd {
@@ -76,23 +77,42 @@
}
// Assemble the final compressed posting lists for the added symbols.
- llvm::DenseMap<Token, PostingList> build() {
+ llvm::DenseMap<Token, PostingList> build() && {
llvm::DenseMap<Token, PostingList> Result(/*InitialReserve=*/
TrigramDocs.size() +
RestrictedCCDocs.size() +
TypeDocs.size() +
ScopeDocs.size() +
ProximityDocs.size());
- for (const auto &E : TrigramDocs)
- Result.try_emplace(Token(Token::Kind::Trigram, E.first.str()), E.second);
- for (const auto &E : TypeDocs)
- Result.try_emplace(Token(Token::Kind::Type, E.first()), E.second);
- for (const auto &E : ScopeDocs)
- Result.try_emplace(Token(Token::Kind::Scope, E.first()), E.second);
- for (const auto &E : ProximityDocs)
- Result.try_emplace(Token(Token::Kind::ProximityURI, E.first()),
E.second);
- if (!RestrictedCCDocs.empty())
- Result.try_emplace(RestrictedForCodeCompletion, RestrictedCCDocs);
+ for (auto &E : TrigramDocs) {
+ E.second.shrink_to_fit();
+ Result.try_emplace(Token(Token::Kind::Trigram, E.first.str()),
+ std::move(E.second));
+ }
+ TrigramDocs.clear();
+ for (auto &E : TypeDocs) {
+ E.second.shrink_to_fit();
+ Result.try_emplace(Token(Token::Kind::Type, E.first()),
+ std::move(E.second));
+ }
+ TypeDocs.clear();
+ for (auto &E : ScopeDocs) {
+ E.second.shrink_to_fit();
+ Result.try_emplace(Token(Token::Kind::Scope, E.first()),
+ std::move(E.second));
+ }
+ ScopeDocs.clear();
+ for (auto &E : ProximityDocs) {
+ E.second.shrink_to_fit();
+ Result.try_emplace(Token(Token::Kind::ProximityURI, E.first()),
+ std::move(E.second));
+ }
+ ProximityDocs.clear();
+ if (!RestrictedCCDocs.empty()) {
+ RestrictedCCDocs.shrink_to_fit();
+ Result.try_emplace(RestrictedForCodeCompletion,
+ std::move(RestrictedCCDocs));
+ }
return Result;
}
};
@@ -125,7 +145,7 @@
IndexBuilder Builder;
for (DocID SymbolRank = 0; SymbolRank < Symbols.size(); ++SymbolRank)
Builder.add(*Symbols[SymbolRank], SymbolRank);
- InvertedIndex = Builder.build();
+ InvertedIndex = std::move(Builder).build();
}
std::unique_ptr<Iterator> Dex::iterator(const Token &Tok) const {
Index: clang-tools-extra/clangd/index/dex/Dex.cpp
===================================================================
--- clang-tools-extra/clangd/index/dex/Dex.cpp
+++ clang-tools-extra/clangd/index/dex/Dex.cpp
@@ -21,6 +21,7 @@
#include "llvm/Support/ScopedPrinter.h"
#include <algorithm>
#include <queue>
+#include <utility>
namespace clang {
namespace clangd {
@@ -76,23 +77,42 @@
}
// Assemble the final compressed posting lists for the added symbols.
- llvm::DenseMap<Token, PostingList> build() {
+ llvm::DenseMap<Token, PostingList> build() && {
llvm::DenseMap<Token, PostingList> Result(/*InitialReserve=*/
TrigramDocs.size() +
RestrictedCCDocs.size() +
TypeDocs.size() +
ScopeDocs.size() +
ProximityDocs.size());
- for (const auto &E : TrigramDocs)
- Result.try_emplace(Token(Token::Kind::Trigram, E.first.str()), E.second);
- for (const auto &E : TypeDocs)
- Result.try_emplace(Token(Token::Kind::Type, E.first()), E.second);
- for (const auto &E : ScopeDocs)
- Result.try_emplace(Token(Token::Kind::Scope, E.first()), E.second);
- for (const auto &E : ProximityDocs)
- Result.try_emplace(Token(Token::Kind::ProximityURI, E.first()), E.second);
- if (!RestrictedCCDocs.empty())
- Result.try_emplace(RestrictedForCodeCompletion, RestrictedCCDocs);
+ for (auto &E : TrigramDocs) {
+ E.second.shrink_to_fit();
+ Result.try_emplace(Token(Token::Kind::Trigram, E.first.str()),
+ std::move(E.second));
+ }
+ TrigramDocs.clear();
+ for (auto &E : TypeDocs) {
+ E.second.shrink_to_fit();
+ Result.try_emplace(Token(Token::Kind::Type, E.first()),
+ std::move(E.second));
+ }
+ TypeDocs.clear();
+ for (auto &E : ScopeDocs) {
+ E.second.shrink_to_fit();
+ Result.try_emplace(Token(Token::Kind::Scope, E.first()),
+ std::move(E.second));
+ }
+ ScopeDocs.clear();
+ for (auto &E : ProximityDocs) {
+ E.second.shrink_to_fit();
+ Result.try_emplace(Token(Token::Kind::ProximityURI, E.first()),
+ std::move(E.second));
+ }
+ ProximityDocs.clear();
+ if (!RestrictedCCDocs.empty()) {
+ RestrictedCCDocs.shrink_to_fit();
+ Result.try_emplace(RestrictedForCodeCompletion,
+ std::move(RestrictedCCDocs));
+ }
return Result;
}
};
@@ -125,7 +145,7 @@
IndexBuilder Builder;
for (DocID SymbolRank = 0; SymbolRank < Symbols.size(); ++SymbolRank)
Builder.add(*Symbols[SymbolRank], SymbolRank);
- InvertedIndex = Builder.build();
+ InvertedIndex = std::move(Builder).build();
}
std::unique_ptr<Iterator> Dex::iterator(const Token &Tok) const {
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
