Author: kadircet Date: Mon Nov 19 10:06:29 2018 New Revision: 347235 URL: http://llvm.org/viewvc/llvm-project?rev=347235&view=rev Log: [clangd] Store source file hash in IndexFile{In,Out}
Summary: Puts the digest of the source file that generated the index into serialized index and stores them back on load, if exists. Reviewers: sammccall Subscribers: ilya-biryukov, ioeric, MaskRay, jkorous, arphaman, cfe-commits Differential Revision: https://reviews.llvm.org/D54693 Modified: clang-tools-extra/trunk/clangd/index/Background.cpp clang-tools-extra/trunk/clangd/index/Serialization.cpp clang-tools-extra/trunk/clangd/index/Serialization.h clang-tools-extra/trunk/unittests/clangd/BackgroundIndexTests.cpp clang-tools-extra/trunk/unittests/clangd/SerializationTests.cpp Modified: clang-tools-extra/trunk/clangd/index/Background.cpp URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Background.cpp?rev=347235&r1=347234&r2=347235&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/index/Background.cpp (original) +++ clang-tools-extra/trunk/clangd/index/Background.cpp Mon Nov 19 10:06:29 2018 @@ -246,6 +246,7 @@ void BackgroundIndex::update(StringRef M IndexFileOut Shard; Shard.Symbols = SS.get(); Shard.Refs = RS.get(); + Shard.Digest = &Hash; if (auto Error = IndexStorage->storeShard(Path, Shard)) elog("Failed to write background-index shard for file {0}: {1}", Path, std::move(Error)); Modified: clang-tools-extra/trunk/clangd/index/Serialization.cpp URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Serialization.cpp?rev=347235&r1=347234&r2=347235&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/index/Serialization.cpp (original) +++ clang-tools-extra/trunk/clangd/index/Serialization.cpp Mon Nov 19 10:06:29 2018 @@ -363,6 +363,13 @@ Expected<IndexFileIn> readRIFF(StringRef return Strings.takeError(); IndexFileIn Result; + if (Chunks.count("hash")) { + Reader Hash(Chunks.lookup("hash")); + llvm::StringRef Digest = Hash.consume(20); + Result.Digest.emplace(); + std::copy(Digest.bytes_begin(), Digest.bytes_end(), Result.Digest->begin()); + } + if (Chunks.count("symb")) { Reader SymbolReader(Chunks.lookup("symb")); SymbolSlab::Builder Symbols; @@ -399,6 +406,12 @@ void writeRIFF(const IndexFileOut &Data, } RIFF.Chunks.push_back({riff::fourCC("meta"), Meta}); + if (Data.Digest) { + llvm::StringRef Hash(reinterpret_cast<const char *>(Data.Digest->data()), + Data.Digest->size()); + RIFF.Chunks.push_back({riff::fourCC("hash"), Hash}); + } + StringTableOut Strings; std::vector<Symbol> Symbols; for (const auto &Sym : *Data.Symbols) { Modified: clang-tools-extra/trunk/clangd/index/Serialization.h URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Serialization.h?rev=347235&r1=347234&r2=347235&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/index/Serialization.h (original) +++ clang-tools-extra/trunk/clangd/index/Serialization.h Mon Nov 19 10:06:29 2018 @@ -39,6 +39,8 @@ enum class IndexFileFormat { struct IndexFileIn { llvm::Optional<SymbolSlab> Symbols; llvm::Optional<RefSlab> Refs; + // Digest of the source file that generated the contents. + llvm::Optional<std::array<uint8_t, 20>> Digest; }; // Parse an index file. The input must be a RIFF or YAML file. llvm::Expected<IndexFileIn> readIndexFile(llvm::StringRef); @@ -47,6 +49,8 @@ llvm::Expected<IndexFileIn> readIndexFil struct IndexFileOut { const SymbolSlab *Symbols = nullptr; const RefSlab *Refs = nullptr; + // Digest of the source file that generated the contents. + const std::array<uint8_t, 20> *Digest = nullptr; // TODO: Support serializing Dex posting lists. IndexFileFormat Format = IndexFileFormat::RIFF; Modified: clang-tools-extra/trunk/unittests/clangd/BackgroundIndexTests.cpp URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/BackgroundIndexTests.cpp?rev=347235&r1=347234&r2=347235&view=diff ============================================================================== --- clang-tools-extra/trunk/unittests/clangd/BackgroundIndexTests.cpp (original) +++ clang-tools-extra/trunk/unittests/clangd/BackgroundIndexTests.cpp Mon Nov 19 10:06:29 2018 @@ -121,8 +121,10 @@ TEST(BackgroundIndexTest, ShardStorageWr void f_b(); class A_CC {}; )cpp"; - FS.Files[testPath("root/A.cc")] = - "#include \"A.h\"\nvoid g() { (void)common; }"; + std::string A_CC = "#include \"A.h\"\nvoid g() { (void)common; }"; + FS.Files[testPath("root/A.cc")] = A_CC; + auto Digest = llvm::SHA1::hash( + {reinterpret_cast<const uint8_t *>(A_CC.data()), A_CC.size()}); llvm::StringMap<std::string> Storage; size_t CacheHits = 0; @@ -156,6 +158,7 @@ TEST(BackgroundIndexTest, ShardStorageWr EXPECT_NE(ShardSource, nullptr); EXPECT_THAT(*ShardSource->Symbols, UnorderedElementsAre()); EXPECT_THAT(*ShardSource->Refs, RefsAre({FileURI("unittest:///root/A.cc")})); + EXPECT_EQ(*ShardSource->Digest, Digest); } } // namespace clangd Modified: clang-tools-extra/trunk/unittests/clangd/SerializationTests.cpp URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/SerializationTests.cpp?rev=347235&r1=347234&r2=347235&view=diff ============================================================================== --- clang-tools-extra/trunk/unittests/clangd/SerializationTests.cpp (original) +++ clang-tools-extra/trunk/unittests/clangd/SerializationTests.cpp Mon Nov 19 10:06:29 2018 @@ -9,6 +9,7 @@ #include "index/Index.h" #include "index/Serialization.h" +#include "llvm/Support/SHA1.h" #include "llvm/Support/ScopedPrinter.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -165,6 +166,33 @@ TEST(SerializationTest, BinaryConversion ASSERT_TRUE(In2->Symbols); ASSERT_TRUE(In2->Refs); + // Assert the YAML serializations match, for nice comparisons and diffs. + EXPECT_THAT(YAMLFromSymbols(*In2->Symbols), + UnorderedElementsAreArray(YAMLFromSymbols(*In->Symbols))); + EXPECT_THAT(YAMLFromRefs(*In2->Refs), + UnorderedElementsAreArray(YAMLFromRefs(*In->Refs))); +} + +TEST(SerializationTest, HashTest) { + auto In = readIndexFile(YAML); + EXPECT_TRUE(bool(In)) << In.takeError(); + + std::string TestContent("TESTCONTENT"); + auto Digest = + llvm::SHA1::hash({reinterpret_cast<const uint8_t *>(TestContent.data()), + TestContent.size()}); + // Write to binary format, and parse again. + IndexFileOut Out(*In); + Out.Format = IndexFileFormat::RIFF; + Out.Digest = &Digest; + std::string Serialized = to_string(Out); + + auto In2 = readIndexFile(Serialized); + ASSERT_TRUE(bool(In2)) << In.takeError(); + ASSERT_EQ(In2->Digest, Digest); + ASSERT_TRUE(In2->Symbols); + ASSERT_TRUE(In2->Refs); + // Assert the YAML serializations match, for nice comparisons and diffs. EXPECT_THAT(YAMLFromSymbols(*In2->Symbols), UnorderedElementsAreArray(YAMLFromSymbols(*In->Symbols))); _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
