jansvoboda11 updated this revision to Diff 391369.
jansvoboda11 added a comment.
Add unit test, IWYU.
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D114966/new/
https://reviews.llvm.org/D114966
Files:
clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
clang/unittests/Tooling/DependencyScannerTest.cpp
llvm/include/llvm/Support/VirtualFileSystem.h
llvm/lib/Support/VirtualFileSystem.cpp
Index: llvm/lib/Support/VirtualFileSystem.cpp
===================================================================
--- llvm/lib/Support/VirtualFileSystem.cpp
+++ llvm/lib/Support/VirtualFileSystem.cpp
@@ -75,6 +75,12 @@
: Name(Name.str()), UID(UID), MTime(MTime), User(User), Group(Group),
Size(Size), Type(Type), Perms(Perms) {}
+Status Status::copyWithNewSize(const Status &In, uint64_t NewSize) {
+ return Status(In.getName(), In.getUniqueID(), In.getLastModificationTime(),
+ In.getUser(), In.getGroup(), NewSize, In.getType(),
+ In.getPermissions());
+}
+
Status Status::copyWithNewName(const Status &In, const Twine &NewName) {
return Status(NewName, In.getUniqueID(), In.getLastModificationTime(),
In.getUser(), In.getGroup(), In.getSize(), In.getType(),
Index: llvm/include/llvm/Support/VirtualFileSystem.h
===================================================================
--- llvm/include/llvm/Support/VirtualFileSystem.h
+++ llvm/include/llvm/Support/VirtualFileSystem.h
@@ -64,6 +64,8 @@
uint64_t Size, llvm::sys::fs::file_type Type,
llvm::sys::fs::perms Perms);
+ /// Get a copy of a Status with a different size.
+ static Status copyWithNewSize(const Status &In, uint64_t NewSize);
/// Get a copy of a Status with a different name.
static Status copyWithNewName(const Status &In, const Twine &NewName);
static Status copyWithNewName(const llvm::sys::fs::file_status &In,
Index: clang/unittests/Tooling/DependencyScannerTest.cpp
===================================================================
--- clang/unittests/Tooling/DependencyScannerTest.cpp
+++ clang/unittests/Tooling/DependencyScannerTest.cpp
@@ -205,9 +205,11 @@
}
namespace dependencies {
-TEST(DependencyScanningFilesystem, IgnoredFilesHaveSeparateCache) {
+TEST(DependencyScanningFilesystem, IgnoredFilesAreCachedSeparately1) {
auto VFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
- VFS->addFile("/mod.h", 0, llvm::MemoryBuffer::getMemBuffer("// hi there!\n"));
+ VFS->addFile("/mod.h", 0,
+ llvm::MemoryBuffer::getMemBuffer("#include <foo.h>\n"
+ "// hi there!\n"));
DependencyScanningFilesystemSharedCache SharedCache;
auto Mappings = std::make_unique<ExcludedPreprocessorDirectiveSkipMapping>();
@@ -223,14 +225,35 @@
auto StatusFull3 = DepFS.status("/mod.h");
EXPECT_TRUE(StatusMinimized0);
- EXPECT_EQ(StatusMinimized0->getSize(), 0u);
+ EXPECT_EQ(StatusMinimized0->getSize(), 17u);
EXPECT_TRUE(StatusFull1);
- EXPECT_EQ(StatusFull1->getSize(), 13u);
+ EXPECT_EQ(StatusFull1->getSize(), 30u);
EXPECT_TRUE(StatusMinimized2);
- EXPECT_EQ(StatusMinimized2->getSize(), 0u);
+ EXPECT_EQ(StatusMinimized2->getSize(), 17u);
EXPECT_TRUE(StatusFull3);
- EXPECT_EQ(StatusFull3->getSize(), 13u);
+ EXPECT_EQ(StatusFull3->getSize(), 30u);
+}
+
+TEST(DependencyScanningFilesystem, IgnoredFilesAreCachedSeparately2) {
+ auto VFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
+ VFS->addFile("/mod.h", 0,
+ llvm::MemoryBuffer::getMemBuffer("#include <foo.h>\n"
+ "// hi there!\n"));
+
+ DependencyScanningFilesystemSharedCache SharedCache;
+ auto Mappings = std::make_unique<ExcludedPreprocessorDirectiveSkipMapping>();
+ DependencyScanningWorkerFilesystem DepFS(SharedCache, VFS, Mappings.get());
+
+ DepFS.disableMinimization("/mod.h");
+ auto StatusFull0 = DepFS.status("/mod.h");
+ DepFS.enableMinimizationOfAllFiles();
+ auto StatusMinimized1 = DepFS.status("/mod.h");
+
+ EXPECT_TRUE(StatusFull0);
+ EXPECT_TRUE(StatusMinimized1);
+ EXPECT_EQ(StatusFull0->getSize(), 30u);
+ EXPECT_EQ(StatusMinimized1->getSize(), 17u);
}
} // end namespace dependencies
Index: clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
===================================================================
--- clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -15,9 +15,9 @@
using namespace tooling;
using namespace dependencies;
-CachedFileSystemEntry CachedFileSystemEntry::createFileEntry(
- StringRef Filename, llvm::vfs::FileSystem &FS, bool Minimize) {
- // Load the file and its content from the file system.
+static llvm::ErrorOr<llvm::vfs::Status>
+readFile(StringRef Filename, llvm::vfs::FileSystem &FS,
+ llvm::SmallString<1> &SharedOriginalContents) {
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MaybeFile =
FS.openFileForRead(Filename);
if (!MaybeFile)
@@ -32,31 +32,27 @@
if (!MaybeBuffer)
return MaybeBuffer.getError();
+ const auto &Buffer = *MaybeBuffer;
+ SharedOriginalContents.reserve(Buffer->getBufferSize() + 1);
+ SharedOriginalContents.append(Buffer->getBufferStart(),
+ Buffer->getBufferEnd());
+ // Implicitly null terminate the contents for Clang's lexer.
+ SharedOriginalContents.push_back('\0');
+ SharedOriginalContents.pop_back();
+ return *Stat;
+}
+
+static bool
+minimizeFile(const SmallString<1> &OriginalContents,
+ SmallString<1> &MinimizedContents,
+ PreprocessorSkippedRangeMapping &PPSkippedRangeMapping) {
llvm::SmallString<1024> MinimizedFileContents;
// Minimize the file down to directives that might affect the dependencies.
- const auto &Buffer = *MaybeBuffer;
SmallVector<minimize_source_to_dependency_directives::Token, 64> Tokens;
- if (!Minimize || minimizeSourceToDependencyDirectives(
- Buffer->getBuffer(), MinimizedFileContents, Tokens)) {
- // Use the original file unless requested otherwise, or
- // if the minimization failed.
- // FIXME: Propage the diagnostic if desired by the client.
- CachedFileSystemEntry Result;
- Result.MaybeStat = std::move(*Stat);
- Result.Contents.reserve(Buffer->getBufferSize() + 1);
- Result.Contents.append(Buffer->getBufferStart(), Buffer->getBufferEnd());
- // Implicitly null terminate the contents for Clang's lexer.
- Result.Contents.push_back('\0');
- Result.Contents.pop_back();
- return Result;
- }
+ if (minimizeSourceToDependencyDirectives(OriginalContents.str(),
+ MinimizedFileContents, Tokens))
+ return true;
- CachedFileSystemEntry Result;
- size_t Size = MinimizedFileContents.size();
- Result.MaybeStat = llvm::vfs::Status(Stat->getName(), Stat->getUniqueID(),
- Stat->getLastModificationTime(),
- Stat->getUser(), Stat->getGroup(), Size,
- Stat->getType(), Stat->getPermissions());
// The contents produced by the minimizer must be null terminated.
assert(MinimizedFileContents.data()[MinimizedFileContents.size()] == '\0' &&
"not null terminated contents");
@@ -65,10 +61,10 @@
// std::move will preserve it even if it needs to do a copy if the
// SmallString still has the small capacity.
MinimizedFileContents.push_back('\0');
- Result.Contents = std::move(MinimizedFileContents);
+ MinimizedContents = std::move(MinimizedFileContents);
// Now make the null terminator implicit again, so that Clang's lexer can find
// it right where the buffer ends.
- Result.Contents.pop_back();
+ MinimizedContents.pop_back();
// Compute the skipped PP ranges that speedup skipping over inactive
// preprocessor blocks.
@@ -86,20 +82,13 @@
}
Mapping[Range.Offset] = Range.Length;
}
- Result.PPSkippedRangeMapping = std::move(Mapping);
+ PPSkippedRangeMapping = std::move(Mapping);
- return Result;
+ return false;
}
-CachedFileSystemEntry
-CachedFileSystemEntry::createDirectoryEntry(llvm::vfs::Status &&Stat) {
- assert(Stat.isDirectory() && "not a directory!");
- auto Result = CachedFileSystemEntry();
- Result.MaybeStat = std::move(Stat);
- return Result;
-}
-
-DependencyScanningFilesystemSharedCache::SingleCache::SingleCache() {
+DependencyScanningFilesystemSharedCache::
+ DependencyScanningFilesystemSharedCache() {
// This heuristic was chosen using a empirical testing on a
// reasonably high core machine (iMacPro 18 cores / 36 threads). The cache
// sharding gives a performance edge by reducing the lock contention.
@@ -110,18 +99,67 @@
CacheShards = std::make_unique<CacheShard[]>(NumShards);
}
-DependencyScanningFilesystemSharedCache::SharedFileSystemEntry &
-DependencyScanningFilesystemSharedCache::SingleCache::get(StringRef Key) {
- CacheShard &Shard = CacheShards[llvm::hash_value(Key) % NumShards];
- std::unique_lock<std::mutex> LockGuard(Shard.CacheLock);
- auto It = Shard.Cache.try_emplace(Key);
- return It.first->getValue();
+SharedStat &
+DependencyScanningFilesystemSharedCache::getStat(StringRef Filename) {
+ CacheShard &Shard = CacheShards[llvm::hash_value(Filename) % NumShards];
+ std::unique_lock<std::mutex> Lock(Shard.CacheLock);
+ return Shard.StatCache.try_emplace(Filename).first->second;
+}
+
+InsertResult<OriginalContents>
+DependencyScanningFilesystemSharedCache::getOriginalContents(
+ StringRef Filename, llvm::sys::fs::UniqueID UID) {
+ CacheShard &Shard = CacheShards[llvm::hash_value(Filename) % NumShards];
+ std::unique_lock<std::mutex> Lock(Shard.CacheLock);
+ auto InsertRes = Shard.OriginalContentsCache.insert({UID, OriginalContents{}});
+ return makeInsertResult(InsertRes.first->second, InsertRes.second);
+}
+
+InsertResult<MinimizedContents>
+DependencyScanningFilesystemSharedCache::getMinimizedContents(
+ StringRef Filename, llvm::sys::fs::UniqueID UID) {
+ CacheShard &Shard = CacheShards[llvm::hash_value(Filename) % NumShards];
+ std::unique_lock<std::mutex> Lock(Shard.CacheLock);
+ auto InsertRes = Shard.MinimizedContentsCache.insert({UID, MinimizedContents{}});
+ return makeInsertResult(InsertRes.first->second, InsertRes.second);
+}
+
+const MaybeStat *
+DependencyScanningFilesystemLocalCache::findStat(StringRef Filename) {
+ auto It = StatCache.find(Filename);
+ return It != StatCache.end() ? It->getValue() : nullptr;
+}
+
+const MaybeStat *
+DependencyScanningFilesystemLocalCache::insertStat(StringRef Filename,
+ const MaybeStat *Stat) {
+ return StatCache.insert({Filename, Stat}).first->second;
+}
+
+const OriginalContents *
+DependencyScanningFilesystemLocalCache::findOriginalContents(
+ llvm::sys::fs::UniqueID UID) {
+ auto It = OriginalContentsCache.find(UID);
+ return It != OriginalContentsCache.end() ? It->second : nullptr;
+}
+
+const OriginalContents *
+DependencyScanningFilesystemLocalCache::insertOriginalContents(
+ llvm::sys::fs::UniqueID UID, const OriginalContents *Contents) {
+ return OriginalContentsCache.insert({UID, Contents}).first->second;
}
-DependencyScanningFilesystemSharedCache::SharedFileSystemEntry &
-DependencyScanningFilesystemSharedCache::get(StringRef Key, bool Minimized) {
- SingleCache &Cache = Minimized ? CacheMinimized : CacheOriginal;
- return Cache.get(Key);
+const MinimizedContents *
+DependencyScanningFilesystemLocalCache::findMinimizedContents(
+ llvm::sys::fs::UniqueID UID) {
+ auto It = MinimizedContentsCache.find(UID);
+ return It != MinimizedContentsCache.end() ? It->second : nullptr;
+}
+
+const MinimizedContents *
+DependencyScanningFilesystemLocalCache::insertMinimizedContents(
+ llvm::sys::fs::UniqueID UID, const MinimizedContents *Contents) {
+ return MinimizedContentsCache.insert({UID, Contents}).first->second;
}
/// Whitelist file extensions that should be minimized, treating no extension as
@@ -167,66 +205,126 @@
return !NotToBeMinimized.contains(Filename);
}
-CachedFileSystemEntry DependencyScanningWorkerFilesystem::createFileSystemEntry(
- llvm::ErrorOr<llvm::vfs::Status> &&MaybeStatus, StringRef Filename,
- bool ShouldMinimize) {
- if (!MaybeStatus)
- return CachedFileSystemEntry(MaybeStatus.getError());
+void DependencyScanningWorkerFilesystem::ensureLocked(
+ StringRef Path, SharedStat *&SS,
+ llvm::Optional<std::unique_lock<std::mutex>> &Lock) {
+ if (!SS)
+ SS = &SharedCache.getStat(Path);
+ if (!Lock)
+ Lock.emplace(SS->Mutex);
+ assert(Lock->mutex() == &SS->Mutex && "Locked the wrong mutex");
+}
+
+const MaybeStat *DependencyScanningWorkerFilesystem::getOrCreateLocalStat(
+ StringRef Path, SharedStat *&SS,
+ llvm::Optional<std::unique_lock<std::mutex>> &Lock) {
+ if (const MaybeStat *LocalStat = Cache.findStat(Path))
+ return LocalStat;
+
+ bool CacheStatFailures = shouldCacheStatFailures(Path);
- if (MaybeStatus->isDirectory())
- return CachedFileSystemEntry::createDirectoryEntry(std::move(*MaybeStatus));
+ ensureLocked(Path, SS, Lock);
- return CachedFileSystemEntry::createFileEntry(Filename, getUnderlyingFS(),
- ShouldMinimize);
+ // Call stat if necessary.
+ if (!SS->isValid() || (!SS->Stat && !CacheStatFailures)) {
+ // HACK: We need to always restat non source files if the stat fails.
+ // This is because Clang first looks up the module cache and module
+ // files before building them, and then looks for them again. If we
+ // cache the stat failure, it won't see them the second time.
+ auto MaybeStatus = getUnderlyingFS().status(Path);
+ SS->Stat = std::move(MaybeStatus);
+ }
+ // Update local cache if possible.
+ return CacheStatFailures ? Cache.insertStat(Path, &SS->Stat) : &SS->Stat;
}
-llvm::ErrorOr<const CachedFileSystemEntry *>
-DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry(
- const StringRef Filename) {
- bool ShouldMinimize = shouldMinimize(Filename);
-
- if (const auto *Entry = Cache.getCachedEntry(Filename, ShouldMinimize))
- return Entry;
-
- // FIXME: Handle PCM/PCH files.
- // FIXME: Handle module map files.
-
- DependencyScanningFilesystemSharedCache::SharedFileSystemEntry
- &SharedCacheEntry = SharedCache.get(Filename, ShouldMinimize);
- const CachedFileSystemEntry *Result;
- {
- std::unique_lock<std::mutex> LockGuard(SharedCacheEntry.ValueLock);
- CachedFileSystemEntry &CacheEntry = SharedCacheEntry.Value;
-
- if (!CacheEntry.isValid()) {
- auto MaybeStatus = getUnderlyingFS().status(Filename);
- if (!MaybeStatus && !shouldCacheStatFailures(Filename))
- // HACK: We need to always restat non source files if the stat fails.
- // This is because Clang first looks up the module cache and module
- // files before building them, and then looks for them again. If we
- // cache the stat failure, it won't see them the second time.
- return MaybeStatus.getError();
- CacheEntry = createFileSystemEntry(std::move(MaybeStatus), Filename,
- ShouldMinimize);
- }
+const OriginalContents *
+DependencyScanningWorkerFilesystem::getOrCreateLocalOriginalContents(
+ StringRef Filename, llvm::sys::fs::UniqueID UID, SharedStat *&SharedStat,
+ llvm::Optional<std::unique_lock<std::mutex>> &Lock) {
+ if (const OriginalContents *LocalOriginalContents =
+ Cache.findOriginalContents(UID))
+ return LocalOriginalContents;
+
+ ensureLocked(Filename, SharedStat, Lock);
- Result = &CacheEntry;
+ auto SharedOriginalContents = SharedCache.getOriginalContents(Filename, UID);
+ // Read the file if necessary.
+ if (SharedOriginalContents.WasInserted)
+ SharedStat->Stat =
+ readFile(Filename, getUnderlyingFS(), SharedOriginalContents.Value);
+
+ return Cache.insertOriginalContents(UID, &SharedOriginalContents.Value);
+}
+
+const MinimizedContents *
+DependencyScanningWorkerFilesystem::getOrCreateLocalMinimizedContents(
+ StringRef Filename, llvm::sys::fs::UniqueID UID, SharedStat *&SharedStat,
+ llvm::Optional<std::unique_lock<std::mutex>> &Lock) {
+ if (const MinimizedContents *LocalMinimizedContents =
+ Cache.findMinimizedContents(UID))
+ return LocalMinimizedContents;
+
+ const OriginalContents *LocalOriginalContents =
+ getOrCreateLocalOriginalContents(Filename, UID, SharedStat, Lock);
+
+ ensureLocked(Filename, SharedStat, Lock);
+
+ auto SharedMinimizedContents =
+ SharedCache.getMinimizedContents(Filename, UID);
+ if (SharedMinimizedContents.WasInserted)
+ if (minimizeFile(*LocalOriginalContents,
+ SharedMinimizedContents.Value.Contents,
+ SharedMinimizedContents.Value.PPSkippedRangeMapping))
+ // Use the original file contents if minimization failed.
+ SharedMinimizedContents.Value.Contents = *LocalOriginalContents;
+ return &SharedMinimizedContents.Value;
+}
+
+static llvm::ErrorOr<llvm::vfs::Status>
+withNewSize(const llvm::ErrorOr<llvm::vfs::Status> &Status, uint64_t NewSize) {
+ if (!Status)
+ return Status;
+ return llvm::vfs::Status::copyWithNewSize(*Status, NewSize);
+}
+
+llvm::ErrorOr<FileSystemEntryResult>
+DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry(
+ StringRef Filename) {
+ SharedStat *SharedStat = nullptr;
+ llvm::Optional<std::unique_lock<std::mutex>> GuardLock;
+
+ const MaybeStat *LocalStat =
+ getOrCreateLocalStat(Filename, SharedStat, GuardLock);
+ if (!(*LocalStat))
+ return LocalStat->getError();
+ if ((*LocalStat)->isDirectory())
+ return FileSystemEntryResult(*LocalStat, nullptr, nullptr);
+
+ llvm::sys::fs::UniqueID UID = (*LocalStat)->getUniqueID();
+
+ if (!shouldMinimize(Filename)) {
+ const OriginalContents *LocalOriginalContents =
+ getOrCreateLocalOriginalContents(Filename, UID, SharedStat, GuardLock);
+ return FileSystemEntryResult(*LocalStat, LocalOriginalContents, nullptr);
}
- // Store the result in the local cache.
- Cache.setCachedEntry(Filename, ShouldMinimize, Result);
- return Result;
+ const MinimizedContents *LocalMinimizedContents =
+ getOrCreateLocalMinimizedContents(Filename, UID, SharedStat, GuardLock);
+ return FileSystemEntryResult(
+ withNewSize(*LocalStat, LocalMinimizedContents->Contents.size()),
+ &LocalMinimizedContents->Contents,
+ &LocalMinimizedContents->PPSkippedRangeMapping);
}
llvm::ErrorOr<llvm::vfs::Status>
DependencyScanningWorkerFilesystem::status(const Twine &Path) {
SmallString<256> OwnedFilename;
StringRef Filename = Path.toStringRef(OwnedFilename);
- const llvm::ErrorOr<const CachedFileSystemEntry *> Result =
- getOrCreateFileSystemEntry(Filename);
+ auto Result = getOrCreateFileSystemEntry(Filename);
if (!Result)
return Result.getError();
- return (*Result)->getStatus();
+ return Result->Status;
}
namespace {
@@ -240,7 +338,7 @@
: Buffer(std::move(Buffer)), Stat(std::move(Stat)) {}
static llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
- create(const CachedFileSystemEntry *Entry,
+ create(const FileSystemEntryResult &Entry,
ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings);
llvm::ErrorOr<llvm::vfs::Status> status() override { return Stat; }
@@ -261,23 +359,21 @@
} // end anonymous namespace
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MinimizedVFSFile::create(
- const CachedFileSystemEntry *Entry,
+ const FileSystemEntryResult &Entry,
ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings) {
- if (Entry->isDirectory())
+ if (Entry.Status->isDirectory())
return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
std::make_error_code(std::errc::is_a_directory));
- llvm::ErrorOr<StringRef> Contents = Entry->getContents();
- if (!Contents)
- return Contents.getError();
auto Result = std::make_unique<MinimizedVFSFile>(
- llvm::MemoryBuffer::getMemBuffer(*Contents, Entry->getName(),
+ llvm::MemoryBuffer::getMemBuffer(*Entry.Contents, Entry.Status->getName(),
/*RequiresNullTerminator=*/false),
- *Entry->getStatus());
- if (!Entry->getPPSkippedRangeMapping().empty() && PPSkipMappings)
+ *Entry.Status);
+
+ if (Entry.PPSkippedRangeMapping && !Entry.PPSkippedRangeMapping->empty() &&
+ PPSkipMappings)
(*PPSkipMappings)[Result->Buffer->getBufferStart()] =
- &Entry->getPPSkippedRangeMapping();
- return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
- std::unique_ptr<llvm::vfs::File>(std::move(Result)));
+ Entry.PPSkippedRangeMapping;
+ return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(std::move(Result));
}
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
@@ -285,9 +381,10 @@
SmallString<256> OwnedFilename;
StringRef Filename = Path.toStringRef(OwnedFilename);
- const llvm::ErrorOr<const CachedFileSystemEntry *> Result =
- getOrCreateFileSystemEntry(Filename);
+ auto Result = getOrCreateFileSystemEntry(Filename);
if (!Result)
return Result.getError();
+ if (!Result->Status)
+ return Result->Status.getError();
return MinimizedVFSFile::create(Result.get(), PPSkipMappings);
}
Index: clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
===================================================================
--- clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
+++ clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
@@ -11,97 +11,92 @@
#include "clang/Basic/LLVM.h"
#include "clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/Optional.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringSet.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/VirtualFileSystem.h"
#include <mutex>
+#include <map>
namespace clang {
namespace tooling {
namespace dependencies {
-/// An in-memory representation of a file system entity that is of interest to
-/// the dependency scanning filesystem.
-///
-/// It represents one of the following:
-/// - an opened source file with minimized contents and a stat value.
-/// - an opened source file with original contents and a stat value.
-/// - a directory entry with its stat value.
-/// - an error value to represent a file system error.
-/// - a placeholder with an invalid stat indicating a not yet initialized entry.
-class CachedFileSystemEntry {
-public:
- /// Default constructor creates an entry with an invalid stat.
- CachedFileSystemEntry() : MaybeStat(llvm::vfs::Status()) {}
-
- CachedFileSystemEntry(std::error_code Error) : MaybeStat(std::move(Error)) {}
-
- /// Create an entry that represents an opened source file with minimized or
- /// original contents.
- ///
- /// The filesystem opens the file even for `stat` calls open to avoid the
- /// issues with stat + open of minimized files that might lead to a
- /// mismatching size of the file. If file is not minimized, the full file is
- /// read and copied into memory to ensure that it's not memory mapped to avoid
- /// running out of file descriptors.
- static CachedFileSystemEntry createFileEntry(StringRef Filename,
- llvm::vfs::FileSystem &FS,
- bool Minimize = true);
-
- /// Create an entry that represents a directory on the filesystem.
- static CachedFileSystemEntry createDirectoryEntry(llvm::vfs::Status &&Stat);
-
- /// \returns True if the entry is valid.
- bool isValid() const { return !MaybeStat || MaybeStat->isStatusKnown(); }
-
- /// \returns True if the current entry points to a directory.
- bool isDirectory() const { return MaybeStat && MaybeStat->isDirectory(); }
-
- /// \returns The error or the file's contents.
- llvm::ErrorOr<StringRef> getContents() const {
- if (!MaybeStat)
- return MaybeStat.getError();
- assert(!MaybeStat->isDirectory() && "not a file");
- assert(isValid() && "not initialized");
- return Contents.str();
- }
-
- /// \returns The error or the status of the entry.
- llvm::ErrorOr<llvm::vfs::Status> getStatus() const {
- assert(isValid() && "not initialized");
- return MaybeStat;
- }
-
- /// \returns the name of the file.
- StringRef getName() const {
- assert(isValid() && "not initialized");
- return MaybeStat->getName();
- }
-
- /// Return the mapping between location -> distance that is used to speed up
- /// the block skipping in the preprocessor.
- const PreprocessorSkippedRangeMapping &getPPSkippedRangeMapping() const {
- return PPSkippedRangeMapping;
- }
-
- CachedFileSystemEntry(CachedFileSystemEntry &&) = default;
- CachedFileSystemEntry &operator=(CachedFileSystemEntry &&) = default;
-
- CachedFileSystemEntry(const CachedFileSystemEntry &) = delete;
- CachedFileSystemEntry &operator=(const CachedFileSystemEntry &) = delete;
+/// The result type that represents one of:
+/// - a file system error,
+/// - status of a directory,
+/// - status of a file,
+/// - status of an opened file and its original contents,
+/// - status of an opened file and its minimized contents.
+struct FileSystemEntryResult {
+ /// The filesystem error, or status.
+ llvm::ErrorOr<llvm::vfs::Status> Status;
+ /// The (potentially minimized) file contents.
+ const llvm::SmallString<1> *Contents;
+ /// The skipped range mappings produced during minimization.
+ const PreprocessorSkippedRangeMapping *PPSkippedRangeMapping;
+
+ FileSystemEntryResult(
+ llvm::ErrorOr<llvm::vfs::Status> Status,
+ const llvm::SmallString<1> *Contents,
+ const PreprocessorSkippedRangeMapping *PPSkippedRangeMapping)
+ : Status(std::move(Status)), Contents(Contents),
+ PPSkippedRangeMapping(PPSkippedRangeMapping) {}
+};
-private:
- llvm::ErrorOr<llvm::vfs::Status> MaybeStat;
- // Store the contents in a small string to allow a
- // move from the small string for the minimized contents.
- // Note: small size of 1 allows us to store an empty string with an implicit
- // null terminator without any allocations.
- llvm::SmallString<1> Contents;
+/// The value type for stat cache.
+using MaybeStat = llvm::ErrorOr<llvm::vfs::Status>;
+
+/// The value type for shared stat cache.
+struct SharedStat {
+ /// The wrapped status value.
+ MaybeStat Stat;
+
+ /// The mutex that must be locked when accessing this shared stat cache entry
+ /// and shared original/minimized content caches for the status' UniqueID.
+ std::mutex Mutex;
+
+ SharedStat() : Stat(std::error_code()) {}
+
+ /// Check whether this cache entry is valid.
+ bool isValid() const { return Stat || Stat.getError() != std::error_code(); }
+};
+
+// Small size of 1 allows us to store an empty string with an implicit null
+// terminator without any allocations.
+using Contents = llvm::SmallString<1>;
+
+/// The value type for original contents cache.
+using OriginalContents = Contents;
+
+/// The value type for minimized contents cache.
+struct MinimizedContents {
+ /// The minimized contents itself.
+ Contents Contents;
PreprocessorSkippedRangeMapping PPSkippedRangeMapping;
+
+ MinimizedContents() : Contents(), PPSkippedRangeMapping(0) {}
};
+/// Result of the "insert" operation on map and set containers that exposes
+/// members with more obvious names compared to \c std::pair::{first,second}.
+template <typename T> struct InsertResult {
+ T &Value;
+ bool WasInserted;
+
+ InsertResult(T &Value, bool WasInserted)
+ : Value(Value), WasInserted(WasInserted) {}
+};
+
+/// A factory function for deducing the template parameter of \c InsertResult.
+template <typename T>
+InsertResult<T> makeInsertResult(T &Value, bool WasInserted) {
+ return InsertResult<T>(Value, WasInserted);
+}
+
/// This class is a shared cache, that caches the 'stat' and 'open' calls to the
/// underlying real file system. It distinguishes between minimized and original
/// files.
@@ -110,67 +105,71 @@
/// the worker threads.
class DependencyScanningFilesystemSharedCache {
public:
- struct SharedFileSystemEntry {
- std::mutex ValueLock;
- CachedFileSystemEntry Value;
- };
+ DependencyScanningFilesystemSharedCache();
+
+ /// Returns the cached stat entry for directory entry. Creates invalid status
+ /// if not found. This is a thread safe call.
+ SharedStat &getStat(StringRef Path);
- /// Returns a cache entry for the corresponding key.
- ///
- /// A new cache entry is created if the key is not in the cache. This is a
- /// thread safe call.
- SharedFileSystemEntry &get(StringRef Key, bool Minimized);
+ /// Returns the cached original contents for file. Creates empty contents if
+ /// not found. This is a thread safe call.
+ InsertResult<OriginalContents>
+ getOriginalContents(StringRef Filename, llvm::sys::fs::UniqueID UID);
+
+ /// Returns the cached minimized contents for file. Creates empty contents if
+ /// not found. This is a thread safe call.
+ InsertResult<MinimizedContents>
+ getMinimizedContents(StringRef Filename, llvm::sys::fs::UniqueID UID);
private:
- class SingleCache {
- public:
- SingleCache();
-
- SharedFileSystemEntry &get(StringRef Key);
-
- private:
- struct CacheShard {
- std::mutex CacheLock;
- llvm::StringMap<SharedFileSystemEntry, llvm::BumpPtrAllocator> Cache;
- };
- std::unique_ptr<CacheShard[]> CacheShards;
- unsigned NumShards;
+ struct CacheShard {
+ /// The mutex that's locked whenever insertion into any of the caches takes
+ /// place.
+ std::mutex CacheLock;
+ /// Cache of stat call results.
+ llvm::StringMap<SharedStat, llvm::BumpPtrAllocator> StatCache;
+ /// Cache of original file contents. Using \c std::map to ensure references
+ /// are not invalidated on insertion.
+ std::map<llvm::sys::fs::UniqueID, OriginalContents> OriginalContentsCache;
+ /// Cache of minimized file contents.
+ std::map<llvm::sys::fs::UniqueID, MinimizedContents> MinimizedContentsCache;
};
-
- SingleCache CacheMinimized;
- SingleCache CacheOriginal;
+ std::unique_ptr<CacheShard[]> CacheShards;
+ unsigned NumShards;
};
/// This class is a local cache, that caches the 'stat' and 'open' calls to the
/// underlying real file system. It distinguishes between minimized and original
/// files.
class DependencyScanningFilesystemLocalCache {
-private:
- using SingleCache =
- llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator>;
-
- SingleCache CacheMinimized;
- SingleCache CacheOriginal;
-
- SingleCache &selectCache(bool Minimized) {
- return Minimized ? CacheMinimized : CacheOriginal;
- }
+ /// Cache of stat call results, pointing into the shared cache.
+ llvm::StringMap<const MaybeStat *, llvm::BumpPtrAllocator> StatCache;
+ /// Cache of original file contents, pointing into the shared cache.
+ llvm::DenseMap<llvm::sys::fs::UniqueID, const OriginalContents *>
+ OriginalContentsCache;
+ /// Cache of minimized file contents, pointing into the shared cache.
+ llvm::DenseMap<llvm::sys::fs::UniqueID, const MinimizedContents *>
+ MinimizedContentsCache;
public:
- void setCachedEntry(StringRef Filename, bool Minimized,
- const CachedFileSystemEntry *Entry) {
- SingleCache &Cache = selectCache(Minimized);
- bool IsInserted = Cache.try_emplace(Filename, Entry).second;
- (void)IsInserted;
- assert(IsInserted && "local cache is updated more than once");
- }
-
- const CachedFileSystemEntry *getCachedEntry(StringRef Filename,
- bool Minimized) {
- SingleCache &Cache = selectCache(Minimized);
- auto It = Cache.find(Filename);
- return It == Cache.end() ? nullptr : It->getValue();
- }
+ /// Returns the status result for the path, or nullptr when not found.
+ const MaybeStat *findStat(StringRef Path);
+ /// Inserts the given status result into the cache for the path.
+ const MaybeStat *insertStat(StringRef Path, const MaybeStat *Status);
+
+ /// Returns the original contents for the file, or nullptr when not found.
+ const OriginalContents *findOriginalContents(llvm::sys::fs::UniqueID UID);
+ /// Inserts the given original contents into the cache for the file.
+ const OriginalContents *
+ insertOriginalContents(llvm::sys::fs::UniqueID UID,
+ const OriginalContents *Contents);
+
+ /// Returns the minimized contents for the file, or nullptr when not found.
+ const MinimizedContents *findMinimizedContents(llvm::sys::fs::UniqueID UID);
+ /// Inserts the given minimized contents into the cache for the file.
+ const MinimizedContents *
+ insertMinimizedContents(llvm::sys::fs::UniqueID UID,
+ const MinimizedContents *Contents);
};
/// A virtual file system optimized for the dependency discovery.
@@ -204,13 +203,32 @@
/// Check whether the file should be minimized.
bool shouldMinimize(StringRef Filename);
- llvm::ErrorOr<const CachedFileSystemEntry *>
- getOrCreateFileSystemEntry(const StringRef Filename);
-
- /// Create a cached file system entry based on the initial status result.
- CachedFileSystemEntry
- createFileSystemEntry(llvm::ErrorOr<llvm::vfs::Status> &&MaybeStatus,
- StringRef Filename, bool ShouldMinimize);
+ /// Ensure \p Lock is locked with the mutex for entry at \p Path.
+ /// Also assigns to \p SS if null.
+ void ensureLocked(StringRef Path, SharedStat *&SS,
+ llvm::Optional<std::unique_lock<std::mutex>> &Lock);
+
+ /// Get the status for \p Path from local cache, populating it if necessary.
+ const MaybeStat *
+ getOrCreateLocalStat(StringRef Path, SharedStat *&SS,
+ llvm::Optional<std::unique_lock<std::mutex>> &Lock);
+
+ /// Get the original contents for \p Filename from local cache, populating it
+ /// if necessary.
+ const OriginalContents *getOrCreateLocalOriginalContents(
+ StringRef Filename, llvm::sys::fs::UniqueID UID, SharedStat *&SharedStat,
+ llvm::Optional<std::unique_lock<std::mutex>> &Lock);
+
+ /// Get the minimized contents for \p Filename from local cache, populating it
+ /// if necessary.
+ const MinimizedContents *getOrCreateLocalMinimizedContents(
+ StringRef Filename, llvm::sys::fs::UniqueID UID, SharedStat *&SharedStat,
+ llvm::Optional<std::unique_lock<std::mutex>> &Lock);
+
+ /// Get the full filesystem entry for \p Path from local cache, populating it
+ /// if necessary.
+ llvm::ErrorOr<FileSystemEntryResult>
+ getOrCreateFileSystemEntry(StringRef Path);
/// The global cache shared between worker threads.
DependencyScanningFilesystemSharedCache &SharedCache;
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
