https://github.com/jansvoboda11 created https://github.com/llvm/llvm-project/pull/131193
This PR adds new `ModuleCache` interface to Clang's implicitly-built modules machinery. The main motivation for this change is to create a second implementation that uses a more efficient kind of `llvm::AdvisoryLock` during dependency scanning. In addition to the lock abstraction, the `ModuleCache` interface also manages the existing `InMemoryModuleCache` instance. I found that compared to keeping these separate/independent, the code is a bit simpler now, since these are two tightly coupled concepts. I can envision a more efficient implementation of the `InMemoryModuleCache` for the single-process case too, which will be much easier to implement with the current setup. This is not intended to be a functional change. >From 3ac8635ed7e647be49d3b5e59b4e5be0d19bc629 Mon Sep 17 00:00:00 2001 From: Jan Svoboda <jan_svob...@apple.com> Date: Thu, 13 Mar 2025 10:31:39 -0700 Subject: [PATCH] [clang][modules] Introduce new `ModuleCache` interface --- clang-tools-extra/clangd/ModulesBuilder.cpp | 6 +-- clang/include/clang/Frontend/ASTUnit.h | 4 +- .../include/clang/Frontend/CompilerInstance.h | 13 +++--- clang/include/clang/Serialization/ASTReader.h | 11 ++--- clang/include/clang/Serialization/ASTWriter.h | 21 ++++----- .../include/clang/Serialization/ModuleCache.h | 45 ++++++++++++++++++ .../clang/Serialization/ModuleManager.h | 12 ++--- clang/lib/Frontend/ASTUnit.cpp | 21 ++++----- clang/lib/Frontend/CompilerInstance.cpp | 42 ++++++++--------- clang/lib/Frontend/PrecompiledPreamble.cpp | 5 +- clang/lib/Serialization/ASTReader.cpp | 46 ++++++++++--------- clang/lib/Serialization/ASTWriter.cpp | 12 ++--- clang/lib/Serialization/CMakeLists.txt | 1 + clang/lib/Serialization/GeneratePCH.cpp | 8 ++-- clang/lib/Serialization/ModuleCache.cpp | 44 ++++++++++++++++++ clang/lib/Serialization/ModuleManager.cpp | 20 ++++---- .../unittests/Frontend/FrontendActionTest.cpp | 7 ++- clang/unittests/Lex/HeaderSearchTest.cpp | 1 - 18 files changed, 206 insertions(+), 113 deletions(-) create mode 100644 clang/include/clang/Serialization/ModuleCache.h create mode 100644 clang/lib/Serialization/ModuleCache.cpp diff --git a/clang-tools-extra/clangd/ModulesBuilder.cpp b/clang-tools-extra/clangd/ModulesBuilder.cpp index 08a7b250a8119..44307b8a28b93 100644 --- a/clang-tools-extra/clangd/ModulesBuilder.cpp +++ b/clang-tools-extra/clangd/ModulesBuilder.cpp @@ -12,7 +12,7 @@ #include "clang/Frontend/FrontendAction.h" #include "clang/Frontend/FrontendActions.h" #include "clang/Serialization/ASTReader.h" -#include "clang/Serialization/InMemoryModuleCache.h" +#include "clang/Serialization/ModuleCache.h" #include "llvm/ADT/ScopeExit.h" #include <queue> @@ -206,9 +206,9 @@ bool IsModuleFileUpToDate(PathRef ModuleFilePath, Preprocessor PP(std::make_shared<PreprocessorOptions>(), *Diags, LangOpts, SourceMgr, HeaderInfo, ModuleLoader); - IntrusiveRefCntPtr<InMemoryModuleCache> ModuleCache = new InMemoryModuleCache; + IntrusiveRefCntPtr<ModuleCache> ModCache = getCrossProcessModuleCache(); PCHContainerOperations PCHOperations; - ASTReader Reader(PP, *ModuleCache, /*ASTContext=*/nullptr, + ASTReader Reader(PP, *ModCache, /*ASTContext=*/nullptr, PCHOperations.getRawReader(), {}); // We don't need any listener here. By default it will use a validator diff --git a/clang/include/clang/Frontend/ASTUnit.h b/clang/include/clang/Frontend/ASTUnit.h index 1f98c6ab328ba..248bbe1657f8b 100644 --- a/clang/include/clang/Frontend/ASTUnit.h +++ b/clang/include/clang/Frontend/ASTUnit.h @@ -70,7 +70,7 @@ class FileManager; class FrontendAction; class HeaderSearch; class InputKind; -class InMemoryModuleCache; +class ModuleCache; class PCHContainerOperations; class PCHContainerReader; class Preprocessor; @@ -110,7 +110,7 @@ class ASTUnit { IntrusiveRefCntPtr<DiagnosticsEngine> Diagnostics; IntrusiveRefCntPtr<FileManager> FileMgr; IntrusiveRefCntPtr<SourceManager> SourceMgr; - IntrusiveRefCntPtr<InMemoryModuleCache> ModuleCache; + IntrusiveRefCntPtr<ModuleCache> ModCache; std::unique_ptr<HeaderSearch> HeaderInfo; IntrusiveRefCntPtr<TargetInfo> Target; std::shared_ptr<Preprocessor> PP; diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h index 8b539dfc92960..4960d40ca7c37 100644 --- a/clang/include/clang/Frontend/CompilerInstance.h +++ b/clang/include/clang/Frontend/CompilerInstance.h @@ -51,8 +51,8 @@ class DiagnosticsEngine; class DiagnosticConsumer; class FileManager; class FrontendAction; -class InMemoryModuleCache; class Module; +class ModuleCache; class Preprocessor; class Sema; class SourceManager; @@ -97,7 +97,7 @@ class CompilerInstance : public ModuleLoader { IntrusiveRefCntPtr<SourceManager> SourceMgr; /// The cache of PCM files. - IntrusiveRefCntPtr<InMemoryModuleCache> ModuleCache; + IntrusiveRefCntPtr<ModuleCache> ModCache; /// The preprocessor. std::shared_ptr<Preprocessor> PP; @@ -209,7 +209,7 @@ class CompilerInstance : public ModuleLoader { explicit CompilerInstance( std::shared_ptr<PCHContainerOperations> PCHContainerOps = std::make_shared<PCHContainerOperations>(), - InMemoryModuleCache *SharedModuleCache = nullptr); + ModuleCache *ModCache = nullptr); ~CompilerInstance() override; /// @name High-Level Operations @@ -746,9 +746,8 @@ class CompilerInstance : public ModuleLoader { static IntrusiveRefCntPtr<ASTReader> createPCHExternalASTSource( StringRef Path, StringRef Sysroot, DisableValidationForModuleKind DisableValidation, - bool AllowPCHWithCompilerErrors, Preprocessor &PP, - InMemoryModuleCache &ModuleCache, ASTContext &Context, - const PCHContainerReader &PCHContainerRdr, + bool AllowPCHWithCompilerErrors, Preprocessor &PP, ModuleCache &ModCache, + ASTContext &Context, const PCHContainerReader &PCHContainerRdr, ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions, ArrayRef<std::shared_ptr<DependencyCollector>> DependencyCollectors, void *DeserializationListener, bool OwnDeserializationListener, @@ -896,7 +895,7 @@ class CompilerInstance : public ModuleLoader { void setExternalSemaSource(IntrusiveRefCntPtr<ExternalSemaSource> ESS); - InMemoryModuleCache &getModuleCache() const { return *ModuleCache; } + ModuleCache &getModuleCache() const { return *ModCache; } }; } // end namespace clang diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h index 47301419c76c6..143b798a8348a 100644 --- a/clang/include/clang/Serialization/ASTReader.h +++ b/clang/include/clang/Serialization/ASTReader.h @@ -89,7 +89,7 @@ struct HeaderFileInfo; class HeaderSearchOptions; class LangOptions; class MacroInfo; -class InMemoryModuleCache; +class ModuleCache; class NamedDecl; class NamespaceDecl; class ObjCCategoryDecl; @@ -1742,8 +1742,8 @@ class ASTReader /// /// \param ReadTimer If non-null, a timer used to track the time spent /// deserializing. - ASTReader(Preprocessor &PP, InMemoryModuleCache &ModuleCache, - ASTContext *Context, const PCHContainerReader &PCHContainerRdr, + ASTReader(Preprocessor &PP, ModuleCache &ModCache, ASTContext *Context, + const PCHContainerReader &PCHContainerRdr, ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions, StringRef isysroot = "", DisableValidationForModuleKind DisableValidationKind = @@ -1954,8 +1954,7 @@ class ASTReader /// /// \returns true if an error occurred, false otherwise. static bool readASTFileControlBlock( - StringRef Filename, FileManager &FileMgr, - const InMemoryModuleCache &ModuleCache, + StringRef Filename, FileManager &FileMgr, const ModuleCache &ModCache, const PCHContainerReader &PCHContainerRdr, bool FindModuleFileExtensions, ASTReaderListener &Listener, bool ValidateDiagnosticOptions, unsigned ClientLoadCapabilities = ARR_ConfigurationMismatch | @@ -1964,7 +1963,7 @@ class ASTReader /// Determine whether the given AST file is acceptable to load into a /// translation unit with the given language and target options. static bool isAcceptableASTFile(StringRef Filename, FileManager &FileMgr, - const InMemoryModuleCache &ModuleCache, + const ModuleCache &ModCache, const PCHContainerReader &PCHContainerRdr, const LangOptions &LangOpts, const TargetOptions &TargetOpts, diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h index 7371a679988e9..6b5cc181e2e13 100644 --- a/clang/include/clang/Serialization/ASTWriter.h +++ b/clang/include/clang/Serialization/ASTWriter.h @@ -60,7 +60,7 @@ class LangOptions; class MacroDefinitionRecord; class MacroInfo; class Module; -class InMemoryModuleCache; +class ModuleCache; class ModuleFileExtension; class ModuleFileExtensionWriter; class NamedDecl; @@ -117,7 +117,7 @@ class ASTWriter : public ASTDeserializationListener, const SmallVectorImpl<char> &Buffer; /// The PCM manager which manages memory buffers for pcm files. - InMemoryModuleCache &ModuleCache; + ModuleCache &ModCache; /// The preprocessor we're writing. Preprocessor *PP = nullptr; @@ -682,7 +682,7 @@ class ASTWriter : public ASTDeserializationListener, /// Create a new precompiled header writer that outputs to /// the given bitstream. ASTWriter(llvm::BitstreamWriter &Stream, SmallVectorImpl<char> &Buffer, - InMemoryModuleCache &ModuleCache, + ModuleCache &ModCache, ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions, bool IncludeTimestamps = true, bool BuildingImplicitModule = false, bool GeneratingReducedBMI = false); @@ -986,9 +986,8 @@ class PCHGenerator : public SemaConsumer { virtual Module *getEmittingModule(ASTContext &Ctx); public: - PCHGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache, - StringRef OutputFile, StringRef isysroot, - std::shared_ptr<PCHBuffer> Buffer, + PCHGenerator(Preprocessor &PP, ModuleCache &ModCache, StringRef OutputFile, + StringRef isysroot, std::shared_ptr<PCHBuffer> Buffer, ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions, bool AllowASTWithErrors = false, bool IncludeTimestamps = true, bool BuildingImplicitModule = false, @@ -1010,14 +1009,14 @@ class CXX20ModulesGenerator : public PCHGenerator { protected: virtual Module *getEmittingModule(ASTContext &Ctx) override; - CXX20ModulesGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache, + CXX20ModulesGenerator(Preprocessor &PP, ModuleCache &ModCache, StringRef OutputFile, bool GeneratingReducedBMI, bool AllowASTWithErrors); public: - CXX20ModulesGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache, + CXX20ModulesGenerator(Preprocessor &PP, ModuleCache &ModCache, StringRef OutputFile, bool AllowASTWithErrors = false) - : CXX20ModulesGenerator(PP, ModuleCache, OutputFile, + : CXX20ModulesGenerator(PP, ModCache, OutputFile, /*GeneratingReducedBMI=*/false, AllowASTWithErrors) {} @@ -1028,9 +1027,9 @@ class ReducedBMIGenerator : public CXX20ModulesGenerator { void anchor() override; public: - ReducedBMIGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache, + ReducedBMIGenerator(Preprocessor &PP, ModuleCache &ModCache, StringRef OutputFile, bool AllowASTWithErrors = false) - : CXX20ModulesGenerator(PP, ModuleCache, OutputFile, + : CXX20ModulesGenerator(PP, ModCache, OutputFile, /*GeneratingReducedBMI=*/true, AllowASTWithErrors) {} }; diff --git a/clang/include/clang/Serialization/ModuleCache.h b/clang/include/clang/Serialization/ModuleCache.h new file mode 100644 index 0000000000000..e5f55111259fd --- /dev/null +++ b/clang/include/clang/Serialization/ModuleCache.h @@ -0,0 +1,45 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_SERIALIZATION_MODULECACHE_H +#define LLVM_CLANG_SERIALIZATION_MODULECACHE_H + +#include "clang/Basic/LLVM.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" + +namespace llvm { +class AdvisoryLock; +} // namespace llvm + +namespace clang { +class InMemoryModuleCache; + +/// The module cache used by implicitly-built modules. +class ModuleCache : public RefCountedBase<ModuleCache> { +public: + /// May perform any work that only needs to be performed once for multiple + /// calls \c getLock() with the same module filename. + virtual void prepareForGetLock(StringRef ModuleFilename) = 0; + + /// Returns lock for the given module file. The lock is initially unlocked. + virtual std::unique_ptr<llvm::AdvisoryLock> + getLock(StringRef ModuleFilename) = 0; + + /// Returns this process's view of the module cache. + virtual InMemoryModuleCache &getInMemoryModuleCache() = 0; + virtual const InMemoryModuleCache &getInMemoryModuleCache() const = 0; + + // TODO: Virtualize writing/reading PCM files, timestamp files, etc. + + virtual ~ModuleCache() = default; +}; + +IntrusiveRefCntPtr<ModuleCache> getCrossProcessModuleCache(); +} // namespace clang + +#endif diff --git a/clang/include/clang/Serialization/ModuleManager.h b/clang/include/clang/Serialization/ModuleManager.h index f898dab39f06d..1eb74aee9787c 100644 --- a/clang/include/clang/Serialization/ModuleManager.h +++ b/clang/include/clang/Serialization/ModuleManager.h @@ -37,7 +37,7 @@ class FileEntry; class FileManager; class GlobalModuleIndex; class HeaderSearch; -class InMemoryModuleCache; +class ModuleCache; class PCHContainerReader; namespace serialization { @@ -65,7 +65,7 @@ class ModuleManager { FileManager &FileMgr; /// Cache of PCM files. - IntrusiveRefCntPtr<InMemoryModuleCache> ModuleCache; + IntrusiveRefCntPtr<ModuleCache> ModCache; /// Knows how to unwrap module containers. const PCHContainerReader &PCHContainerRdr; @@ -133,9 +133,9 @@ class ModuleManager { SmallVectorImpl<std::unique_ptr<ModuleFile>>::reverse_iterator>; using ModuleOffset = std::pair<uint32_t, StringRef>; - explicit ModuleManager(FileManager &FileMgr, InMemoryModuleCache &ModuleCache, - const PCHContainerReader &PCHContainerRdr, - const HeaderSearch &HeaderSearchInfo); + ModuleManager(FileManager &FileMgr, ModuleCache &ModCache, + const PCHContainerReader &PCHContainerRdr, + const HeaderSearch &HeaderSearchInfo); /// Forward iterator to traverse all loaded modules. ModuleIterator begin() { return Chain.begin(); } @@ -306,7 +306,7 @@ class ModuleManager { /// View the graphviz representation of the module graph. void viewGraph(); - InMemoryModuleCache &getModuleCache() const { return *ModuleCache; } + ModuleCache &getModuleCache() const { return *ModCache; } }; } // namespace serialization diff --git a/clang/lib/Frontend/ASTUnit.cpp b/clang/lib/Frontend/ASTUnit.cpp index 8df5465ad990d..3eb9959f45857 100644 --- a/clang/lib/Frontend/ASTUnit.cpp +++ b/clang/lib/Frontend/ASTUnit.cpp @@ -61,7 +61,7 @@ #include "clang/Serialization/ASTReader.h" #include "clang/Serialization/ASTWriter.h" #include "clang/Serialization/ContinuousRangeMap.h" -#include "clang/Serialization/InMemoryModuleCache.h" +#include "clang/Serialization/ModuleCache.h" #include "clang/Serialization/ModuleFile.h" #include "clang/Serialization/PCHContainerOperations.h" #include "llvm/ADT/ArrayRef.h" @@ -219,8 +219,8 @@ struct ASTUnit::ASTWriterData { llvm::BitstreamWriter Stream; ASTWriter Writer; - ASTWriterData(InMemoryModuleCache &ModuleCache) - : Stream(Buffer), Writer(Stream, Buffer, ModuleCache, {}) {} + ASTWriterData(ModuleCache &ModCache) + : Stream(Buffer), Writer(Stream, Buffer, ModCache, {}) {} }; void ASTUnit::clearFileLevelDecls() { @@ -829,7 +829,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile( AST->SourceMgr = new SourceManager(AST->getDiagnostics(), AST->getFileManager(), UserFilesAreVolatile); - AST->ModuleCache = new InMemoryModuleCache; + AST->ModCache = getCrossProcessModuleCache(); AST->HSOpts = HSOpts ? HSOpts : std::make_shared<HeaderSearchOptions>(); AST->HSOpts->ModuleFormat = std::string(PCHContainerRdr.getFormats().front()); AST->HeaderInfo.reset(new HeaderSearch(AST->HSOpts, @@ -861,8 +861,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile( if (::getenv("LIBCLANG_DISABLE_PCH_VALIDATION")) disableValid = DisableValidationForModuleKind::All; AST->Reader = new ASTReader( - PP, *AST->ModuleCache, AST->Ctx.get(), PCHContainerRdr, {}, - /*isysroot=*/"", + PP, *AST->ModCache, AST->Ctx.get(), PCHContainerRdr, {}, /*isysroot=*/"", /*DisableValidationKind=*/disableValid, AllowASTWithCompilerErrors); unsigned Counter = 0; @@ -1546,7 +1545,7 @@ ASTUnit::create(std::shared_ptr<CompilerInvocation> CI, AST->UserFilesAreVolatile = UserFilesAreVolatile; AST->SourceMgr = new SourceManager(AST->getDiagnostics(), *AST->FileMgr, UserFilesAreVolatile); - AST->ModuleCache = new InMemoryModuleCache; + AST->ModCache = getCrossProcessModuleCache(); return AST; } @@ -1833,7 +1832,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromCommandLine( AST->FileMgr = new FileManager(AST->FileSystemOpts, VFS); AST->StorePreamblesInMemory = StorePreamblesInMemory; AST->PreambleStoragePath = PreambleStoragePath; - AST->ModuleCache = new InMemoryModuleCache; + AST->ModCache = getCrossProcessModuleCache(); AST->OnlyLocalDecls = OnlyLocalDecls; AST->CaptureDiagnostics = CaptureDiagnostics; AST->TUKind = TUKind; @@ -1844,7 +1843,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromCommandLine( AST->Invocation = CI; AST->SkipFunctionBodies = SkipFunctionBodies; if (ForSerialization) - AST->WriterData.reset(new ASTWriterData(*AST->ModuleCache)); + AST->WriterData.reset(new ASTWriterData(*AST->ModCache)); // Zero out now to ease cleanup during crash recovery. CI = nullptr; Diags = nullptr; @@ -2379,8 +2378,8 @@ bool ASTUnit::serialize(raw_ostream &OS) { SmallString<128> Buffer; llvm::BitstreamWriter Stream(Buffer); - InMemoryModuleCache ModuleCache; - ASTWriter Writer(Stream, Buffer, ModuleCache, {}); + IntrusiveRefCntPtr<ModuleCache> ModCache = getCrossProcessModuleCache(); + ASTWriter Writer(Stream, Buffer, *ModCache, {}); return serializeUnit(Writer, Buffer, getSema(), OS); } diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index fc350f2ba42e0..b5c4de309c4bf 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -39,16 +39,17 @@ #include "clang/Serialization/ASTReader.h" #include "clang/Serialization/GlobalModuleIndex.h" #include "clang/Serialization/InMemoryModuleCache.h" +#include "clang/Serialization/ModuleCache.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/Statistic.h" #include "llvm/Config/llvm-config.h" +#include "llvm/Support/AdvisoryLock.h" #include "llvm/Support/BuryPointer.h" #include "llvm/Support/CrashRecoveryContext.h" #include "llvm/Support/Errc.h" #include "llvm/Support/FileSystem.h" -#include "llvm/Support/LockFileManager.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/Program.h" @@ -66,11 +67,10 @@ using namespace clang; CompilerInstance::CompilerInstance( std::shared_ptr<PCHContainerOperations> PCHContainerOps, - InMemoryModuleCache *SharedModuleCache) - : ModuleLoader(/* BuildingModule = */ SharedModuleCache), + ModuleCache *ModCache) + : ModuleLoader(/*BuildingModule=*/ModCache), Invocation(new CompilerInvocation()), - ModuleCache(SharedModuleCache ? SharedModuleCache - : new InMemoryModuleCache), + ModCache(ModCache ? ModCache : getCrossProcessModuleCache()), ThePCHContainerOperations(std::move(PCHContainerOps)) {} CompilerInstance::~CompilerInstance() { @@ -205,7 +205,7 @@ IntrusiveRefCntPtr<ASTReader> CompilerInstance::getASTReader() const { return TheASTReader; } void CompilerInstance::setASTReader(IntrusiveRefCntPtr<ASTReader> Reader) { - assert(ModuleCache.get() == &Reader->getModuleManager().getModuleCache() && + assert(ModCache.get() == &Reader->getModuleManager().getModuleCache() && "Expected ASTReader to use the same PCM cache"); TheASTReader = std::move(Reader); } @@ -625,9 +625,8 @@ void CompilerInstance::createPCHExternalASTSource( IntrusiveRefCntPtr<ASTReader> CompilerInstance::createPCHExternalASTSource( StringRef Path, StringRef Sysroot, DisableValidationForModuleKind DisableValidation, - bool AllowPCHWithCompilerErrors, Preprocessor &PP, - InMemoryModuleCache &ModuleCache, ASTContext &Context, - const PCHContainerReader &PCHContainerRdr, + bool AllowPCHWithCompilerErrors, Preprocessor &PP, ModuleCache &ModCache, + ASTContext &Context, const PCHContainerReader &PCHContainerRdr, ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions, ArrayRef<std::shared_ptr<DependencyCollector>> DependencyCollectors, void *DeserializationListener, bool OwnDeserializationListener, @@ -635,7 +634,7 @@ IntrusiveRefCntPtr<ASTReader> CompilerInstance::createPCHExternalASTSource( HeaderSearchOptions &HSOpts = PP.getHeaderSearchInfo().getHeaderSearchOpts(); IntrusiveRefCntPtr<ASTReader> Reader(new ASTReader( - PP, ModuleCache, &Context, PCHContainerRdr, Extensions, + PP, ModCache, &Context, PCHContainerRdr, Extensions, Sysroot.empty() ? "" : Sysroot.data(), DisableValidation, AllowPCHWithCompilerErrors, /*AllowConfigurationMismatch*/ false, HSOpts.ModulesValidateSystemHeaders, HSOpts.ValidateASTInputFilesContent, @@ -1166,7 +1165,8 @@ compileModuleImpl(CompilerInstance &ImportingInstance, SourceLocation ImportLoc, // Never compile a module that's already finalized - this would cause the // existing module to be freed, causing crashes if it is later referenced - if (ImportingInstance.getModuleCache().isPCMFinal(ModuleFileName)) { + if (ImportingInstance.getModuleCache().getInMemoryModuleCache().isPCMFinal( + ModuleFileName)) { ImportingInstance.getDiagnostics().Report( ImportLoc, diag::err_module_rebuild_finalized) << ModuleName; @@ -1477,15 +1477,13 @@ static bool compileModuleAndReadASTBehindLock( Diags.Report(ModuleNameLoc, diag::remark_module_lock) << ModuleFileName << Module->Name; - // FIXME: have LockFileManager return an error_code so that we can - // avoid the mkdir when the directory already exists. - StringRef Dir = llvm::sys::path::parent_path(ModuleFileName); - llvm::sys::fs::create_directories(Dir); + auto &ModuleCache = ImportingInstance.getModuleCache(); + ModuleCache.prepareForGetLock(ModuleFileName); while (true) { - llvm::LockFileManager Lock(ModuleFileName); + auto Lock = ModuleCache.getLock(ModuleFileName); bool Owned; - if (llvm::Error Err = Lock.tryLock().moveInto(Owned)) { + if (llvm::Error Err = Lock->tryLock().moveInto(Owned)) { // ModuleCache takes care of correctness and locks are only necessary for // performance. Fallback to building the module in case of any lock // related errors. @@ -1502,19 +1500,19 @@ static bool compileModuleAndReadASTBehindLock( // Someone else is responsible for building the module. Wait for them to // finish. - switch (Lock.waitForUnlockFor(std::chrono::seconds(90))) { + switch (Lock->waitForUnlockFor(std::chrono::seconds(90))) { case llvm::WaitForUnlockResult::Success: break; // The interesting case. case llvm::WaitForUnlockResult::OwnerDied: continue; // try again to get the lock. case llvm::WaitForUnlockResult::Timeout: - // Since ModuleCache takes care of correctness, we try waiting for - // another process to complete the build so clang does not do it done - // twice. If case of timeout, build it ourselves. + // Since the InMemoryModuleCache takes care of correctness, we try waiting + // for someone else to complete the build so that it does not happen + // twice. In case of timeout, build it ourselves. Diags.Report(ModuleNameLoc, diag::remark_module_lock_timeout) << Module->Name; // Clear the lock file so that future invocations can make progress. - Lock.unsafeMaybeUnlock(); + Lock->unsafeMaybeUnlock(); continue; } diff --git a/clang/lib/Frontend/PrecompiledPreamble.cpp b/clang/lib/Frontend/PrecompiledPreamble.cpp index d1bac09830b04..49fa2e911fea4 100644 --- a/clang/lib/Frontend/PrecompiledPreamble.cpp +++ b/clang/lib/Frontend/PrecompiledPreamble.cpp @@ -292,10 +292,9 @@ class PrecompilePreambleAction : public ASTFrontendAction { class PrecompilePreambleConsumer : public PCHGenerator { public: PrecompilePreambleConsumer(PrecompilePreambleAction &Action, Preprocessor &PP, - InMemoryModuleCache &ModuleCache, - StringRef isysroot, + ModuleCache &ModCache, StringRef isysroot, std::shared_ptr<PCHBuffer> Buffer) - : PCHGenerator(PP, ModuleCache, "", isysroot, std::move(Buffer), + : PCHGenerator(PP, ModCache, "", isysroot, std::move(Buffer), ArrayRef<std::shared_ptr<ModuleFileExtension>>(), /*AllowASTWithErrors=*/true), Action(Action) {} diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index d0a1a9221ce6f..8e9978829c512 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -90,6 +90,7 @@ #include "clang/Serialization/ContinuousRangeMap.h" #include "clang/Serialization/GlobalModuleIndex.h" #include "clang/Serialization/InMemoryModuleCache.h" +#include "clang/Serialization/ModuleCache.h" #include "clang/Serialization/ModuleFile.h" #include "clang/Serialization/ModuleFileExtension.h" #include "clang/Serialization/ModuleManager.h" @@ -3318,9 +3319,12 @@ ASTReader::ReadControlBlock(ModuleFile &F, StoredSignature, Capabilities); // If we diagnosed a problem, produce a backtrace. - bool recompilingFinalized = - Result == OutOfDate && (Capabilities & ARR_OutOfDate) && - getModuleManager().getModuleCache().isPCMFinal(F.FileName); + bool recompilingFinalized = Result == OutOfDate && + (Capabilities & ARR_OutOfDate) && + getModuleManager() + .getModuleCache() + .getInMemoryModuleCache() + .isPCMFinal(F.FileName); if (isDiagnosedResult(Result, Capabilities) || recompilingFinalized) Diag(diag::note_module_file_imported_by) << F.FileName << !F.ModuleName.empty() << F.ModuleName; @@ -5006,7 +5010,7 @@ ASTReader::ReadASTCore(StringRef FileName, bool ShouldFinalizePCM = false; auto FinalizeOrDropPCM = llvm::make_scope_exit([&]() { - auto &MC = getModuleManager().getModuleCache(); + auto &MC = getModuleManager().getModuleCache().getInMemoryModuleCache(); if (ShouldFinalizePCM) MC.finalizePCM(FileName); else @@ -5143,7 +5147,8 @@ ASTReader::readUnhashedControlBlock(ModuleFile &F, bool WasImportedBy, // validation will fail during the as-system import since the PCM on disk // doesn't guarantee that -Werror was respected. However, the -Werror // flags were checked during the initial as-user import. - if (getModuleManager().getModuleCache().isPCMFinal(F.FileName)) { + if (getModuleManager().getModuleCache().getInMemoryModuleCache().isPCMFinal( + F.FileName)) { Diag(diag::warn_module_system_bit_conflict) << F.FileName; return Success; } @@ -5651,14 +5656,14 @@ namespace { } // namespace bool ASTReader::readASTFileControlBlock( - StringRef Filename, FileManager &FileMgr, - const InMemoryModuleCache &ModuleCache, + StringRef Filename, FileManager &FileMgr, const ModuleCache &ModCache, const PCHContainerReader &PCHContainerRdr, bool FindModuleFileExtensions, ASTReaderListener &Listener, bool ValidateDiagnosticOptions, unsigned ClientLoadCapabilities) { // Open the AST file. std::unique_ptr<llvm::MemoryBuffer> OwnedBuffer; - llvm::MemoryBuffer *Buffer = ModuleCache.lookupPCM(Filename); + llvm::MemoryBuffer *Buffer = + ModCache.getInMemoryModuleCache().lookupPCM(Filename); if (!Buffer) { // FIXME: We should add the pcm to the InMemoryModuleCache if it could be // read again later, but we do not have the context here to determine if it @@ -5947,19 +5952,15 @@ bool ASTReader::readASTFileControlBlock( return false; } -bool ASTReader::isAcceptableASTFile(StringRef Filename, FileManager &FileMgr, - const InMemoryModuleCache &ModuleCache, - const PCHContainerReader &PCHContainerRdr, - const LangOptions &LangOpts, - const TargetOptions &TargetOpts, - const PreprocessorOptions &PPOpts, - StringRef ExistingModuleCachePath, - bool RequireStrictOptionMatches) { +bool ASTReader::isAcceptableASTFile( + StringRef Filename, FileManager &FileMgr, const ModuleCache &ModCache, + const PCHContainerReader &PCHContainerRdr, const LangOptions &LangOpts, + const TargetOptions &TargetOpts, const PreprocessorOptions &PPOpts, + StringRef ExistingModuleCachePath, bool RequireStrictOptionMatches) { SimplePCHValidator validator(LangOpts, TargetOpts, PPOpts, ExistingModuleCachePath, FileMgr, RequireStrictOptionMatches); - return !readASTFileControlBlock(Filename, FileMgr, ModuleCache, - PCHContainerRdr, + return !readASTFileControlBlock(Filename, FileMgr, ModCache, PCHContainerRdr, /*FindModuleFileExtensions=*/false, validator, /*ValidateDiagnosticOptions=*/true); } @@ -6496,7 +6497,10 @@ ASTReader::getModulePreprocessedEntities(ModuleFile &Mod) const { bool ASTReader::canRecoverFromOutOfDate(StringRef ModuleFileName, unsigned int ClientLoadCapabilities) { return ClientLoadCapabilities & ARR_OutOfDate && - !getModuleManager().getModuleCache().isPCMFinal(ModuleFileName); + !getModuleManager() + .getModuleCache() + .getInMemoryModuleCache() + .isPCMFinal(ModuleFileName); } llvm::iterator_range<ASTReader::ModuleDeclIterator> @@ -10862,7 +10866,7 @@ void ASTReader::pushExternalDeclIntoScope(NamedDecl *D, DeclarationName Name) { } } -ASTReader::ASTReader(Preprocessor &PP, InMemoryModuleCache &ModuleCache, +ASTReader::ASTReader(Preprocessor &PP, ModuleCache &ModCache, ASTContext *Context, const PCHContainerReader &PCHContainerRdr, ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions, @@ -10878,7 +10882,7 @@ ASTReader::ASTReader(Preprocessor &PP, InMemoryModuleCache &ModuleCache, SourceMgr(PP.getSourceManager()), FileMgr(PP.getFileManager()), PCHContainerRdr(PCHContainerRdr), Diags(PP.getDiagnostics()), StackHandler(Diags), PP(PP), ContextObj(Context), - ModuleMgr(PP.getFileManager(), ModuleCache, PCHContainerRdr, + ModuleMgr(PP.getFileManager(), ModCache, PCHContainerRdr, PP.getHeaderSearchInfo()), DummyIdResolver(PP), ReadTimer(std::move(ReadTimer)), isysroot(isysroot), DisableValidationKind(DisableValidationKind), diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index 0f53d9e137910..9e6da4de680e2 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -75,6 +75,7 @@ #include "clang/Serialization/ASTReader.h" #include "clang/Serialization/ASTRecordWriter.h" #include "clang/Serialization/InMemoryModuleCache.h" +#include "clang/Serialization/ModuleCache.h" #include "clang/Serialization/ModuleFile.h" #include "clang/Serialization/ModuleFileExtension.h" #include "clang/Serialization/SerializationDiagnostic.h" @@ -5326,12 +5327,11 @@ void ASTWriter::SetSelectorOffset(Selector Sel, uint32_t Offset) { } ASTWriter::ASTWriter(llvm::BitstreamWriter &Stream, - SmallVectorImpl<char> &Buffer, - InMemoryModuleCache &ModuleCache, + SmallVectorImpl<char> &Buffer, ModuleCache &ModCache, ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions, bool IncludeTimestamps, bool BuildingImplicitModule, bool GeneratingReducedBMI) - : Stream(Stream), Buffer(Buffer), ModuleCache(ModuleCache), + : Stream(Stream), Buffer(Buffer), ModCache(ModCache), IncludeTimestamps(IncludeTimestamps), BuildingImplicitModule(BuildingImplicitModule), GeneratingReducedBMI(GeneratingReducedBMI) { @@ -5389,9 +5389,9 @@ ASTWriter::WriteAST(llvm::PointerUnion<Sema *, Preprocessor *> Subject, if (ShouldCacheASTInMemory) { // Construct MemoryBuffer and update buffer manager. - ModuleCache.addBuiltPCM(OutputFile, - llvm::MemoryBuffer::getMemBufferCopy( - StringRef(Buffer.begin(), Buffer.size()))); + ModCache.getInMemoryModuleCache().addBuiltPCM( + OutputFile, llvm::MemoryBuffer::getMemBufferCopy( + StringRef(Buffer.begin(), Buffer.size()))); } return Signature; } diff --git a/clang/lib/Serialization/CMakeLists.txt b/clang/lib/Serialization/CMakeLists.txt index b1fc0345047f2..c22987b2c78df 100644 --- a/clang/lib/Serialization/CMakeLists.txt +++ b/clang/lib/Serialization/CMakeLists.txt @@ -18,6 +18,7 @@ add_clang_library(clangSerialization GeneratePCH.cpp GlobalModuleIndex.cpp InMemoryModuleCache.cpp + ModuleCache.cpp ModuleFile.cpp ModuleFileExtension.cpp ModuleManager.cpp diff --git a/clang/lib/Serialization/GeneratePCH.cpp b/clang/lib/Serialization/GeneratePCH.cpp index 12751beb8d715..46ab1be39902a 100644 --- a/clang/lib/Serialization/GeneratePCH.cpp +++ b/clang/lib/Serialization/GeneratePCH.cpp @@ -23,7 +23,7 @@ using namespace clang; PCHGenerator::PCHGenerator( - Preprocessor &PP, InMemoryModuleCache &ModuleCache, StringRef OutputFile, + Preprocessor &PP, ModuleCache &ModCache, StringRef OutputFile, StringRef isysroot, std::shared_ptr<PCHBuffer> Buffer, ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions, bool AllowASTWithErrors, bool IncludeTimestamps, @@ -31,7 +31,7 @@ PCHGenerator::PCHGenerator( bool GeneratingReducedBMI) : PP(PP), Subject(&PP), OutputFile(OutputFile), isysroot(isysroot.str()), Buffer(std::move(Buffer)), Stream(this->Buffer->Data), - Writer(Stream, this->Buffer->Data, ModuleCache, Extensions, + Writer(Stream, this->Buffer->Data, ModCache, Extensions, IncludeTimestamps, BuildingImplicitModule, GeneratingReducedBMI), AllowASTWithErrors(AllowASTWithErrors), ShouldCacheASTInMemory(ShouldCacheASTInMemory) { @@ -100,12 +100,12 @@ ASTDeserializationListener *PCHGenerator::GetASTDeserializationListener() { void PCHGenerator::anchor() {} CXX20ModulesGenerator::CXX20ModulesGenerator(Preprocessor &PP, - InMemoryModuleCache &ModuleCache, + ModuleCache &ModCache, StringRef OutputFile, bool GeneratingReducedBMI, bool AllowASTWithErrors) : PCHGenerator( - PP, ModuleCache, OutputFile, llvm::StringRef(), + PP, ModCache, OutputFile, llvm::StringRef(), std::make_shared<PCHBuffer>(), /*Extensions=*/ArrayRef<std::shared_ptr<ModuleFileExtension>>(), AllowASTWithErrors, /*IncludeTimestamps=*/false, diff --git a/clang/lib/Serialization/ModuleCache.cpp b/clang/lib/Serialization/ModuleCache.cpp new file mode 100644 index 0000000000000..31ff6d1bdfc46 --- /dev/null +++ b/clang/lib/Serialization/ModuleCache.cpp @@ -0,0 +1,44 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/Serialization/ModuleCache.h" + +#include "clang/Serialization/InMemoryModuleCache.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/LockFileManager.h" +#include "llvm/Support/Path.h" + +using namespace clang; + +namespace { +class CrossProcessModuleCache : public ModuleCache { + InMemoryModuleCache InMemory; + +public: + void prepareForGetLock(StringRef ModuleFilename) override { + // FIXME: Do this in LockFileManager and only if the directory doesn't + // exist. + StringRef Dir = llvm::sys::path::parent_path(ModuleFilename); + llvm::sys::fs::create_directories(Dir); + } + + std::unique_ptr<llvm::AdvisoryLock> + getLock(StringRef ModuleFilename) override { + return std::make_unique<llvm::LockFileManager>(ModuleFilename); + } + + InMemoryModuleCache &getInMemoryModuleCache() override { return InMemory; } + const InMemoryModuleCache &getInMemoryModuleCache() const override { + return InMemory; + } +}; +} // namespace + +IntrusiveRefCntPtr<ModuleCache> clang::getCrossProcessModuleCache() { + return llvm::makeIntrusiveRefCnt<CrossProcessModuleCache>(); +} diff --git a/clang/lib/Serialization/ModuleManager.cpp b/clang/lib/Serialization/ModuleManager.cpp index 4ecb776513d6f..61c4e9ed88e9d 100644 --- a/clang/lib/Serialization/ModuleManager.cpp +++ b/clang/lib/Serialization/ModuleManager.cpp @@ -18,6 +18,7 @@ #include "clang/Lex/ModuleMap.h" #include "clang/Serialization/GlobalModuleIndex.h" #include "clang/Serialization/InMemoryModuleCache.h" +#include "clang/Serialization/ModuleCache.h" #include "clang/Serialization/ModuleFile.h" #include "clang/Serialization/PCHContainerOperations.h" #include "llvm/ADT/STLExtras.h" @@ -182,17 +183,20 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type, // Load the contents of the module if (std::unique_ptr<llvm::MemoryBuffer> Buffer = lookupBuffer(FileName)) { // The buffer was already provided for us. - NewModule->Buffer = &ModuleCache->addBuiltPCM(FileName, std::move(Buffer)); + NewModule->Buffer = &getModuleCache().getInMemoryModuleCache().addBuiltPCM( + FileName, std::move(Buffer)); // Since the cached buffer is reused, it is safe to close the file // descriptor that was opened while stat()ing the PCM in // lookupModuleFile() above, it won't be needed any longer. Entry->closeFile(); } else if (llvm::MemoryBuffer *Buffer = - getModuleCache().lookupPCM(FileName)) { + getModuleCache().getInMemoryModuleCache().lookupPCM( + FileName)) { NewModule->Buffer = Buffer; // As above, the file descriptor is no longer needed. Entry->closeFile(); - } else if (getModuleCache().shouldBuildPCM(FileName)) { + } else if (getModuleCache().getInMemoryModuleCache().shouldBuildPCM( + FileName)) { // Report that the module is out of date, since we tried (and failed) to // import it earlier. Entry->closeFile(); @@ -213,7 +217,8 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type, return Missing; } - NewModule->Buffer = &getModuleCache().addPCM(FileName, std::move(*Buf)); + NewModule->Buffer = &getModuleCache().getInMemoryModuleCache().addPCM( + FileName, std::move(*Buf)); } // Initialize the stream. @@ -324,12 +329,11 @@ void ModuleManager::moduleFileAccepted(ModuleFile *MF) { ModulesInCommonWithGlobalIndex.push_back(MF); } -ModuleManager::ModuleManager(FileManager &FileMgr, - InMemoryModuleCache &ModuleCache, +ModuleManager::ModuleManager(FileManager &FileMgr, ModuleCache &ModCache, const PCHContainerReader &PCHContainerRdr, const HeaderSearch &HeaderSearchInfo) - : FileMgr(FileMgr), ModuleCache(&ModuleCache), - PCHContainerRdr(PCHContainerRdr), HeaderSearchInfo(HeaderSearchInfo) {} + : FileMgr(FileMgr), ModCache(&ModCache), PCHContainerRdr(PCHContainerRdr), + HeaderSearchInfo(HeaderSearchInfo) {} void ModuleManager::visit(llvm::function_ref<bool(ModuleFile &M)> Visitor, llvm::SmallPtrSetImpl<ModuleFile *> *ModuleFilesHit) { diff --git a/clang/unittests/Frontend/FrontendActionTest.cpp b/clang/unittests/Frontend/FrontendActionTest.cpp index 75e166767c667..a6bb767e45538 100644 --- a/clang/unittests/Frontend/FrontendActionTest.cpp +++ b/clang/unittests/Frontend/FrontendActionTest.cpp @@ -18,6 +18,7 @@ #include "clang/Lex/PreprocessorOptions.h" #include "clang/Sema/Sema.h" #include "clang/Serialization/InMemoryModuleCache.h" +#include "clang/Serialization/ModuleCache.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/ToolOutputFile.h" #include "llvm/Support/VirtualFileSystem.h" @@ -287,10 +288,12 @@ TEST(GeneratePCHFrontendAction, CacheGeneratedPCH) { // Check whether the PCH was cached. if (ShouldCache) EXPECT_EQ(InMemoryModuleCache::Final, - Compiler.getModuleCache().getPCMState(PCHFilename)); + Compiler.getModuleCache().getInMemoryModuleCache().getPCMState( + PCHFilename)); else EXPECT_EQ(InMemoryModuleCache::Unknown, - Compiler.getModuleCache().getPCMState(PCHFilename)); + Compiler.getModuleCache().getInMemoryModuleCache().getPCMState( + PCHFilename)); } } diff --git a/clang/unittests/Lex/HeaderSearchTest.cpp b/clang/unittests/Lex/HeaderSearchTest.cpp index 4d07150c04e8d..89d096824e600 100644 --- a/clang/unittests/Lex/HeaderSearchTest.cpp +++ b/clang/unittests/Lex/HeaderSearchTest.cpp @@ -16,7 +16,6 @@ #include "clang/Basic/TargetInfo.h" #include "clang/Basic/TargetOptions.h" #include "clang/Lex/HeaderSearchOptions.h" -#include "clang/Serialization/InMemoryModuleCache.h" #include "llvm/Support/MemoryBuffer.h" #include "gtest/gtest.h" #include <memory> _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits