https://github.com/jansvoboda11 updated https://github.com/llvm/llvm-project/pull/184917
>From bb383121ee530fa1e9f00c5fcfc34e0b83d49a3a Mon Sep 17 00:00:00 2001 From: Jan Svoboda <[email protected]> Date: Thu, 5 Mar 2026 14:21:08 -0800 Subject: [PATCH 1/3] [clang][deps] Expand `DependencyActionController` APIs --- .../DependencyScannerImpl.h | 3 +- .../DependencyScanningUtils.h | 6 ++- .../DependencyScanningWorker.h | 47 ++++++++++++++++++ .../clang/Tooling/DependencyScanningTool.h | 19 ++++---- .../DependencyScannerImpl.cpp | 48 ++++++++++++++----- .../DependencyScanning/ModuleDepCollector.cpp | 3 ++ clang/lib/Tooling/DependencyScanningTool.cpp | 39 +++++++++++---- clang/tools/clang-scan-deps/ClangScanDeps.cpp | 2 +- 8 files changed, 135 insertions(+), 32 deletions(-) diff --git a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h index a54a6269dbdc4..9b1aa22a5c9f8 100644 --- a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h +++ b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h @@ -96,7 +96,8 @@ void canonicalizeDefines(PreprocessorOptions &PPOpts); /// Creates a CompilerInvocation suitable for the dependency scanner. std::shared_ptr<CompilerInvocation> createScanCompilerInvocation(const CompilerInvocation &Invocation, - const DependencyScanningService &Service); + const DependencyScanningService &Service, + DependencyActionController &Controller); /// Creates dependency output options to be reported to the dependency consumer, /// deducing missing information if necessary. diff --git a/clang/include/clang/DependencyScanning/DependencyScanningUtils.h b/clang/include/clang/DependencyScanning/DependencyScanningUtils.h index 124f1eaa6cbba..0eb9a9e777f1d 100644 --- a/clang/include/clang/DependencyScanning/DependencyScanningUtils.h +++ b/clang/include/clang/DependencyScanning/DependencyScanningUtils.h @@ -147,12 +147,16 @@ class CallbackActionController : public DependencyActionController { } } + std::unique_ptr<DependencyActionController> clone() const override { + return std::make_unique<CallbackActionController>(LookupModuleOutput); + } + std::string lookupModuleOutput(const ModuleDeps &MD, ModuleOutputKind Kind) override { return LookupModuleOutput(MD, Kind); } -private: +protected: LookupModuleOutputCallback LookupModuleOutput; }; diff --git a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h index 92da219d85d56..bf7717cac48f5 100644 --- a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h @@ -75,8 +75,55 @@ class DependencyActionController { public: virtual ~DependencyActionController(); + /// Creates a thread-safe copy of the controller. + virtual std::unique_ptr<DependencyActionController> clone() const = 0; + + /// Provides output path for a given module dependency. Must be thread-safe. virtual std::string lookupModuleOutput(const ModuleDeps &MD, ModuleOutputKind Kind) = 0; + + /// Initializes the scan invocation. + virtual void initializeScanInvocation(CompilerInvocation &ScanInvocation) {} + + /// Initializes the scan instance and modifies the resulting TU invocation. + /// Returns true on success, false on failure. + virtual bool initialize(CompilerInstance &ScanInstance, + CompilerInvocation &NewInvocation) { + return true; + } + + /// Finalizes the scan instance and modifies the resulting TU invocation. + /// Returns true on success, false on failure. + virtual bool finalize(CompilerInstance &ScanInstance, + CompilerInvocation &NewInvocation) { + return true; + } + + /// Returns the cache key for the resulting invocation, or nullopt. + virtual std::optional<std::string> + getCacheKey(const CompilerInvocation &NewInvocation) { + return std::nullopt; + } + + /// Initializes the module scan instance. + /// Returns true on success, false on failure. + virtual bool initializeModuleBuild(CompilerInstance &ModuleScanInstance) { + return true; + } + + /// Finalizes the module scan instance. + /// Returns true on success, false on failure. + virtual bool finalizeModuleBuild(CompilerInstance &ModuleScanInstance) { + return true; + } + + /// Modifies the resulting module invocation and the associated structure. + /// Returns true on success, false on failure. + virtual bool finalizeModuleInvocation(CompilerInstance &ScanInstance, + CowCompilerInvocation &CI, + const ModuleDeps &MD) { + return true; + } }; /// An individual dependency scanning worker that is able to run on its own diff --git a/clang/include/clang/Tooling/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanningTool.h index 30846ae0ebf3e..c845e212ce153 100644 --- a/clang/include/clang/Tooling/DependencyScanningTool.h +++ b/clang/include/clang/Tooling/DependencyScanningTool.h @@ -172,7 +172,8 @@ class CompilerInstanceWithContext { const std::vector<std::string> &CMD) : Worker(Worker), CWD(CWD), CommandLine(CMD) {}; - bool initialize(std::unique_ptr<dependencies::DiagnosticsEngineWithDiagOpts> + bool initialize(dependencies::DependencyActionController &Controller, + std::unique_ptr<dependencies::DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts, IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS); @@ -187,10 +188,11 @@ class CompilerInstanceWithContext { /// command. /// @param DC A diagnostics consumer to report error if the initialization /// fails. - static std::optional<CompilerInstanceWithContext> - initializeFromCommandline(DependencyScanningTool &Tool, StringRef CWD, - ArrayRef<std::string> CommandLine, - DiagnosticConsumer &DC); + static std::optional<CompilerInstanceWithContext> initializeFromCommandline( + DependencyScanningTool &Tool, StringRef CWD, + ArrayRef<std::string> CommandLine, + dependencies::DependencyActionController &Controller, + DiagnosticConsumer &DC); /// @brief Initializing the context and the compiler instance. /// This method must be called before calling @@ -198,9 +200,10 @@ class CompilerInstanceWithContext { /// @param CWD The current working directory used during the scan. /// @param CommandLine The commandline used for the scan. /// @return Error if the initializaiton fails. - static llvm::Expected<CompilerInstanceWithContext> - initializeOrError(DependencyScanningTool &Tool, StringRef CWD, - ArrayRef<std::string> CommandLine); + static llvm::Expected<CompilerInstanceWithContext> initializeOrError( + DependencyScanningTool &Tool, StringRef CWD, + ArrayRef<std::string> CommandLine, + dependencies::LookupModuleOutputCallback LookupModuleOutput); bool computeDependencies(StringRef ModuleName, diff --git a/clang/lib/DependencyScanning/DependencyScannerImpl.cpp b/clang/lib/DependencyScanning/DependencyScannerImpl.cpp index 0e345af8817ae..a8c9bdbcf4d92 100644 --- a/clang/lib/DependencyScanning/DependencyScannerImpl.cpp +++ b/clang/lib/DependencyScanning/DependencyScannerImpl.cpp @@ -434,7 +434,8 @@ void dependencies::initializeScanCompilerInstance( std::shared_ptr<CompilerInvocation> dependencies::createScanCompilerInvocation( const CompilerInvocation &Invocation, - const DependencyScanningService &Service) { + const DependencyScanningService &Service, + DependencyActionController &Controller) { auto ScanInvocation = std::make_shared<CompilerInvocation>(Invocation); sanitizeDiagOpts(ScanInvocation->getDiagnosticOpts()); @@ -475,6 +476,8 @@ std::shared_ptr<CompilerInvocation> dependencies::createScanCompilerInvocation( // and thus won't write out the extra '.d' files to disk. ScanInvocation->getDependencyOutputOpts() = {}; + Controller.initializeScanInvocation(*ScanInvocation); + return ScanInvocation; } @@ -583,11 +586,13 @@ class AsyncModuleCompiles { struct SingleModuleWithAsyncModuleCompiles : PreprocessOnlyAction { DependencyScanningService &Service; + DependencyActionController &Controller; AsyncModuleCompiles &Compiles; SingleModuleWithAsyncModuleCompiles(DependencyScanningService &Service, + DependencyActionController &Controller, AsyncModuleCompiles &Compiles) - : Service(Service), Compiles(Compiles) {} + : Service(Service), Controller(Controller), Compiles(Compiles) {} bool BeginSourceFileAction(CompilerInstance &CI) override; }; @@ -597,11 +602,13 @@ struct SingleModuleWithAsyncModuleCompiles : PreprocessOnlyAction { struct AsyncModuleCompile : PPCallbacks { CompilerInstance &CI; DependencyScanningService &Service; + DependencyActionController &Controller; AsyncModuleCompiles &Compiles; AsyncModuleCompile(CompilerInstance &CI, DependencyScanningService &Service, + DependencyActionController &Controller, AsyncModuleCompiles &Compiles) - : CI(CI), Service(Service), Compiles(Compiles) {} + : CI(CI), Service(Service), Controller(Controller), Compiles(Compiles) {} void moduleLoadSkipped(Module *M) override { M = M->getTopLevelModule(); @@ -665,16 +672,20 @@ struct AsyncModuleCompile : PPCallbacks { auto ModCI2 = CI.cloneForModuleCompile(SourceLocation(), M, ModuleFileName, CloneConfig); + auto ModController = Controller.clone(); + // Note: This lock belongs to a module cache that might not outlive the // thread. This works, because the in-process lock only refers to an object // managed by the service, which does outlive the thread. Compiles.add([Lock = std::move(Lock), ModCI1 = std::move(ModCI1), ModCI2 = std::move(ModCI2), DC = std::move(DC), - Service = &Service, Compiles = &Compiles] { + ModController = std::move(ModController), Service = &Service, + Compiles = &Compiles] { llvm::CrashRecoveryContext CRC; (void)CRC.RunSafely([&] { // Quickly discovers and compiles modules for the real scan below. - SingleModuleWithAsyncModuleCompiles Action1(*Service, *Compiles); + SingleModuleWithAsyncModuleCompiles Action1(*Service, *ModController, + *Compiles); (void)ModCI1->ExecuteAction(Action1); // The real scan below. ModCI2->getPreprocessorOpts().SingleModuleParseMode = false; @@ -689,16 +700,18 @@ struct AsyncModuleCompile : PPCallbacks { /// modules asynchronously without blocking or importing them. struct SingleTUWithAsyncModuleCompiles : PreprocessOnlyAction { DependencyScanningService &Service; + DependencyActionController &Controller; AsyncModuleCompiles &Compiles; SingleTUWithAsyncModuleCompiles(DependencyScanningService &Service, + DependencyActionController &Controller, AsyncModuleCompiles &Compiles) - : Service(Service), Compiles(Compiles) {} + : Service(Service), Controller(Controller), Compiles(Compiles) {} bool BeginSourceFileAction(CompilerInstance &CI) override { CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true; - CI.getPreprocessor().addPPCallbacks( - std::make_unique<AsyncModuleCompile>(CI, Service, Compiles)); + CI.getPreprocessor().addPPCallbacks(std::make_unique<AsyncModuleCompile>( + CI, Service, Controller, Compiles)); return true; } }; @@ -707,7 +720,7 @@ bool SingleModuleWithAsyncModuleCompiles::BeginSourceFileAction( CompilerInstance &CI) { CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true; CI.getPreprocessor().addPPCallbacks( - std::make_unique<AsyncModuleCompile>(CI, Service, Compiles)); + std::make_unique<AsyncModuleCompile>(CI, Service, Controller, Compiles)); return true; } @@ -723,12 +736,18 @@ bool DependencyScanningAction::runInvocation( canonicalizeDefines(OriginalInvocation->getPreprocessorOpts()); if (Scanned) { + CompilerInstance &ScanInstance = *ScanInstanceStorage; + // Scanning runs once for the first -cc1 invocation in a chain of driver // jobs. For any dependent jobs, reuse the scanning result and just // update the new invocation. // FIXME: to support multi-arch builds, each arch requires a separate scan if (MDC) MDC->applyDiscoveredDependencies(*OriginalInvocation); + + if (!Controller.finalize(ScanInstance, *OriginalInvocation)) + return false; + Consumer.handleBuildCommand( {Executable, OriginalInvocation->getCC1CommandLine()}); return true; @@ -738,7 +757,7 @@ bool DependencyScanningAction::runInvocation( // Create a compiler instance to handle the actual work. auto ScanInvocation = - createScanCompilerInvocation(*OriginalInvocation, Service); + createScanCompilerInvocation(*OriginalInvocation, Service, Controller); // Quickly discovers and compiles modules for the real scan below. std::optional<AsyncModuleCompiles> AsyncCompiles; @@ -765,7 +784,7 @@ bool DependencyScanningAction::runInvocation( ScanInstance.getLangOpts().CompilingPCH = true; AsyncCompiles.emplace(); - SingleTUWithAsyncModuleCompiles Action(Service, *AsyncCompiles); + SingleTUWithAsyncModuleCompiles Action(Service, Controller, *AsyncCompiles); (void)ScanInstance.ExecuteAction(Action); } @@ -793,12 +812,19 @@ bool DependencyScanningAction::runInvocation( if (ScanInstance.getDiagnostics().hasErrorOccurred()) return false; + if (!Controller.initialize(ScanInstance, *OriginalInvocation)) + return false; + ReadPCHAndPreprocessAction Action; const bool Result = ScanInstance.ExecuteAction(Action); if (Result) { if (MDC) MDC->applyDiscoveredDependencies(*OriginalInvocation); + + if (!Controller.finalize(ScanInstance, *OriginalInvocation)) + return false; + Consumer.handleBuildCommand( {Executable, OriginalInvocation->getCC1CommandLine()}); } diff --git a/clang/lib/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/DependencyScanning/ModuleDepCollector.cpp index a20abf3c8171f..812881b23b0bd 100644 --- a/clang/lib/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/DependencyScanning/ModuleDepCollector.cpp @@ -791,6 +791,9 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { } }); + // FIXME: Propagate errors up. + (void)MDC.Controller.finalizeModuleInvocation(MDC.ScanInstance, CI, MD); + // Check provided input paths from the invocation for determining // IsInStableDirectories. if (MD.IsInStableDirectories) diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp index 4a013ad86788c..9932a0cea2f6c 100644 --- a/clang/lib/Tooling/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanningTool.cpp @@ -238,6 +238,10 @@ std::optional<P1689Rule> DependencyScanningTool::getP1689ModuleDependencyFile( ModuleOutputKind Kind) override { return ""; } + + std::unique_ptr<DependencyActionController> clone() const override { + return std::make_unique<P1689ActionController>(); + } }; P1689Rule Rule; @@ -342,8 +346,8 @@ DependencyScanningTool::getModuleDependencies( StringRef ModuleName, ArrayRef<std::string> CommandLine, StringRef CWD, const llvm::DenseSet<ModuleID> &AlreadySeen, LookupModuleOutputCallback LookupModuleOutput) { - auto MaybeCIWithContext = - CompilerInstanceWithContext::initializeOrError(*this, CWD, CommandLine); + auto MaybeCIWithContext = CompilerInstanceWithContext::initializeOrError( + *this, CWD, CommandLine, LookupModuleOutput); if (auto Error = MaybeCIWithContext.takeError()) return Error; @@ -376,7 +380,8 @@ static std::optional<SmallVector<std::string, 0>> getFirstCC1CommandLine( std::optional<CompilerInstanceWithContext> CompilerInstanceWithContext::initializeFromCommandline( DependencyScanningTool &Tool, StringRef CWD, - ArrayRef<std::string> CommandLine, DiagnosticConsumer &DC) { + ArrayRef<std::string> CommandLine, DependencyActionController &Controller, + DiagnosticConsumer &DC) { auto [OverlayFS, ModifiedCommandLine] = initVFSForByNameScanning(&Tool.Worker.getVFS(), CommandLine, CWD); auto DiagEngineWithCmdAndOpts = @@ -387,8 +392,8 @@ CompilerInstanceWithContext::initializeFromCommandline( // The input command line is already a -cc1 invocation; initialize the // compiler instance directly from it. CompilerInstanceWithContext CIWithContext(Tool.Worker, CWD, CommandLine); - if (!CIWithContext.initialize(std::move(DiagEngineWithCmdAndOpts), - OverlayFS)) + if (!CIWithContext.initialize( + Controller, std::move(DiagEngineWithCmdAndOpts), OverlayFS)) return std::nullopt; return std::move(CIWithContext); } @@ -405,7 +410,8 @@ CompilerInstanceWithContext::initializeFromCommandline( MaybeFirstCC1->end()); CompilerInstanceWithContext CIWithContext(Tool.Worker, CWD, std::move(CC1CommandLine)); - if (!CIWithContext.initialize(std::move(DiagEngineWithCmdAndOpts), OverlayFS)) + if (!CIWithContext.initialize(Controller, std::move(DiagEngineWithCmdAndOpts), + OverlayFS)) return std::nullopt; return std::move(CIWithContext); } @@ -413,11 +419,16 @@ CompilerInstanceWithContext::initializeFromCommandline( llvm::Expected<CompilerInstanceWithContext> CompilerInstanceWithContext::initializeOrError( DependencyScanningTool &Tool, StringRef CWD, - ArrayRef<std::string> CommandLine) { + ArrayRef<std::string> CommandLine, + LookupModuleOutputCallback LookupModuleOutput) { + // It might seem wasteful to create fresh controller just for initializing the + // compiler instance, but repeated calls to computeDependenciesByNameOrError() + // do that as well, so this gets amortized. + CallbackActionController Controller(LookupModuleOutput); auto DiagPrinterWithOS = std::make_unique<TextDiagnosticsPrinterWithOutput>(CommandLine); - auto Result = initializeFromCommandline(Tool, CWD, CommandLine, + auto Result = initializeFromCommandline(Tool, CWD, CommandLine, Controller, DiagPrinterWithOS->DiagPrinter); if (Result) { Result->DiagPrinterWithOS = std::move(DiagPrinterWithOS); @@ -441,6 +452,7 @@ CompilerInstanceWithContext::computeDependenciesByNameOrError( } bool CompilerInstanceWithContext::initialize( + DependencyActionController &Controller, std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts, IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS) { assert(DiagEngineWithDiagOpts && "Valid diagnostics engine required!"); @@ -473,7 +485,7 @@ bool CompilerInstanceWithContext::initialize( std::shared_ptr<ModuleCache> ModCache = makeInProcessModuleCache(Worker.Service.getModuleCacheEntries()); CIPtr = std::make_unique<CompilerInstance>( - createScanCompilerInvocation(*OriginalInvocation, Worker.Service), + createScanCompilerInvocation(*OriginalInvocation, Worker.Service, Controller), Worker.PCHContainerOps, std::move(ModCache)); auto &CI = *CIPtr; @@ -532,6 +544,10 @@ bool CompilerInstanceWithContext::computeDependencies( invocations of computeDependencies. */ *OriginalInvocation, Controller, PrebuiltModuleASTMap, StableDirs); + CompilerInvocation ModuleInvocation(*OriginalInvocation); + if (!Controller.initialize(CI, ModuleInvocation)) + return false; + if (!SrcLocOffset) { // When SrcLocOffset is zero, we are at the beginning of the fake source // file. In this case, we call BeginSourceFile to initialize. @@ -588,8 +604,11 @@ bool CompilerInstanceWithContext::computeDependencies( if (!ModResult) return false; - CompilerInvocation ModuleInvocation(*OriginalInvocation); MDC->applyDiscoveredDependencies(ModuleInvocation); + + if (!Controller.finalize(CI, ModuleInvocation)) + return false; + Consumer.handleBuildCommand( {CommandLine[0], ModuleInvocation.getCC1CommandLine()}); diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index 737360f9266e6..8666222984ac0 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -1072,7 +1072,7 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { HadErrors = true; } else { auto CIWithCtx = CompilerInstanceWithContext::initializeOrError( - WorkerTool, CWD, Input->CommandLine); + WorkerTool, CWD, Input->CommandLine, LookupOutput); if (llvm::Error Err = CIWithCtx.takeError()) { handleErrorWithInfoString( "Compiler instance with context setup error", std::move(Err), >From 1be145ce8c6ae90b3dc6b0a875ee0f9c725d2243 Mon Sep 17 00:00:00 2001 From: Jan Svoboda <[email protected]> Date: Thu, 5 Mar 2026 15:37:38 -0800 Subject: [PATCH 2/3] clang-format --- clang/lib/Tooling/DependencyScanningTool.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp index 9932a0cea2f6c..adf0c10612468 100644 --- a/clang/lib/Tooling/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanningTool.cpp @@ -485,7 +485,8 @@ bool CompilerInstanceWithContext::initialize( std::shared_ptr<ModuleCache> ModCache = makeInProcessModuleCache(Worker.Service.getModuleCacheEntries()); CIPtr = std::make_unique<CompilerInstance>( - createScanCompilerInvocation(*OriginalInvocation, Worker.Service, Controller), + createScanCompilerInvocation(*OriginalInvocation, Worker.Service, + Controller), Worker.PCHContainerOps, std::move(ModCache)); auto &CI = *CIPtr; >From d008e9b8a586dc1d39dc2d2290cb4446e4fbf728 Mon Sep 17 00:00:00 2001 From: Jan Svoboda <[email protected]> Date: Tue, 10 Mar 2026 09:12:36 -0700 Subject: [PATCH 3/3] Clarify comment --- .../include/clang/DependencyScanning/DependencyScanningWorker.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h index bf7717cac48f5..88ed0f0188913 100644 --- a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h @@ -75,7 +75,7 @@ class DependencyActionController { public: virtual ~DependencyActionController(); - /// Creates a thread-safe copy of the controller. + /// Creates a copy of the controller. The result must be both thread-safe. virtual std::unique_ptr<DependencyActionController> clone() const = 0; /// Provides output path for a given module dependency. Must be thread-safe. _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
