llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-clang-modules <details> <summary>Changes</summary> (to be edited) (not ready for review yet) -- Patch is 66.97 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/66462.diff 28 Files Affected: - (modified) clang-tools-extra/clangd/CMakeLists.txt (+3) - (modified) clang-tools-extra/clangd/ClangdServer.cpp (+1) - (modified) clang-tools-extra/clangd/ClangdServer.h (+3) - (modified) clang-tools-extra/clangd/GlobalCompilationDatabase.cpp (+21) - (modified) clang-tools-extra/clangd/GlobalCompilationDatabase.h (+6) - (added) clang-tools-extra/clangd/ModuleDependencyScanner.cpp (+86) - (added) clang-tools-extra/clangd/ModuleDependencyScanner.h (+78) - (added) clang-tools-extra/clangd/ModuleFilesInfo.cpp (+282) - (added) clang-tools-extra/clangd/ModuleFilesInfo.h (+118) - (modified) clang-tools-extra/clangd/ParsedAST.cpp (+8) - (modified) clang-tools-extra/clangd/Preamble.cpp (+17-6) - (modified) clang-tools-extra/clangd/Preamble.h (+7) - (modified) clang-tools-extra/clangd/TUScheduler.cpp (+13) - (modified) clang-tools-extra/clangd/TUScheduler.h (+3) - (modified) clang-tools-extra/clangd/test/CMakeLists.txt (+1) - (added) clang-tools-extra/clangd/test/modules.test (+79) - (modified) clang-tools-extra/clangd/tool/Check.cpp (+6-2) - (modified) clang-tools-extra/clangd/tool/ClangdMain.cpp (+8) - (modified) clang-tools-extra/clangd/unittests/CMakeLists.txt (+2) - (modified) clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp (+14-4) - (modified) clang-tools-extra/clangd/unittests/FileIndexTests.cpp (+4-1) - (added) clang-tools-extra/clangd/unittests/ModuleDependencyScannerTest.cpp (+173) - (added) clang-tools-extra/clangd/unittests/ModuleFilesInfoTest.cpp (+223) - (added) clang-tools-extra/clangd/unittests/ModulesTestSetup.h (+105) - (modified) clang-tools-extra/clangd/unittests/ParsedASTTests.cpp (+6-2) - (modified) clang-tools-extra/clangd/unittests/PreambleTests.cpp (+4-2) - (modified) clang-tools-extra/clangd/unittests/TestTU.cpp (+10-4) - (modified) clang-tools-extra/docs/ReleaseNotes.rst (+3) <pre> diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt index 3911fb6c6c746a8..bcfb49551a02591 100644 --- a/clang-tools-extra/clangd/CMakeLists.txt +++ b/clang-tools-extra/clangd/CMakeLists.txt @@ -97,6 +97,8 @@ add_clang_library(clangDaemon IncludeFixer.cpp InlayHints.cpp JSONTransport.cpp + ModuleDependencyScanner.cpp + ModuleFilesInfo.cpp PathMapping.cpp Protocol.cpp Quality.cpp @@ -161,6 +163,7 @@ clang_target_link_libraries(clangDaemon clangAST clangASTMatchers clangBasic + clangDependencyScanning clangDriver clangFormat clangFrontend diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp index 13d788162817fb4..e4c85858b6882ae 100644 --- a/clang-tools-extra/clangd/ClangdServer.cpp +++ b/clang-tools-extra/clangd/ClangdServer.cpp @@ -202,6 +202,7 @@ ClangdServer::Options::operator TUScheduler::Options() const { Opts.UpdateDebounce = UpdateDebounce; Opts.ContextProvider = ContextProvider; Opts.PreambleThrottler = PreambleThrottler; + Opts.ExperimentalModulesSupport = ExperimentalModulesSupport; return Opts; } diff --git a/clang-tools-extra/clangd/ClangdServer.h b/clang-tools-extra/clangd/ClangdServer.h index a416602251428b0..dc546b118cb8f5e 100644 --- a/clang-tools-extra/clangd/ClangdServer.h +++ b/clang-tools-extra/clangd/ClangdServer.h @@ -112,6 +112,9 @@ class ClangdServer { /// This throttler controls which preambles may be built at a given time. clangd::PreambleThrottler *PreambleThrottler = nullptr; + /// Enable experimental support for modules. + bool ExperimentalModulesSupport = false; + /// If true, ClangdServer builds a dynamic in-memory index for symbols in /// opened files and uses the index to augment code completion results. bool BuildDynamicSymbolIndex = false; diff --git a/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp b/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp index d1833759917a30f..bcc8f4f0dd9e5ac 100644 --- a/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp +++ b/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp @@ -729,6 +729,20 @@ DirectoryBasedGlobalCompilationDatabase::getProjectInfo(PathRef File) const { return Res-&gt;PI; } +std::vector&lt;std::string&gt; +DirectoryBasedGlobalCompilationDatabase::getAllFilesInProjectOf( + PathRef File) const { + CDBLookupRequest Req; + Req.FileName = File; + Req.ShouldBroadcast = false; + Req.FreshTime = Req.FreshTimeMissing = + std::chrono::steady_clock::time_point::min(); + auto Res = lookupCDB(Req); + if (!Res) + return {}; + return Res-&gt;CDB-&gt;getAllFiles(); +} + OverlayCDB::OverlayCDB(const GlobalCompilationDatabase *Base, std::vector&lt;std::string&gt; FallbackFlags, CommandMangler Mangler) @@ -805,6 +819,13 @@ std::optional&lt;ProjectInfo&gt; DelegatingCDB::getProjectInfo(PathRef File) const { return Base-&gt;getProjectInfo(File); } +std::vector&lt;std::string&gt; +DelegatingCDB::getAllFilesInProjectOf(PathRef File) const { + if (!Base) + return {}; + return Base-&gt;getAllFilesInProjectOf(File); +} + tooling::CompileCommand DelegatingCDB::getFallbackCommand(PathRef File) const { if (!Base) return GlobalCompilationDatabase::getFallbackCommand(File); diff --git a/clang-tools-extra/clangd/GlobalCompilationDatabase.h b/clang-tools-extra/clangd/GlobalCompilationDatabase.h index 2bf8c973c534c6f..eaeff8d627a0960 100644 --- a/clang-tools-extra/clangd/GlobalCompilationDatabase.h +++ b/clang-tools-extra/clangd/GlobalCompilationDatabase.h @@ -45,6 +45,10 @@ class GlobalCompilationDatabase { return std::nullopt; } + virtual std::vector&lt;std::string&gt; getAllFilesInProjectOf(PathRef File) const { + return {}; + } + /// Makes a guess at how to build a file. /// The default implementation just runs clang on the file. /// Clangd should treat the results as unreliable. @@ -75,6 +79,7 @@ class DelegatingCDB : public GlobalCompilationDatabase { getCompileCommand(PathRef File) const override; std::optional&lt;ProjectInfo&gt; getProjectInfo(PathRef File) const override; + std::vector&lt;std::string&gt; getAllFilesInProjectOf(PathRef File) const override; tooling::CompileCommand getFallbackCommand(PathRef File) const override; @@ -121,6 +126,7 @@ class DirectoryBasedGlobalCompilationDatabase /// Returns the path to first directory containing a compilation database in /// \p File&#x27;s parents. std::optional&lt;ProjectInfo&gt; getProjectInfo(PathRef File) const override; + std::vector&lt;std::string&gt; getAllFilesInProjectOf(PathRef File) const override; bool blockUntilIdle(Deadline Timeout) const override; diff --git a/clang-tools-extra/clangd/ModuleDependencyScanner.cpp b/clang-tools-extra/clangd/ModuleDependencyScanner.cpp new file mode 100644 index 000000000000000..d706d2eb2fc8d6e --- /dev/null +++ b/clang-tools-extra/clangd/ModuleDependencyScanner.cpp @@ -0,0 +1,86 @@ +//===---------------- ModuleDependencyScanner.cpp ----------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include &quot;ModuleDependencyScanner.h&quot; + +namespace clang { +namespace clangd { +using P1689Rule = tooling::dependencies::P1689Rule; + +std::optional&lt;P1689Rule&gt; ModuleDependencyScanner::scan(PathRef FilePath) { + if (ScanningCache.count(FilePath)) + return ScanningCache[FilePath]; + + std::optional&lt;tooling::CompileCommand&gt; Cmd = CDB.getCompileCommand(FilePath); + + if (!Cmd) + return std::nullopt; + + using namespace clang::tooling::dependencies; + + llvm::SmallString&lt;128&gt; FilePathDir(FilePath); + llvm::sys::path::remove_filename(FilePathDir); + DependencyScanningTool ScanningTool( + Service, + TFS ? TFS-&gt;view(FilePathDir) : llvm::vfs::createPhysicalFileSystem()); + + llvm::Expected&lt;P1689Rule&gt; Result = + ScanningTool.getP1689ModuleDependencyFile(*Cmd, Cmd-&gt;Directory); + + if (auto E = Result.takeError()) { + // Ignore any error. + llvm::consumeError(std::move(E)); + return std::nullopt; + } + + if (Result-&gt;Provides) + ModuleNameToSourceMapper[Result-&gt;Provides-&gt;ModuleName] = FilePath; + + ScanningCache[FilePath] = *Result; + return *Result; +} + +void ModuleDependencyScanner::globalScan(PathRef File) { + std::vector&lt;std::string&gt; AllFiles = CDB.getAllFilesInProjectOf(File); + + for (auto &amp;File : AllFiles) + scan(File); +} + +PathRef ModuleDependencyScanner::getSourceForModuleName(StringRef ModuleName) const { + if (!ModuleNameToSourceMapper.count(ModuleName)) + return {}; + + return ModuleNameToSourceMapper[ModuleName]; +} + +std::vector&lt;std::string&gt; +ModuleDependencyScanner::getRequiredModules(PathRef File) const { + if (!ScanningCache.count(File)) + return {}; + + const P1689Rule &amp;CachedResult = ScanningCache[File]; + std::vector&lt;std::string&gt; Result; + + for (const auto &amp;Info : CachedResult.Requires) + Result.push_back(Info.ModuleName); + + return Result; +} + +StringRef ModuleDependencyScanner::getModuleName(PathRef File) const { + if (!ScanningCache.count(File)) + return {}; + + if (!ScanningCache[File].Provides) + return {}; + + return ScanningCache[File].Provides-&gt;ModuleName; +} +} // namespace clangd +} // namespace clang diff --git a/clang-tools-extra/clangd/ModuleDependencyScanner.h b/clang-tools-extra/clangd/ModuleDependencyScanner.h new file mode 100644 index 000000000000000..1d6eb58fda59e20 --- /dev/null +++ b/clang-tools-extra/clangd/ModuleDependencyScanner.h @@ -0,0 +1,78 @@ +//===-------------- ModuleDependencyScanner.h --------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_MODULEDEPENDENCYSCANNER_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_MODULEDEPENDENCYSCANNER_H + +#include &quot;GlobalCompilationDatabase.h&quot; +#include &quot;support/Path.h&quot; +#include &quot;support/ThreadsafeFS.h&quot; + +#include &quot;clang/Tooling/DependencyScanning/DependencyScanningService.h&quot; +#include &quot;clang/Tooling/DependencyScanning/DependencyScanningTool.h&quot; + +#include &quot;llvm/ADT/SmallString.h&quot; +#include &quot;llvm/ADT/StringMap.h&quot; + +namespace clang { +namespace clangd { + +/// A scanner to produce P1689 format for C++20 Modules. +/// +/// The scanner can scan a single file with `scan(PathRef)` member function +/// or scan the whole project with `globalScan(PathRef)` member function. See +/// the comments of `globalScan` to see the details. +class ModuleDependencyScanner { +public: + ModuleDependencyScanner(const GlobalCompilationDatabase &amp;CDB, + const ThreadsafeFS *TFS) + : CDB(CDB), TFS(TFS), + Service(tooling::dependencies::ScanningMode::CanonicalPreprocessing, + tooling::dependencies::ScanningOutputFormat::P1689) {} + + /// Scanning the single file specified by \param FilePath. + std::optional&lt;clang::tooling::dependencies::P1689Rule&gt; scan(PathRef FilePath); + + /// Scanning every source file in the current project to get the + /// &lt;module-name&gt; to &lt;module-unit-source&gt; map. + /// It looks unefficiency to scan the whole project especially for + /// every version of every file! + /// TODO: We should find a efficient method to get the &lt;module-name&gt; + /// to &lt;module-unit-source&gt; map. We can make it either by providing + /// a global module dependency scanner to monitor every file. Or we + /// can simply require the build systems (or even if the end users) + /// to provide the map. + void globalScan(PathRef File); + + PathRef getSourceForModuleName(StringRef ModuleName) const; + + /// Return the direct required modules. Indirect required modules are not + /// included. + std::vector&lt;std::string&gt; getRequiredModules(PathRef File) const; + StringRef getModuleName(PathRef File) const; + + const ThreadsafeFS *getThreadsafeFS() const { return TFS; } + + const GlobalCompilationDatabase &amp;getCompilationDatabase() const { return CDB; } + +private: + const GlobalCompilationDatabase &amp;CDB; + const ThreadsafeFS *TFS; + + clang::tooling::dependencies::DependencyScanningService Service; + + // Map source file to P1689 Result. + llvm::StringMap&lt;clang::tooling::dependencies::P1689Rule&gt; ScanningCache; + // Map module name to source file path. + llvm::StringMap&lt;std::string&gt; ModuleNameToSourceMapper; +}; + +} // namespace clangd +} // namespace clang + +#endif diff --git a/clang-tools-extra/clangd/ModuleFilesInfo.cpp b/clang-tools-extra/clangd/ModuleFilesInfo.cpp new file mode 100644 index 000000000000000..845ff01ca09dff3 --- /dev/null +++ b/clang-tools-extra/clangd/ModuleFilesInfo.cpp @@ -0,0 +1,282 @@ +//===----------------- ModuleFilesInfo.cpp -----------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include &quot;ModuleFilesInfo.h&quot; +#include &quot;support/Logger.h&quot; + +#include &quot;clang/Frontend/FrontendAction.h&quot; +#include &quot;clang/Frontend/FrontendActions.h&quot; +#include &quot;clang/Serialization/ASTReader.h&quot; + +namespace clang { +namespace clangd { + +namespace { +llvm::SmallString&lt;128&gt; getAbsolutePath(const tooling::CompileCommand &amp;Cmd) { + llvm::SmallString&lt;128&gt; AbsolutePath; + if (llvm::sys::path::is_absolute(Cmd.Filename)) { + AbsolutePath = Cmd.Filename; + } else { + AbsolutePath = Cmd.Directory; + llvm::sys::path::append(AbsolutePath, Cmd.Filename); + llvm::sys::path::remove_dots(AbsolutePath, true); + } + return AbsolutePath; +} +} // namespace + +ModuleFilesInfo::ModuleFilesInfo(PathRef MainFile, + const GlobalCompilationDatabase &amp;CDB) { + std::optional&lt;ProjectInfo&gt; PI = CDB.getProjectInfo(MainFile); + if (!PI) + return; + + llvm::SmallString&lt;128&gt; Result(PI-&gt;SourceRoot); + llvm::sys::path::append(Result, &quot;.cache&quot;); + llvm::sys::path::append(Result, &quot;clangd&quot;); + llvm::sys::path::append(Result, &quot;module_files&quot;); + llvm::sys::fs::create_directories(Result, /*IgnoreExisting=*/true); + + llvm::sys::path::append(Result, llvm::sys::path::filename(MainFile)); + llvm::sys::fs::createUniqueDirectory(Result, UniqueModuleFilesPathPrefix); + + log(&quot;Initialized module files to {0}&quot;, UniqueModuleFilesPathPrefix.str()); +} + +ModuleFilesInfo::~ModuleFilesInfo() { + DependentModuleNames.clear(); + Successed = false; + + if (UniqueModuleFilesPathPrefix.empty()) + return; + + llvm::sys::fs::remove_directories(UniqueModuleFilesPathPrefix); + UniqueModuleFilesPathPrefix.clear(); +} + +llvm::SmallString&lt;256&gt; +ModuleFilesInfo::getModuleFilePath(StringRef ModuleName) const { + llvm::SmallString&lt;256&gt; ModuleFilePath; + + ModuleFilePath = UniqueModuleFilesPathPrefix; + auto [PrimaryModuleName, PartitionName] = ModuleName.split(&#x27;:&#x27;); + llvm::sys::path::append(ModuleFilePath, PrimaryModuleName); + if (!PartitionName.empty()) { + ModuleFilePath.append(&quot;-&quot;); + ModuleFilePath.append(PartitionName); + } + ModuleFilePath.append(&quot;.pcm&quot;); + + return ModuleFilePath; +} + +bool ModuleFilesInfo::IsModuleUnitBuilt(StringRef ModuleName) const { + if (!DependentModuleNames.count(ModuleName)) + return false; + + auto BMIPath = getModuleFilePath(ModuleName); + if (llvm::sys::fs::exists(BMIPath)) + return true; + + Successed = false; + + DependentModuleNames.erase(ModuleName); + return false; +} + +void ModuleFilesInfo::ReplaceHeaderSearchOptions( + HeaderSearchOptions &amp;Options) const { + if (!IsInited()) + return; + + Options.PrebuiltModulePaths.insert(Options.PrebuiltModulePaths.begin(), + UniqueModuleFilesPathPrefix.str().str()); + + for (auto Iter = Options.PrebuiltModuleFiles.begin(); + Iter != Options.PrebuiltModuleFiles.end();) { + if (IsModuleUnitBuilt(Iter-&gt;first)) { + Iter = Options.PrebuiltModuleFiles.erase(Iter); + continue; + } + + Iter++; + } +} + +void ModuleFilesInfo::ReplaceCompileCommands( + tooling::CompileCommand &amp;Cmd) const { + if (!IsInited()) + return; + + std::vector&lt;std::string&gt; CommandLine(std::move(Cmd.CommandLine)); + + Cmd.CommandLine.emplace_back(CommandLine[0]); + Cmd.CommandLine.emplace_back( + llvm::Twine(&quot;-fprebuilt-module-path=&quot; + UniqueModuleFilesPathPrefix) + .str()); + + for (std::size_t I = 1; I &lt; CommandLine.size(); I++) { + const std::string &amp;Arg = CommandLine[I]; + const auto &amp;[LHS, RHS] = StringRef(Arg).split(&quot;=&quot;); + + // Remove original `-fmodule-file=&lt;module-name&gt;=&lt;module-path&gt;` form if it + // already built. + if (LHS == &quot;-fmodule-file&quot; &amp;&amp; RHS.contains(&quot;=&quot;)) { + const auto &amp;[ModuleName, _] = RHS.split(&quot;=&quot;); + if (IsModuleUnitBuilt(ModuleName)) + continue; + } + + Cmd.CommandLine.emplace_back(Arg); + } +} + +void ModuleFilesInfo::ReplaceCompileCommands(tooling::CompileCommand &amp;Cmd, + StringRef OutputModuleName) const { + if (!IsInited()) + return; + + ReplaceCompileCommands(Cmd); + + Cmd.Output = getModuleFilePath(OutputModuleName).str().str(); +} + +bool ModuleFilesInfo::buildModuleFile(PathRef ModuleUnitFileName, + ModuleDependencyScanner &amp;Scanner) { + if (ModuleUnitFileName.empty()) + return false; + + for (auto &amp;ModuleName : Scanner.getRequiredModules(ModuleUnitFileName)) { + // Return early if there are errors building the module file. + if (!IsModuleUnitBuilt(ModuleName) &amp;&amp; + !buildModuleFile(Scanner.getSourceForModuleName(ModuleName), Scanner)) { + log(&quot;Failed to build module {0}&quot;, ModuleName); + return false; + } + } + + auto Cmd = + Scanner.getCompilationDatabase().getCompileCommand(ModuleUnitFileName); + if (!Cmd) + return false; + + ReplaceCompileCommands(*Cmd, Scanner.getModuleName(ModuleUnitFileName)); + + ParseInputs Inputs; + Inputs.TFS = Scanner.getThreadsafeFS(); + Inputs.CompileCommand = std::move(*Cmd); + + IgnoreDiagnostics IgnoreDiags; + auto CI = buildCompilerInvocation(Inputs, IgnoreDiags); + if (!CI) + return false; + + auto FS = Inputs.TFS-&gt;view(Inputs.CompileCommand.Directory); + auto AbsolutePath = getAbsolutePath(Inputs.CompileCommand); + auto Buf = FS-&gt;getBufferForFile(AbsolutePath); + if (!Buf) + return false; + + // Hash the contents of input files and store the hash value to the BMI files. + // So that we can check if the files are still valid when we want to reuse the + // BMI files. + CI-&gt;getHeaderSearchOpts().ValidateASTInputFilesContent = true; + + CI-&gt;getFrontendOpts().OutputFile = Inputs.CompileCommand.Output; + auto Clang = + prepareCompilerInstance(std::move(CI), /*Preamble=*/nullptr, + std::move(*Buf), std::move(FS), IgnoreDiags); + if (!Clang) + retu... <truncated> </pre> </details> https://github.com/llvm/llvm-project/pull/66462 _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits