https://github.com/kyulee-com updated
https://github.com/llvm/llvm-project/pull/90934
>From 0378aa86d0ad24069c191250d76bc0f3800eb7ae Mon Sep 17 00:00:00 2001
From: Kyungwoo Lee <kyu...@meta.com>
Date: Fri, 26 Apr 2024 20:02:52 -0700
Subject: [PATCH 1/3] [ThinLTO][NFC] Prep for two-codegen rounds
---
clang/lib/CodeGen/BackendUtil.cpp | 8 ++--
llvm/include/llvm/LTO/LTOBackend.h | 1 +
llvm/lib/LTO/LTO.cpp | 75 ++++++++++++++++--------------
llvm/lib/LTO/LTOBackend.cpp | 6 ++-
4 files changed, 49 insertions(+), 41 deletions(-)
diff --git a/clang/lib/CodeGen/BackendUtil.cpp
b/clang/lib/CodeGen/BackendUtil.cpp
index 62c6a57e8b7c80..abc936f2c686dd 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -1321,10 +1321,10 @@ static void runThinLTOBackend(
Conf.CGFileType = getCodeGenFileType(Action);
break;
}
- if (Error E =
- thinBackend(Conf, -1, AddStream, *M, *CombinedIndex, ImportList,
- ModuleToDefinedGVSummaries[M->getModuleIdentifier()],
- /* ModuleMap */ nullptr, CGOpts.CmdArgs)) {
+ if (Error E = thinBackend(
+ Conf, -1, AddStream, *M, *CombinedIndex, ImportList,
+ ModuleToDefinedGVSummaries[M->getModuleIdentifier()],
+ /* ModuleMap */ nullptr, Conf.CodeGenOnly, CGOpts.CmdArgs)) {
handleAllErrors(std::move(E), [&](ErrorInfoBase &EIB) {
errs() << "Error running ThinLTO backend: " << EIB.message() << '\n';
});
diff --git a/llvm/include/llvm/LTO/LTOBackend.h
b/llvm/include/llvm/LTO/LTOBackend.h
index de89f4bb10dff2..8516398510d4b8 100644
--- a/llvm/include/llvm/LTO/LTOBackend.h
+++ b/llvm/include/llvm/LTO/LTOBackend.h
@@ -56,6 +56,7 @@ Error thinBackend(const Config &C, unsigned Task, AddStreamFn
AddStream,
const FunctionImporter::ImportMapTy &ImportList,
const GVSummaryMapTy &DefinedGlobals,
MapVector<StringRef, BitcodeModule> *ModuleMap,
+ bool CodeGenOnly,
const std::vector<uint8_t> &CmdArgs =
std::vector<uint8_t>());
Error finalizeOptimizationRemarks(
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index a88124dacfaefd..f4c25f80811a85 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -1473,7 +1473,8 @@ class InProcessThinBackend : public ThinBackendProc {
return MOrErr.takeError();
return thinBackend(Conf, Task, AddStream, **MOrErr, CombinedIndex,
- ImportList, DefinedGlobals, &ModuleMap);
+ ImportList, DefinedGlobals, &ModuleMap,
+ Conf.CodeGenOnly);
};
auto ModuleID = BM.getModuleIdentifier();
@@ -1839,45 +1840,49 @@ Error LTO::runThinLTO(AddStreamFn AddStream, FileCache
Cache,
TimeTraceScopeExit.release();
- std::unique_ptr<ThinBackendProc> BackendProc =
- ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
- AddStream, Cache);
-
auto &ModuleMap =
ThinLTO.ModulesToCompile ? *ThinLTO.ModulesToCompile : ThinLTO.ModuleMap;
- auto ProcessOneModule = [&](int I) -> Error {
- auto &Mod = *(ModuleMap.begin() + I);
- // Tasks 0 through ParallelCodeGenParallelismLevel-1 are reserved for
- // combined module and parallel code generation partitions.
- return BackendProc->start(RegularLTO.ParallelCodeGenParallelismLevel + I,
- Mod.second, ImportLists[Mod.first],
- ExportLists[Mod.first], ResolvedODR[Mod.first],
- ThinLTO.ModuleMap);
+ auto RunBackends = [&](ThinBackendProc *BackendProcess) -> Error {
+ auto ProcessOneModule = [&](int I) -> Error {
+ auto &Mod = *(ModuleMap.begin() + I);
+ // Tasks 0 through ParallelCodeGenParallelismLevel-1 are reserved for
+ // combined module and parallel code generation partitions.
+ return BackendProcess->start(
+ RegularLTO.ParallelCodeGenParallelismLevel + I, Mod.second,
+ ImportLists[Mod.first], ExportLists[Mod.first],
+ ResolvedODR[Mod.first], ThinLTO.ModuleMap);
+ };
+
+ if (BackendProcess->getThreadCount() == 1) {
+ // Process the modules in the order they were provided on the
+ // command-line. It is important for this codepath to be used for
+ // WriteIndexesThinBackend, to ensure the emitted LinkedObjectsFile lists
+ // ThinLTO objects in the same order as the inputs, which otherwise would
+ // affect the final link order.
+ for (int I = 0, E = ModuleMap.size(); I != E; ++I)
+ if (Error E = ProcessOneModule(I))
+ return E;
+ } else {
+ // When executing in parallel, process largest bitsize modules first to
+ // improve parallelism, and avoid starving the thread pool near the end.
+ // This saves about 15 sec on a 36-core machine while link `clang.exe`
+ // (out of 100 sec).
+ std::vector<BitcodeModule *> ModulesVec;
+ ModulesVec.reserve(ModuleMap.size());
+ for (auto &Mod : ModuleMap)
+ ModulesVec.push_back(&Mod.second);
+ for (int I : generateModulesOrdering(ModulesVec))
+ if (Error E = ProcessOneModule(I))
+ return E;
+ }
+ return BackendProcess->wait();
};
- if (BackendProc->getThreadCount() == 1) {
- // Process the modules in the order they were provided on the command-line.
- // It is important for this codepath to be used for
WriteIndexesThinBackend,
- // to ensure the emitted LinkedObjectsFile lists ThinLTO objects in the
same
- // order as the inputs, which otherwise would affect the final link order.
- for (int I = 0, E = ModuleMap.size(); I != E; ++I)
- if (Error E = ProcessOneModule(I))
- return E;
- } else {
- // When executing in parallel, process largest bitsize modules first to
- // improve parallelism, and avoid starving the thread pool near the end.
- // This saves about 15 sec on a 36-core machine while link `clang.exe` (out
- // of 100 sec).
- std::vector<BitcodeModule *> ModulesVec;
- ModulesVec.reserve(ModuleMap.size());
- for (auto &Mod : ModuleMap)
- ModulesVec.push_back(&Mod.second);
- for (int I : generateModulesOrdering(ModulesVec))
- if (Error E = ProcessOneModule(I))
- return E;
- }
- return BackendProc->wait();
+ std::unique_ptr<ThinBackendProc> BackendProc =
+ ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
+ AddStream, Cache);
+ return RunBackends(BackendProc.get());
}
Expected<std::unique_ptr<ToolOutputFile>> lto::setupLLVMOptimizationRemarks(
diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp
index 4e58cd369c3ac9..880567989baffb 100644
--- a/llvm/lib/LTO/LTOBackend.cpp
+++ b/llvm/lib/LTO/LTOBackend.cpp
@@ -565,7 +565,7 @@ Error lto::thinBackend(const Config &Conf, unsigned Task,
AddStreamFn AddStream,
const FunctionImporter::ImportMapTy &ImportList,
const GVSummaryMapTy &DefinedGlobals,
MapVector<StringRef, BitcodeModule> *ModuleMap,
- const std::vector<uint8_t> &CmdArgs) {
+ bool CodeGenOnly, const std::vector<uint8_t> &CmdArgs) {
Expected<const Target *> TOrErr = initAndLookupTarget(Conf, Mod);
if (!TOrErr)
return TOrErr.takeError();
@@ -586,7 +586,9 @@ Error lto::thinBackend(const Config &Conf, unsigned Task,
AddStreamFn AddStream,
Mod.setPartialSampleProfileRatio(CombinedIndex);
LLVM_DEBUG(dbgs() << "Running ThinLTO\n");
- if (Conf.CodeGenOnly) {
+ if (CodeGenOnly) {
+ // If CodeGenOnly is set, we only perform code generation and skip
+ // optimization.
codegen(Conf, TM.get(), AddStream, Task, Mod, CombinedIndex);
return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
}
>From e79f5a640562aceed477f817baed1726ea18e2c6 Mon Sep 17 00:00:00 2001
From: Kyungwoo Lee <kyu...@fb.com>
Date: Mon, 16 Sep 2024 13:37:12 -0700
Subject: [PATCH 2/3] Address comments from minglotus-6
---
llvm/include/llvm/LTO/LTOBackend.h | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/include/llvm/LTO/LTOBackend.h
b/llvm/include/llvm/LTO/LTOBackend.h
index 8516398510d4b8..098c0491dfe70a 100644
--- a/llvm/include/llvm/LTO/LTOBackend.h
+++ b/llvm/include/llvm/LTO/LTOBackend.h
@@ -50,7 +50,8 @@ Error backend(const Config &C, AddStreamFn AddStream,
/// already been mapped to memory and the corresponding BitcodeModule objects
/// are saved in the ModuleMap. If \p ModuleMap is nullptr, module files will
/// be mapped to memory on demand and at any given time during importing, only
-/// one source module will be kept open at the most.
+/// one source module will be kept open at the most. If \p CodeGenOnly is true,
+/// the backend will skip optimization and only perform code generation.
Error thinBackend(const Config &C, unsigned Task, AddStreamFn AddStream,
Module &M, const ModuleSummaryIndex &CombinedIndex,
const FunctionImporter::ImportMapTy &ImportList,
>From 12050e44ea192d2bef516fbddcb753f077f4455c Mon Sep 17 00:00:00 2001
From: Kyungwoo Lee <kyu...@meta.com>
Date: Thu, 3 Oct 2024 08:32:53 -0700
Subject: [PATCH 3/3] Address comments from teresajohnson
---
llvm/lib/LTO/LTOBackend.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp
index 880567989baffb..06eeed3e1bc41f 100644
--- a/llvm/lib/LTO/LTOBackend.cpp
+++ b/llvm/lib/LTO/LTOBackend.cpp
@@ -588,7 +588,7 @@ Error lto::thinBackend(const Config &Conf, unsigned Task,
AddStreamFn AddStream,
LLVM_DEBUG(dbgs() << "Running ThinLTO\n");
if (CodeGenOnly) {
// If CodeGenOnly is set, we only perform code generation and skip
- // optimization.
+ // optimization. This value may differ from Conf.CodeGenOnly.
codegen(Conf, TM.get(), AddStream, Task, Mod, CombinedIndex);
return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
}
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
