yaxunl updated this revision to Diff 232405.
yaxunl added a comment.
split the patch
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D69582/new/
https://reviews.llvm.org/D69582
Files:
clang/include/clang/Driver/Driver.h
clang/include/clang/Driver/Job.h
clang/include/clang/Driver/Options.td
clang/lib/Driver/Compilation.cpp
clang/lib/Driver/Driver.cpp
clang/lib/Driver/Job.cpp
clang/lib/Driver/ToolChains/Clang.cpp
Index: clang/lib/Driver/ToolChains/Clang.cpp
===================================================================
--- clang/lib/Driver/ToolChains/Clang.cpp
+++ clang/lib/Driver/ToolChains/Clang.cpp
@@ -6483,7 +6483,7 @@
C.addCommand(std::make_unique<Command>(
JA, *this,
TCArgs.MakeArgString(getToolChain().GetProgramPath(getShortName())),
- CmdArgs, None));
+ CmdArgs, Inputs));
}
void OffloadBundler::ConstructJobMultipleOutputs(
@@ -6549,7 +6549,7 @@
C.addCommand(std::make_unique<Command>(
JA, *this,
TCArgs.MakeArgString(getToolChain().GetProgramPath(getShortName())),
- CmdArgs, None));
+ CmdArgs, Inputs));
}
void OffloadWrapper::ConstructJob(Compilation &C, const JobAction &JA,
Index: clang/lib/Driver/Job.cpp
===================================================================
--- clang/lib/Driver/Job.cpp
+++ clang/lib/Driver/Job.cpp
@@ -39,9 +39,11 @@
ArrayRef<InputInfo> Inputs)
: Source(Source), Creator(Creator), Executable(Executable),
Arguments(Arguments) {
- for (const auto &II : Inputs)
+ for (const auto &II : Inputs) {
if (II.isFilename())
InputFilenames.push_back(II.getFilename());
+ DependentActions.push_back(II.getAction());
+ }
}
/// Check if the compiler flag in question should be skipped when
Index: clang/lib/Driver/Driver.cpp
===================================================================
--- clang/lib/Driver/Driver.cpp
+++ clang/lib/Driver/Driver.cpp
@@ -38,13 +38,14 @@
#include "ToolChains/NaCl.h"
#include "ToolChains/NetBSD.h"
#include "ToolChains/OpenBSD.h"
-#include "ToolChains/PS4CPU.h"
#include "ToolChains/PPCLinux.h"
+#include "ToolChains/PS4CPU.h"
#include "ToolChains/RISCVToolchain.h"
#include "ToolChains/Solaris.h"
#include "ToolChains/TCE.h"
#include "ToolChains/WebAssembly.h"
#include "ToolChains/XCore.h"
+#include "clang/Basic/OptionUtils.h"
#include "clang/Basic/Version.h"
#include "clang/Config/config.h"
#include "clang/Driver/Action.h"
@@ -55,6 +56,7 @@
#include "clang/Driver/SanitizerArgs.h"
#include "clang/Driver/Tool.h"
#include "clang/Driver/ToolChain.h"
+#include "clang/Driver/Util.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallSet.h"
@@ -130,7 +132,7 @@
CCLogDiagnostics(false), CCGenDiagnostics(false),
TargetTriple(TargetTriple), CCCGenericGCCName(""), Saver(Alloc),
CheckInputsExist(true), GenReproducer(false),
- SuppressMissingInputWarning(false) {
+ SuppressMissingInputWarning(false), NumParallelJobs(1) {
// Provide a sane fallback if no VFS is specified.
if (!this->VFS)
@@ -1103,6 +1105,9 @@
BitcodeEmbed = static_cast<BitcodeEmbedMode>(Model);
}
+ setNumberOfParallelJobs(
+ getLastArgIntValue(Args, options::OPT_parallel_jobs_EQ, 1, Diags));
+
std::unique_ptr<llvm::opt::InputArgList> UArgs =
std::make_unique<InputArgList>(std::move(Args));
Index: clang/lib/Driver/Compilation.cpp
===================================================================
--- clang/lib/Driver/Compilation.cpp
+++ clang/lib/Driver/Compilation.cpp
@@ -15,6 +15,7 @@
#include "clang/Driver/Options.h"
#include "clang/Driver/ToolChain.h"
#include "clang/Driver/Util.h"
+#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
@@ -25,8 +26,11 @@
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/raw_ostream.h"
#include <cassert>
+#include <functional>
+#include <mutex>
#include <string>
#include <system_error>
+#include <thread>
#include <utility>
using namespace clang;
@@ -220,22 +224,134 @@
return !ActionFailed(&C.getSource(), FailingCommands);
}
+namespace {
+class JobScheduler {
+public:
+ enum JobState { JS_WAIT, JS_RUN, JS_DONE, JS_FAIL };
+ JobScheduler(const JobList &Jobs, size_t NJobs = 1)
+ : Jobs(Jobs), NumJobs(NJobs) {
+#if !LLVM_ENABLE_THREADS
+ NumJobs = 1;
+#endif
+ for (auto &Job : Jobs) {
+ JState[&Job] = JS_WAIT;
+ for (const auto *AI : Job.getDependentActions()) {
+ for (const auto *CI : ActToCmds[AI]) {
+ DependentCmds[&Job].push_back(CI);
+ }
+ }
+ for (const auto *CI : ActToCmds[&Job.getSource()]) {
+ DependentCmds[&Job].push_back(CI);
+ }
+ ActToCmds[&Job.getSource()].push_back(&Job);
+ }
+ }
+ /// \return true if all jobs are done. Otherwise, \p Next contains the
+ /// the next job ready to be executed if it is not null pointer. Otherwise
+ /// all jobs are running or waiting.
+ bool IsDone(const Command *&Next) {
+ std::lock_guard<std::mutex> lock(Mutex);
+ Next = nullptr;
+ unsigned Done = 0;
+ unsigned Running = 0;
+ for (auto &Cmd : Jobs) {
+ switch (JState[&Cmd]) {
+ case JS_RUN:
+ ++Running;
+ break;
+ case JS_DONE:
+ case JS_FAIL:
+ ++Done;
+ break;
+ case JS_WAIT: {
+ bool InputsReady = true;
+ for (const auto *CI : DependentCmds[&Cmd]) {
+ if (JState[CI] == JS_FAIL) {
+ JState[&Cmd] = JS_FAIL;
+ ++Done;
+ InputsReady = false;
+ break;
+ }
+ if (JState[CI] != JS_DONE) {
+ InputsReady = false;
+ break;
+ }
+ }
+ if (!Next && InputsReady) {
+ Next = &Cmd;
+ }
+ break;
+ }
+ }
+ }
+ if (Running >= NumJobs)
+ Next = nullptr;
+ return Done == Jobs.size();
+ }
+
+ void setJobState(const Command *Cmd, JobState JS) {
+ std::lock_guard<std::mutex> lock(Mutex);
+ JState[Cmd] = JS;
+ }
+
+ void launch(std::function<void()> Work) {
+#if LLVM_ENABLE_THREADS
+ if (NumJobs == 1) {
+ Work();
+ return;
+ }
+ std::thread Th(Work);
+ Th.detach();
+#else
+ Work();
+#endif
+ }
+
+private:
+ std::mutex Mutex;
+ const JobList &Jobs;
+ llvm::DenseMap<const Command *, JobState> JState;
+ llvm::DenseMap<const Action *, llvm::SmallVector<const Command *, 4>>
+ ActToCmds;
+ llvm::DenseMap<const Command *, llvm::SmallVector<const Command *, 4>>
+ DependentCmds;
+ size_t NumJobs; // Number of parallel jobs to run
+};
+} // namespace
void Compilation::ExecuteJobs(const JobList &Jobs,
FailingCommandList &FailingCommands) const {
// According to UNIX standard, driver need to continue compiling all the
// inputs on the command line even one of them failed.
// In all but CLMode, execute all the jobs unless the necessary inputs for the
// job is missing due to previous failures.
- for (const auto &Job : Jobs) {
- if (!InputsOk(Job, FailingCommands))
+ JobScheduler JS(Jobs, getDriver().getNumberOfParallelJobs());
+
+ const Command *Next = nullptr;
+ while (!JS.IsDone(Next)) {
+ if (!Next) {
+ std::this_thread::yield();
continue;
- const Command *FailingCommand = nullptr;
- if (int Res = ExecuteCommand(Job, FailingCommand)) {
- FailingCommands.push_back(std::make_pair(Res, FailingCommand));
+ }
+
+ if (!InputsOk(*Next, FailingCommands)) {
+ JS.setJobState(Next, JobScheduler::JS_FAIL);
// Bail as soon as one command fails in cl driver mode.
if (TheDriver.IsCLMode())
return;
+ continue;
}
+
+ JS.setJobState(Next, JobScheduler::JS_RUN);
+ auto Work = [&, Next]() {
+ const Command *FailingCommand = nullptr;
+ if (int Res = ExecuteCommand(*Next, FailingCommand)) {
+ JS.setJobState(Next, JobScheduler::JS_FAIL);
+ FailingCommands.push_back(std::make_pair(Res, FailingCommand));
+ } else {
+ JS.setJobState(Next, JobScheduler::JS_DONE);
+ }
+ };
+ JS.launch(Work);
}
}
Index: clang/include/clang/Driver/Options.td
===================================================================
--- clang/include/clang/Driver/Options.td
+++ clang/include/clang/Driver/Options.td
@@ -401,6 +401,8 @@
def Ofast : Joined<["-"], "Ofast">, Group<O_Group>, Flags<[CC1Option]>;
def P : Flag<["-"], "P">, Flags<[CC1Option]>, Group<Preprocessor_Group>,
HelpText<"Disable linemarker output in -E mode">;
+def parallel_jobs_EQ : Joined<["-"], "parallel-jobs=">, Flags<[DriverOption]>,
+ HelpText<"Number of parallel jobs">;
def Qy : Flag<["-"], "Qy">, Flags<[CC1Option]>,
HelpText<"Emit metadata containing compiler name and version">;
def Qn : Flag<["-"], "Qn">, Flags<[CC1Option]>,
Index: clang/include/clang/Driver/Job.h
===================================================================
--- clang/include/clang/Driver/Job.h
+++ clang/include/clang/Driver/Job.h
@@ -73,6 +73,9 @@
/// See Command::setEnvironment
std::vector<const char *> Environment;
+ /// Dependent actions
+ llvm::SmallVector<const Action *, 4> DependentActions;
+
/// When a response file is needed, we try to put most arguments in an
/// exclusive file, while others remains as regular command line arguments.
/// This functions fills a vector with the regular command line arguments,
@@ -130,6 +133,10 @@
/// Set whether to print the input filenames when executing.
void setPrintInputFilenames(bool P) { PrintInputFilenames = P; }
+
+ const llvm::SmallVector<const Action *, 4> &getDependentActions() const {
+ return DependentActions;
+ }
};
/// Like Command, but with a fallback which is executed in case
Index: clang/include/clang/Driver/Driver.h
===================================================================
--- clang/include/clang/Driver/Driver.h
+++ clang/include/clang/Driver/Driver.h
@@ -247,6 +247,9 @@
/// stored in it, and will clean them up when torn down.
mutable llvm::StringMap<std::unique_ptr<ToolChain>> ToolChains;
+ /// Number of parallel jobs.
+ unsigned NumParallelJobs;
+
private:
/// TranslateInputArgs - Create a new derived argument list from the input
/// arguments, after applying the standard argument translations.
@@ -549,6 +552,12 @@
/// Get the specific kind of LTO being performed.
LTOKind getLTOMode() const { return LTOMode; }
+ /// Get the number of parallel jobs.
+ unsigned getNumberOfParallelJobs() const { return NumParallelJobs; }
+
+ /// Set the number of parallel jobs.
+ void setNumberOfParallelJobs(unsigned N) { NumParallelJobs = N; }
+
private:
/// Tries to load options from configuration file.
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
