saudi created this revision.
saudi added reviewers: arphaman, dexonsmith, Bigcheese.
saudi added projects: clang, LLVM.
saudi requested review of this revision.
Herald added a subscriber: cfe-commits.
Switching back to `std::thread` to simplify the logic, as `ThreadPool` adds a
layer of task management. However, keeping the `ThreadPoolStrategy` use, to
benefit the `ThreadPool` improvements from https://reviews.llvm.org/D71775
This patch partly reverts https://reviews.llvm.org/D74569
Added a few optimizations:
- Prevent from spawning more threads than there are files to process;
- Perform computations on the main thread too, to save one thread spawn. This
way, `clang-scan-deps -j1` will be really single-threaded, and the code path is
the same as when LLVM threads are disabled.
- Call `reserve()` to `std::vector`s which size is known when initializing.
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D102633
Files:
clang/test/ClangScanDeps/Inputs/num-workers-1cmd.json
clang/test/ClangScanDeps/Inputs/num-workers-2cmds.json
clang/test/ClangScanDeps/num-workers.cpp
clang/tools/clang-scan-deps/ClangScanDeps.cpp
Index: clang/tools/clang-scan-deps/ClangScanDeps.cpp
===================================================================
--- clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -20,7 +20,6 @@
#include "llvm/Support/JSON.h"
#include "llvm/Support/Program.h"
#include "llvm/Support/Signals.h"
-#include "llvm/Support/ThreadPool.h"
#include "llvm/Support/Threading.h"
#include <mutex>
#include <thread>
@@ -555,16 +554,29 @@
DependencyScanningService Service(ScanMode, Format, ReuseFileManager,
SkipExcludedPPRanges);
- llvm::ThreadPool Pool(llvm::hardware_concurrency(NumThreads));
- std::vector<std::unique_ptr<DependencyScanningTool>> WorkerTools;
- for (unsigned I = 0; I < Pool.getThreadCount(); ++I)
- WorkerTools.push_back(std::make_unique<DependencyScanningTool>(Service));
std::vector<SingleCommandCompilationDatabase> Inputs;
- for (tooling::CompileCommand Cmd :
- AdjustingCompilations->getAllCompileCommands())
- Inputs.emplace_back(Cmd);
+ {
+ auto AdjustedCommands = AdjustingCompilations->getAllCompileCommands();
+ Inputs.reserve(AdjustedCommands.size());
+ for (tooling::CompileCommand Cmd : AdjustedCommands)
+ Inputs.emplace_back(Cmd);
+ }
+ unsigned NumWorkers =
+ llvm::hardware_concurrency(NumThreads).compute_thread_count();
+
+ if (NumWorkers > Inputs.size())
+ NumWorkers = static_cast<unsigned>(Inputs.size());
+
+ std::vector<std::unique_ptr<DependencyScanningTool>> WorkerTools;
+ WorkerTools.reserve(NumWorkers);
+ for (unsigned I = 0; I < NumWorkers; ++I)
+ WorkerTools.push_back(std::make_unique<DependencyScanningTool>(Service));
+
+ std::vector<std::thread> WorkerThreads;
+ if (NumWorkers > 1)
+ WorkerThreads.reserve(NumWorkers - 1);
std::atomic<bool> HadErrors(false);
FullDeps FD;
std::mutex Lock;
@@ -572,11 +584,11 @@
if (Verbose) {
llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
- << " files using " << Pool.getThreadCount() << " workers\n";
+ << " files using " << NumWorkers << " workers\n";
}
- for (unsigned I = 0; I < Pool.getThreadCount(); ++I) {
- Pool.async([I, &Lock, &Index, &Inputs, &HadErrors, &FD, &WorkerTools,
- &DependencyOS, &Errs]() {
+ for (unsigned I = 0; I < NumWorkers; ++I) {
+ auto Worker = [I, &Lock, &Index, &Inputs, &HadErrors, &FD, &WorkerTools,
+ &DependencyOS, &Errs]() {
llvm::StringSet<> AlreadySeenModules;
while (true) {
const SingleCommandCompilationDatabase *Input;
@@ -608,9 +620,17 @@
HadErrors = true;
}
}
- });
+ };
+ // Leave the last worker for the main thread.
+ if (I < NumWorkers - 1) {
+ assert(llvm::llvm_is_multithreaded());
+ WorkerThreads.emplace_back(std::move(Worker));
+ } else {
+ Worker();
+ }
}
- Pool.wait();
+ for (auto &W : WorkerThreads)
+ W.join();
if (Format == ScanningOutputFormat::Full)
FD.printFullOutput(llvm::outs());
Index: clang/test/ClangScanDeps/num-workers.cpp
===================================================================
--- /dev/null
+++ clang/test/ClangScanDeps/num-workers.cpp
@@ -0,0 +1,31 @@
+// REQUIRES: thread_support
+// RUN: rm -rf %t.dir
+// RUN: rm -f %t.*.cdb
+// RUN: mkdir -p %t.dir
+// RUN: cp %s %t.dir/file.cpp
+// RUN: cp %s %t.dir/file2.cpp
+// RUN: echo > %t.dir/header.h
+
+// RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/num-workers-1cmd.json > %t.1cmd.cdb
+// RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/num-workers-2cmds.json > %t.2cmds.cdb
+//
+// RUN: clang-scan-deps -compilation-database %t.1cmd.cdb -j 2 -v | \
+// RUN: FileCheck --check-prefixes=CHECK,CHECK1F1W %s
+//
+// RUN: clang-scan-deps -compilation-database %t.2cmds.cdb -j 1 -v | \
+// RUN: FileCheck --check-prefixes=CHECK,CHECK2F1W %s
+//
+// RUN: clang-scan-deps -compilation-database %t.2cmds.cdb -j 2 -v | \
+// RUN: FileCheck --check-prefixes=CHECK,CHECK2F2W %s
+//
+// RUN: clang-scan-deps -compilation-database %t.2cmds.cdb -j 10 -v | \
+// RUN: FileCheck --check-prefixes=CHECK,CHECK2F2W %s
+
+#include "header.h"
+
+// CHECK1F1W: Running clang-scan-deps on 1 files using 1 workers
+// CHECK2F1W: Running clang-scan-deps on 2 files using 1 workers
+// CHECK2F2W: Running clang-scan-deps on 2 files using 2 workers
+
+// CHECK: file.cpp
+// CHECK-NEXT: header.h
Index: clang/test/ClangScanDeps/Inputs/num-workers-2cmds.json
===================================================================
--- /dev/null
+++ clang/test/ClangScanDeps/Inputs/num-workers-2cmds.json
@@ -0,0 +1,12 @@
+[
+{
+ "directory": "DIR",
+ "command": "clang -E -fsyntax-only DIR/file.cpp",
+ "file": "DIR/file.cpp"
+},
+{
+ "directory": "DIR",
+ "command": "clang -E -fsyntax-only DIR/file2.cpp",
+ "file": "DIR/file2.cpp"
+}
+]
Index: clang/test/ClangScanDeps/Inputs/num-workers-1cmd.json
===================================================================
--- /dev/null
+++ clang/test/ClangScanDeps/Inputs/num-workers-1cmd.json
@@ -0,0 +1,7 @@
+[
+{
+ "directory": "DIR",
+ "command": "clang -E -fsyntax-only DIR/file.cpp",
+ "file": "DIR/file.cpp"
+}
+]
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
