branch: externals/el-job
commit 5fcd4f9eba3e5751e648a30d5e810b098d4cfbc0
Author: zawatton21 <kurozaw...@gmail.com>
Commit: zawatton21 <kurozaw...@gmail.com>
Fix child process error on Windows by improving core count detection
- Resolves "error: Could not create child process" on Windows by capping
the number of subprocesses.
- Added `actual-core-count` to allow manual override of physical CPU core
count.
- Introduced `windows-core-count` function using `wmic` via cmd.exe to get
the physical core count.
- Implemented `get-max-concurrency` to centralize logic for concurrency
control.
- Used physical core count - 1 instead of logical thread count to avoid
overloading Windows process limits.
- Non-Windows platforms fallback to (1- (num-processors)).
This change ensures stable job spawning on Windows systems with
Hyper-Threading.
Example usage in org-mem.el:
(el-job-launch
:id 'org-mem
:if-busy 'takeover
:inject-vars (org-mem--mk-work-vars)
:load-features '(org-mem-parser)
:inputs #'org-mem--list-files-from-fs
:funcall-per-input #'org-mem-parser--parse-file
:callback #'org-mem--finalize-full
:max-concurrency (get-max-concurrency))
---
el-job.el | 48 +++++++++++++++++++++++++++++++++++++++++++-----
1 file changed, 43 insertions(+), 5 deletions(-)
diff --git a/el-job.el b/el-job.el
index 1393017a4a..1790c01924 100644
--- a/el-job.el
+++ b/el-job.el
@@ -335,6 +335,36 @@ with one character of your choosing, such as a dot."
input-sets
result-sets)
+(defvar actual-core-count 0
+ "Actual number of physical CPU cores. If zero or nil, auto-detect based on
OS.")
+
+(defun windows-core-count ()
+ "Retrieve physical CPU core count on Windows using cmd.exe."
+ (string-to-number
+ (with-temp-buffer
+ (call-process "cmd.exe" nil t nil
+ "/C"
+ "wmic CPU get NumberOfCores /value")
+ (goto-char (point-min))
+ (if (re-search-forward "NumberOfCores=\\([0-9]+\\)" nil t)
+ (match-string 1)
+ "1")))) ;; fallback if wmic fails
+
+(defun get-max-concurrency ()
+ "Return appropriate max concurrency value based on the OS and
actual-core-count.
+On Windows, uses `wmic` via cmd.exe to detect physical cores.
+On non-Windows systems, falls back to (1- (num-processors)).
+Always returns at least 1."
+ (let ((core-count (or (and (numberp actual-core-count)
+ (> actual-core-count 0)
+ actual-core-count)
+ (when (eq system-type 'windows-nt)
+ (windows-core-count))
+ (when (fboundp 'num-processors)
+ (1- (num-processors)))
+ 1)))
+ (max 1 (1- core-count))))
+
;;;###autoload
(cl-defun el-job-launch (&key id
(if-busy 'wait)
@@ -342,7 +372,8 @@ with one character of your choosing, such as a dot."
inject-vars
inputs
funcall-per-input
- callback)
+ callback
+ max-concurrency)
"Run FUNCALL-PER-INPUT in one or more headless Elisp processes.
Then merge the return values \(lists of N lists) into one list
\(of N lists) and pass it to CALLBACK.
@@ -469,10 +500,17 @@ For debugging, see these commands:
(when do-exec
(setf .callback callback)
;; Prevent spawning a dozen processes when we need only one or two
- (let ((machine-cores (max 1 (1- (num-processors)))))
- (setf .n-cores-to-use (if (length< .queued-inputs machine-cores)
- (length .queued-inputs)
- machine-cores))
+ (let* ((machine-cores (max 1 (1- (num-processors))))
+ (user-limit (and (integerp max-concurrency)
+ (> max-concurrency 0)
+ max-concurrency))
+ (max-usable-cores (if user-limit
+ (min machine-cores user-limit)
+ machine-cores)))
+ (setf .n-cores-to-use
+ (if (length< .queued-inputs max-usable-cores)
+ (length .queued-inputs)
+ max-usable-cores))
(when (or (length< .ready .n-cores-to-use)
(not (cl-every #'process-live-p .ready)))
(setq do-respawn t)))
