Hello,
I have created a patch to improve make's behaviour with regards to parallel
execution and system load.
On Linux do the load averages refresh only every 5 seconds. Further do load
averages only ever represent past but not present load. As a consequence does
this lead to idle times during parallel builds where make is waiting for the
load averages to come down. It can also spawn excessive amounts of processes
(although it tries to balance it with its own heuristic).
A better way to achieve a stable load is to use the active number of running
processes & threads over of the load averages. This number can be found in
/proc/loadavg on Linux and is updated in real-time. An example:
$ cat /proc/loadavg
0.00 2.72 6.89 1/404 1890
Here it shows 1 (out of 404) running processes. The patch allows make to use the
4th value from /proc/loadavg.
The overall improvement for a parallel build on an 8-core CPU:
-j -l 24 (old): 269.993128122 seconds time elapsed
-j -l 24 (new): 231.300406406 seconds time elapsed
In comparison:
-j 24: 230.482726977 seconds time elapsed
The behaviour of the -l <value> option with the patch is now much closer to that
of the -j <value> option and shows almost identical build times for an empty system.
The test suite passes successfully with the patch.
You'll find two patch files against make-4.2.1 in the attachment. One for
configure.ac (includes a test for /proc/loadavg usability) and the other for
job.c (modifies the function load_too_high()).
Regards,
Sven
--- make-4.2.1/configure.ac 2016-06-06 13:27:31.000000000 +0100
+++ make-4.2.1.1/configure.ac 2016-10-09 18:33:34.574257275 +0100
@@ -176,10 +176,30 @@
AS_IF([test "$have_guile" = yes],
[AC_DEFINE([HAVE_GUILE], [1], [Embed GNU Guile support])])
AM_CONDITIONAL([HAVE_GUILE], [test "$have_guile" = yes])
+AC_CHECK_FUNCS[open close read lseek atoi]
+AC_CACHE_CHECK([for /proc/loadavg usability], [ac_cv_proc_loadavg],
+ [ac_cv_proc_loadavg=no
+ AC_RUN_IFELSE([AC_LANG_SOURCE([[#include <fcntl.h>
+#include <unistd.h>
+#include <stdlib.h>
+int main() {
+ int fd, i, n; char str[64];
+ fd = open("/proc/loadavg", O_RDONLY);
+ if (fd < 0 || read(fd, str, 64) < 0 || lseek(fd, 0, SEEK_SET) < 0) return 1;
+ for (str[63] = i = n = 0; i < 3 && n < 64; ++n) if (str[n] == ' ') ++i;
+ return i < 3 || atoi(&str[n]) < 1 || close(fd) < 0;}]])],
+ [ac_cv_proc_loadavg=yes],
+ [ac_cv_proc_loadavg=no],
+ [ac_cv_proc_loadavg="no (cross-compiling)"])])
+AS_IF([test "$ac_cv_proc_loadavg" = yes],
+[ AC_DEFINE([HAVE_PROC_LOADAVG], [1],
+ [Define to 1 when /proc/loadavg exists and appears usable])
+])
+
AC_FUNC_GETLOADAVG
# AC_FUNC_GETLOADAVG is documented to set the NLIST_STRUCT value, but it
# doesn't. So, we will.
--- make-4.2.1/job.c 2016-05-21 21:22:32.000000000 +0100
+++ make-4.2.1.1/job.c 2016-10-09 19:12:07.425523927 +0100
@@ -1925,10 +1925,60 @@
static int
load_too_high (void)
{
#if defined(__MSDOS__) || defined(VMS) || defined(_AMIGA) ||
defined(__riscos__)
return 1;
+#elif defined(HAVE_PROC_LOADAVG)
+ /* Read from /proc/loadavg and use the number of active running
+ tasks & threads to determine if we can start another process. */
+#define PROC_LOADAVG_SIZE 64
+ static int fd = -2;
+ char proc_loadavg[PROC_LOADAVG_SIZE];
+ int i, p;
+
+ if (max_load_average < 0)
+ return 0;
+
+ if (fd == -2)
+ {
+ fd = open ("/proc/loadavg", O_RDONLY);
+ if (fd < 0)
+ {
+ perror_with_name (_("cannot open /proc/loadavg: "), "open");
+ fd = -1;
+ return 0;
+ }
+ }
+ if (read (fd, proc_loadavg, PROC_LOADAVG_SIZE) < 0)
+ {
+ perror_with_name (_("cannot read /proc/loadavg: "), "read");
+ close(fd);
+ fd = -1;
+ return 0;
+ }
+ if (lseek (fd, 0, SEEK_SET) < 0)
+ {
+ perror_with_name (_("cannot seek on /proc/loadavg: "), "lseek");
+ close(fd);
+ fd = -1;
+ return 0;
+ }
+ proc_loadavg[PROC_LOADAVG_SIZE-1] = '\0';
+ for (i = p = 0; i < 3 && p < PROC_LOADAVG_SIZE; ++p)
+ {
+ if (proc_loadavg[p] == ' ')
+ ++i;
+ }
+ if (i < 3)
+ {
+ errno = ENOTSUP;
+ perror_with_name (_("unsupported format of /proc/loadavg: "),
"load_too_high");
+ close (fd);
+ fd = -1;
+ return 0;
+ }
+ return max_load_average < (double) atoi (&proc_loadavg[p]);
#else
static double last_sec;
static time_t last_now;
double load, guess;
time_t now;
_______________________________________________
Bug-make mailing list
Bug-make@gnu.org
https://lists.gnu.org/mailman/listinfo/bug-make
