Hello,
here is an updated version of the patch improving the performance of the
-l option. It's a diff against the current git master.
The update includes O_CLOEXEC for opening the file descriptor so it
won't "bleed" an open file descriptor to child processes. The configure
script checks for a useable /proc/loadavg file, but the result can be
overwritten manually with --enable-proc-loadavg={yes,no}.
Cheers,
Sven
On 09/10/16 19:58, Sven C. Dack wrote:
Hello,
I have created a patch to improve make's behaviour with regards to
parallel execution and system load.
On Linux do the load averages refresh only every 5 seconds. Further do
load averages only ever represent past but not present load. As a
consequence does this lead to idle times during parallel builds where
make is waiting for the load averages to come down. It can also spawn
excessive amounts of processes (although it tries to balance it with
its own heuristic).
A better way to achieve a stable load is to use the active number of
running processes & threads over of the load averages. This number can
be found in /proc/loadavg on Linux and is updated in real-time. An
example:
$ cat /proc/loadavg
0.00 2.72 6.89 1/404 1890
Here it shows 1 (out of 404) running processes. The patch allows make
to use the 4th value from /proc/loadavg.
The overall improvement for a parallel build on an 8-core CPU:
-j -l 24 (old): 269.993128122 seconds time elapsed
-j -l 24 (new): 231.300406406 seconds time elapsed
In comparison:
-j 24: 230.482726977 seconds time elapsed
The behaviour of the -l <value> option with the patch is now much
closer to that of the -j <value> option and shows almost identical
build times for an empty system.
The test suite passes successfully with the patch.
You'll find two patch files against make-4.2.1 in the attachment. One
for configure.ac (includes a test for /proc/loadavg usability) and the
other for job.c (modifies the function load_too_high()).
Regards,
Sven
_______________________________________________
Bug-make mailing list
Bug-make@gnu.org
https://lists.gnu.org/mailman/listinfo/bug-make
diff --git a/configure.ac b/configure.ac
index 1b03135..38930b7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -132,6 +132,32 @@ AS_IF([test "$ac_cv_func_gettimeofday" = yes],
[Define to 1 if you have a standard gettimeofday function])
])
+dnl The --enable-proc-loadavg option will automatically test for the
+dnl presence of a working /proc/loadavg file, but it can also be set
+dnl manually in order to override the test result.
+AC_CACHE_CHECK(for usable /proc/loadavg, ac_cv_proc_loadavg_test,
+ [AC_RUN_IFELSE([AC_LANG_SOURCE([[
+#include <fcntl.h>
+#include <unistd.h>
+#include <stdlib.h>
+int main() {
+ int fd, i, n; char str[64];
+ fd = open("/proc/loadavg", O_RDONLY | O_CLOEXEC);
+ if (fd < 0 || read(fd, str, 64) < 0 || lseek(fd, 0, SEEK_SET) < 0) return 1;
+ for (str[63] = i = n = 0; i < 3 && n < 64; ++n) if (str[n] == ' ') ++i;
+ return i < 3 || strtoul(&str[n], NULL, 10) < 1 || close(fd) < 0; } ]])],
+ ac_cv_proc_loadavg_test=yes,
+ ac_cv_proc_loadavg_test=no)])
+
+AC_MSG_CHECKING(wether to use /proc/loadavg)
+AC_ARG_ENABLE(proc-loadavg,
+ AS_HELP_STRING([--enable-proc-loadavg={yes|no}],[Allow to limit the processes started by make to not exceed the total number of active processes on a system based on /proc/loadavg [default=autodetect]]),
+ ac_cv_proc_loadavg=$enableval,
+ ac_cv_proc_loadavg=$ac_cv_proc_loadavg_test)
+AS_IF(test x$ac_cv_proc_loadavg = xyes,
+ AC_DEFINE(USE_PROC_LOADAVG, 1, [Defined when load limiting shall be based on /proc/loadavg]))
+AC_MSG_RESULT($ac_cv_proc_loadavg)
+
AC_CHECK_FUNCS([strdup strndup umask mkstemp mktemp fdopen \
dup dup2 getcwd realpath sigsetmask sigaction \
getgroups seteuid setegid setlinebuf setreuid setregid \
diff --git a/job.c b/job.c
index 5f24597..b1ff68e 100644
--- a/job.c
+++ b/job.c
@@ -1963,6 +1963,59 @@ load_too_high (void)
{
#if defined(__MSDOS__) || defined(VMS) || defined(_AMIGA) || defined(__riscos__)
return 1;
+#elif defined(USE_PROC_LOADAVG)
+ /* Read from /proc/loadavg and use the number of active running
+ tasks & threads to determine if we can start another process. */
+#define PROC_LOADAVG_SIZE 64
+ static int fd = -2;
+ char proc_loadavg[PROC_LOADAVG_SIZE];
+ int i, p;
+ unsigned long load;
+
+ if (max_load_average < 0.0 || fd == -1)
+ return 0;
+
+ if (fd == -2)
+ {
+ fd = open ("/proc/loadavg", O_RDONLY | O_CLOEXEC);
+ if (fd < 0)
+ {
+ perror_with_name (_("cannot open /proc/loadavg: "), "open");
+ fd = -1;
+ return 0;
+ }
+ }
+ p = read (fd, proc_loadavg, PROC_LOADAVG_SIZE);
+ if (p < 0)
+ {
+ perror_with_name (_("cannot read /proc/loadavg: "), "read");
+ close(fd);
+ fd = -1;
+ return 0;
+ }
+ if (lseek (fd, 0, SEEK_SET) < 0)
+ {
+ perror_with_name (_("cannot seek on /proc/loadavg: "), "lseek");
+ close(fd);
+ fd = -1;
+ return 0;
+ }
+ proc_loadavg[p < PROC_LOADAVG_SIZE ? p : PROC_LOADAVG_SIZE - 1] = '\0';
+ for (i = p = 0; i < 3 && p < PROC_LOADAVG_SIZE; ++p)
+ {
+ if (proc_loadavg[p] == ' ')
+ ++i;
+ }
+ if (i < 3)
+ {
+ errno = ENOTSUP;
+ perror_with_name (_("unsupported format of /proc/loadavg: "), "load_too_high");
+ close (fd);
+ fd = -1;
+ return 0;
+ }
+ load = strtoul(&proc_loadavg[p], NULL, 10);
+ return max_load_average < (double) load;
#else
static double last_sec;
static time_t last_now;
_______________________________________________
Bug-make mailing list
Bug-make@gnu.org
https://lists.gnu.org/mailman/listinfo/bug-make
