[PATCH 1/2] Fix libX11 early init causing mmap layout changed

[Jinyang He]

After libX.so release 1.8 with `--enable-thread-safety-constructor`,
the libX.so calls `malloc` indirectly by calling XInitThreads() in
init_array. The raw_ preempts `malloc` in EXE and calls gcl_init_alloc()
before main. It set rlimit_stack before execve itself, and in linux
kernel this action changes the mmap way from topdown to downtop
(in many archs). Thus, saved the rlimit_stack if before_main and
restore it if it need re-execve itself.
---
 gcl/h/unrandomize.h | 16 ++++++++++++++++
 gcl/o/alloc.c       | 10 ++++++++++
 gcl/o/main.c        |  2 ++
 3 files changed, 28 insertions(+)

diff --git a/gcl/h/unrandomize.h b/gcl/h/unrandomize.h
index 04b7c507e..9ca66d0a9 100644
--- a/gcl/h/unrandomize.h
+++ b/gcl/h/unrandomize.h
@@ -5,6 +5,9 @@
 #include <string.h>
 #include <alloca.h>
 #include <errno.h>
+#if defined(__linux__) && defined(RLIMIT_STACK)
+#include <sys/resource.h>
+#endif
 
 {
   errno=0;
@@ -53,6 +56,19 @@
        errno=0;
 #ifdef HAVE_GCL_CLEANUP
        gcl_cleanup(0);
+#endif
+#if defined(__linux__) && defined(RLIMIT_STACK)
+       {
+         extern struct rlimit rl_stack_saved;
+         /* Reset the rlim_cur incase*/
+         if (rl_stack_saved.rlim_cur != 0 && rl_stack_saved.rlim_max != 0)
+           if(setrlimit(RLIMIT_STACK, &rl_stack_saved)) {
+             printf("restore rlimit_stack failure %d\n",errno);
+             exit(-1);
+           }
+         rl_stack_saved = (struct rlimit){0, 0};
+         errno=0;
+       }
 #endif
        execve(*a,a,n);
        printf("execve failure %d\n",errno);
diff --git a/gcl/o/alloc.c b/gcl/o/alloc.c
index 8620df495..4d29ec860 100644
--- a/gcl/o/alloc.c
+++ b/gcl/o/alloc.c
@@ -1189,6 +1189,10 @@ init_tm(enum type t, char *name, int elsize, int nelts, 
int sgc,int distinct) {
    simplicity.  set_maxpage is overloaded, and the positioning of its
    call is too fragile.  20050115 CM*/
 static int gcl_alloc_initialized;
+#if defined(__linux__) && defined(RLIMIT_STACK)
+extern int before_main;
+struct rlimit rl_stack_saved;
+#endif
 
 object malloc_list=Cnil;
 
@@ -1238,6 +1242,12 @@ gcl_init_alloc(void *cs_start) {
     }
 
     massert(!getrlimit(RLIMIT_STACK, &rl));
+#ifdef __linux__
+    if (before_main)
+      rl_stack_saved = rl;
+    else
+      rl_stack_saved = (struct rlimit){0, 0};
+#endif
     if (rl.rlim_cur!=RLIM_INFINITY && (rl.rlim_max == RLIM_INFINITY || 
rl.rlim_max > rl.rlim_cur)) {
       rl.rlim_cur = rl.rlim_max; /* == RLIM_INFINITY ? rl.rlim_max : 
rl.rlim_max/64; */
       massert(!setrlimit(RLIMIT_STACK,&rl));
diff --git a/gcl/o/main.c b/gcl/o/main.c
index 6621c3a16..be241d1af 100644
--- a/gcl/o/main.c
+++ b/gcl/o/main.c
@@ -574,8 +574,10 @@ 
DEFUN("KCL-SELF",object,fSkcl_self,SI,0,0,NONE,OO,OO,OO,OO,(void),"") {
 
 }
 
+int before_main=1;
 int
 main(int argc, char **argv, char **envp) {
+  before_main=0;
 
   GET_FULL_PATH_SELF(kcl_self);
   *argv=kcl_self;
-- 
2.45.2