Josh Elsasser <j...@elsasser.org> writes:
> On Mon, Dec 03, 2018 at 09:14:55AM +0200, Timo Myyrä wrote:
>> Hi,
>>
>> Here's an update to SBCL to bring it to latest version as I noticed the
>> release
>> fixes the thread support on OpenBSD.
>
> The recent release does indeed fix one threading problem, but
> unfortunately the sb-concurrency tests still hang for me on a
> quad-core machine.
>
> It is now possible to build the threads flavor on powerpc, but I'm not
> convinced that it's worth spending time on.
>
Yeah, the thread support seems to be restored in previous 'almost working' state
at least on amd64. Before 1.4.14 the concurrency tests failed always, now they
seem to pass so I guess it is same state as before.
>> This also changes the thread backend to futex and adds support for core
>> compression using zlib.
>
> Using futexes is probably a good idea, however I'd prefer to wait
> until any known issues with the portable pthreads backend are resolved
> first.
>
Good point.
>> I haven't tested this that much yet but tests seem to pass on amd64 at
>> least with it.
>
> As Solene noted, the build fails when clisp is used as the
> cross-compilation host. I've tracked down the breakage to sbcl
> revision 83bc16d48d70fdf1b1c0194a0e45753df1fbd480 but haven't made
> progress past that.
>
I did few test builds with sbcl releases and 1.4.8 was latest to compile,
compiling 1.4.9 already failed with clisp with similar message as the
1.4.14. Didn't have time yet to dig deeper myself.
>> Timo
>
> Further comments below.
>
>> Index: Makefile
>> ===================================================================
>> RCS file: /cvs/ports/lang/sbcl/Makefile,v
>> retrieving revision 1.38
>> diff -u -p -u -p -r1.38 Makefile
>> --- Makefile 24 Jun 2018 18:33:36 -0000 1.38
>> +++ Makefile 3 Dec 2018 07:06:41 -0000
>> @@ -6,7 +6,7 @@ USE_WXNEEDED = Yes
>>
>> COMMENT= compiler and runtime system for ANSI Common Lisp
>>
>> -V = 1.4.8
>> +V = 1.4.14
>> DISTNAME= sbcl-${V}-source
>> PKGNAME= sbcl-${V}
>> WRKDIST= ${WRKDIR}/sbcl-${V}
>> @@ -21,7 +21,7 @@ PERMIT_PACKAGE_CDROM= Yes
>>
>> MASTER_SITES= ${MASTER_SITE_SOURCEFORGE:=sbcl/}
>>
>> -WANTLIB= c m util
>> +WANTLIB= c m util z
>>
>> PSEUDO_FLAVORS= native_bootstrap
>>
>> @@ -30,10 +30,10 @@ PSEUDO_FLAVORS= native_bootstrap
>> FLAVORS= threads
>> FLAVOR?=
>>
>> -EXTRA_PARAMS=
>> +EXTRA_PARAMS= --with-sb-core-compression
>> .if ${FLAVOR:Mthreads}
>> ONLY_FOR_ARCHS = amd64
>> -EXTRA_PARAMS+= --with-sb-thread
>> +EXTRA_PARAMS+= --with-sb-thread --with-sb-futex
>> WANTLIB+= pthread
>> .endif
>>
>> Index: distinfo
>> ===================================================================
>> RCS file: /cvs/ports/lang/sbcl/distinfo,v
>> retrieving revision 1.16
>> diff -u -p -u -p -r1.16 distinfo
>> --- distinfo 24 Jun 2018 18:33:36 -0000 1.16
>> +++ distinfo 3 Dec 2018 07:06:41 -0000
>> @@ -1,2 +1,2 @@
>> -SHA256 (sbcl-1.4.8-source.tar.bz2) =
>> ziS2K4hIJiHCKKT9bIAjUDMLYmPhsbmEYO60qB1/szU=
>> -SIZE (sbcl-1.4.8-source.tar.bz2) = 6040563
>> +SHA256 (sbcl-1.4.14-source.tar.bz2) =
>> TfJu1E1FWAzuy/nhpunkCV3nPHaZxrlFu+jMhxAwHCw=
>> +SIZE (sbcl-1.4.14-source.tar.bz2) = 6187821
>> Index: patches/patch-src_runtime_Config_generic-openbsd
>> ===================================================================
>> RCS file:
>> /cvs/ports/lang/sbcl/patches/patch-src_runtime_Config_generic-openbsd,v
>> retrieving revision 1.3
>> diff -u -p -u -p -r1.3 patch-src_runtime_Config_generic-openbsd
>> --- patches/patch-src_runtime_Config_generic-openbsd 8 Mar 2018 15:17:39
>> -0000 1.3
>> +++ patches/patch-src_runtime_Config_generic-openbsd 3 Dec 2018 07:06:41
>> -0000
>> @@ -12,12 +12,12 @@ Index: src/runtime/Config.generic-openbs
>> # provided with absolutely no warranty. See the COPYING and CREDITS
>> # files for more information.
>>
>> --LINKFLAGS += -export-dynamic -Wl,-z,wxneeded
>> +-LINKFLAGS += -Wl,-z,wxneeded
>> +LINKFLAGS += -Wl,--export-dynamic -Wl,-z,wxneeded
>
> This whole hunk can be dropped, another makefile fragment already adds
> -Wl,--export-dynamic
>
>> OS_LIBS += -lutil
>>
>> ifdef LISP_FEATURE_SB_THREAD
>> -@@ -17,9 +17,7 @@ CFLAGS += -pthread
>> +@@ -17,9 +17,7 @@ CFLAGS += -pthread -DOS_THREAD_STACK
>> OS_LIBS += -pthread
>> endif
>>
>> Index: patches/patch-src_runtime_GNUmakefile
>> ===================================================================
>> RCS file: /cvs/ports/lang/sbcl/patches/patch-src_runtime_GNUmakefile,v
>> retrieving revision 1.8
>> diff -u -p -u -p -r1.8 patch-src_runtime_GNUmakefile
>> --- patches/patch-src_runtime_GNUmakefile 8 Mar 2018 15:17:39 -0000
>> 1.8
>> +++ patches/patch-src_runtime_GNUmakefile 3 Dec 2018 07:06:41 -0000
>> @@ -2,7 +2,7 @@ $OpenBSD: patch-src_runtime_GNUmakefile,
>> Index: src/runtime/GNUmakefile
>> --- src/runtime/GNUmakefile.orig
>> +++ src/runtime/GNUmakefile
>> -@@ -30,7 +30,7 @@ __LDFLAGS__ =
>> +@@ -34,7 +34,7 @@ __LDFLAGS__ =
>>
>> include ../../output/prefix.def
>>
>> Index: patches/patch-src_runtime_bsd-os_c
>> ===================================================================
>> RCS file: patches/patch-src_runtime_bsd-os_c
>> diff -N patches/patch-src_runtime_bsd-os_c
>> --- /dev/null 1 Jan 1970 00:00:00 -0000
>> +++ patches/patch-src_runtime_bsd-os_c 3 Dec 2018 07:06:41 -0000
>> @@ -0,0 +1,63 @@
>> +OB$OpenBSD$
>> +
>> +Index: src/runtime/bsd-os.c
>> +--- src/runtime/bsd-os.c.orig
>> ++++ src/runtime/bsd-os.c
>> +@@ -693,7 +693,57 @@ os_dlsym(void *handle, const char *symbol)
>> + return ret;
>> + }
>> +
>> ++#if defined(LISP_FEATURE_SB_THREAD) && defined(LISP_FEATURE_SB_FUTEX) \
>> ++ && !defined(LISP_FEATURE_SB_PTHREAD_FUTEX)
>> ++
>> ++#include <sys/syscall.h>
>> ++#include <unistd.h>
>> ++
>> ++int __syscall(quad_t, ...);
>> ++
>> ++#define FUTEX_WAIT 1
>> ++#define FUTEX_WAKE 2
>> ++#define FUTEX_REQUEUE 3
>> ++
>> ++#define FUTEX_PRIVATE_FLAG 128
>> ++
>> ++#define FUTEX_WAIT_PRIVATE (FUTEX_WAIT | FUTEX_PRIVATE_FLAG)
>> ++#define FUTEX_WAKE_PRIVATE (FUTEX_WAKE | FUTEX_PRIVATE_FLAG)
>> ++#define FUTEX_REQUEUE_PRIVATE (FUTEX_REQUEUE | FUTEX_PRIVATE_FLAG)
>
> Why are you adding all of this and using syscall() instead of just
> including <sys/futex.h> and calling futex()?
>
I don't recall what my train of thought was when I made that. Adding new patch
with direct futex call.
>> ++
>> ++int
>> ++futex_wait(int *lock_word, int oldval, long sec, unsigned long usec)
>> ++{
>> ++ struct timespec timeout;
>> ++ int t;
>> ++
>> ++ if (sec < 0) {
>> ++ t = __syscall(SYS_futex, lock_word, FUTEX_WAIT, oldval, NULL, NULL);
>> ++ }
>> ++ else {
>> ++ timeout.tv_sec = sec;
>> ++ timeout.tv_nsec = usec * 1000;
>> ++ t = __syscall(SYS_futex, lock_word, FUTEX_WAIT, oldval, &timeout,
>> NULL);
>> ++ }
>> ++ if (t == 0)
>> ++ return 0;
>> ++ else if (errno == ETIMEDOUT)
>> ++ return 1;
>> ++ else if (errno == EINTR)
>> ++ return 2;
>> ++ else
>> ++ /* EWOULDBLOCK and others, need to check the lock */
>> ++ return -1;
>> ++}
>> ++
>> ++int
>> ++futex_wake(int *lock_word, int n)
>> ++{
>> ++ return (__syscall(SYS_futex, lock_word, FUTEX_WAKE, n, NULL, NULL));
>> ++}
>> ++
>> + #endif
>> ++#endif /* __OpenBSD__ */
>> +
>> + #if defined(LISP_FEATURE_SB_WTIMER) && !defined(LISP_FEATURE_DARWIN)
>> + /*
>> Index: patches/patch-src_runtime_run-program_c
>> ===================================================================
>> RCS file: /cvs/ports/lang/sbcl/patches/patch-src_runtime_run-program_c,v
>> retrieving revision 1.1
>> diff -u -p -u -p -r1.1 patch-src_runtime_run-program_c
>> --- patches/patch-src_runtime_run-program_c 8 Mar 2018 15:17:39 -0000
>> 1.1
>> +++ patches/patch-src_runtime_run-program_c 3 Dec 2018 07:06:41 -0000
>> @@ -6,7 +6,7 @@ copy of environ than libc.
>> Index: src/runtime/run-program.c
>> --- src/runtime/run-program.c.orig
>> +++ src/runtime/run-program.c
>> -@@ -200,14 +200,18 @@ int spawn(char *program, char *argv[], int sin, int so
>> +@@ -202,14 +202,18 @@ int spawn(char *program, char *argv[], int sin, int so
>> if (pwd && chdir(pwd) < 0) {
>> failure_code = 3;
>> } else {
>> Index: patches/patch-src_runtime_thread_c
>> ===================================================================
>> RCS file: patches/patch-src_runtime_thread_c
>> diff -N patches/patch-src_runtime_thread_c
>> --- /dev/null 1 Jan 1970 00:00:00 -0000
>> +++ patches/patch-src_runtime_thread_c 3 Dec 2018 07:06:41 -0000
>> @@ -0,0 +1,22 @@
>> +$OpenBSD: patch-src_runtime_thread_c,v 1.1 2018/03/08 15:17:39 sthen Exp $
>> +
>> +Index: src/runtime/thread.c
>> +--- src/runtime/thread.c.orig
>> ++++ src/runtime/thread.c
>> +@@ -684,9 +684,16 @@ create_thread_struct(lispobj initial_function) {
>> + * on the alignment passed from os_validate, since that might
>> + * assume the current (e.g. 4k) pagesize, while we calculate with
>> + * the biggest (e.g. 64k) pagesize allowed by the ABI. */
>> ++#if defined(LISP_FEATURE_OPENBSD) && defined(MAP_STACK)
>> ++ void *spaces = mmap(0, THREAD_STRUCT_SIZE, PROT_READ|PROT_WRITE,
>> ++ MAP_PRIVATE|MAP_ANON|MAP_STACK, -1, 0);
>> ++ if(spaces == MAP_FAILED)
>> ++ return NULL;
>> ++#else
>
> Please don't reintroduce this patch, it is no longer needed.
>
>> + void *spaces = os_validate(IS_THREAD_STRUCT, NULL, THREAD_STRUCT_SIZE);
>> + if(!spaces)
>> + return NULL;
>> ++#endif
>> + /* Aligning up is safe as THREAD_STRUCT_SIZE has
>> + * THREAD_ALIGNMENT_BYTES padding. */
>> + char *aligned_spaces = PTR_ALIGN_UP(spaces, THREAD_ALIGNMENT_BYTES);
>> Index: pkg/PLIST
>> ===================================================================
>> RCS file: /cvs/ports/lang/sbcl/pkg/PLIST,v
>> retrieving revision 1.11
>> diff -u -p -u -p -r1.11 PLIST
>> --- pkg/PLIST 8 Mar 2018 15:17:39 -0000 1.11
>> +++ pkg/PLIST 3 Dec 2018 07:06:41 -0000
>> @@ -9,6 +9,8 @@ lib/sbcl/contrib/sb-aclrepl.asd
>> lib/sbcl/contrib/sb-aclrepl.fasl
>> lib/sbcl/contrib/sb-bsd-sockets.asd
>> lib/sbcl/contrib/sb-bsd-sockets.fasl
>> +lib/sbcl/contrib/sb-capstone.asd
>> +lib/sbcl/contrib/sb-capstone.fasl
>> lib/sbcl/contrib/sb-cltl2.asd
>> lib/sbcl/contrib/sb-cltl2.fasl
>> lib/sbcl/contrib/sb-concurrency.asd
Here's a new diff fixing some of the feedback.
Compiles and compile tests pass on amd64 but when running actual tests 'make
test' I get following:
::: UNEXPECTED-FAILURE :WITHOUT-INTERRUPTS+CONDITION-WAIT due to SIMPLE-ERROR:
"The assertion (THREAD-ALIVE-P THREAD) failed with THREAD =
#<THREAD ABORTED {10052E66C3}>."
and a bit later it gets stuck on:
::: Running :DEADLOCK-DETECTION.3
What do you think about adding print/texinfo dependency and use it to generate
the info file. It complains about unknown UTF-8 encoding during the makeinfo
call:
/expanding variables in variables.texinfo
/creating encodings.texi-temp
makeinfo -I "docstrings/" -I "../../contrib/" asdf.texinfo
asdf.texinfo:2219: warning: @strong{Note...} produces a spurious
cross-reference in Info; reword to avoid that.
asdf.texinfo:2222: warning: @strong{Note...} produces a spurious
cross-reference in Info; reword to avoid that.
makeinfo -I "docstrings/" -I "../../contrib/" sbcl.texinfo
sbcl.texinfo:5: warning: unrecognized encoding name `UTF-8'.
The ports versions should understand that but I'm not sure if the manuals
contain stuff which really requires UTF-8 encoding..
Once the clisp bootstrap is fixed the port would be ready for update without the
futex stuff.
Timo
Index: Makefile
===================================================================
RCS file: /cvs/ports/lang/sbcl/Makefile,v
retrieving revision 1.38
diff -u -p -u -p -r1.38 Makefile
--- Makefile 24 Jun 2018 18:33:36 -0000 1.38
+++ Makefile 9 Dec 2018 10:04:13 -0000
@@ -6,7 +6,7 @@ USE_WXNEEDED = Yes
COMMENT= compiler and runtime system for ANSI Common Lisp
-V = 1.4.8
+V = 1.4.14
DISTNAME= sbcl-${V}-source
PKGNAME= sbcl-${V}
WRKDIST= ${WRKDIR}/sbcl-${V}
@@ -21,7 +21,7 @@ PERMIT_PACKAGE_CDROM= Yes
MASTER_SITES= ${MASTER_SITE_SOURCEFORGE:=sbcl/}
-WANTLIB= c m util
+WANTLIB= c m util z
PSEUDO_FLAVORS= native_bootstrap
@@ -30,7 +30,7 @@ PSEUDO_FLAVORS= native_bootstrap
FLAVORS= threads
FLAVOR?=
-EXTRA_PARAMS=
+EXTRA_PARAMS= --with-sb-core-compression
.if ${FLAVOR:Mthreads}
ONLY_FOR_ARCHS = amd64
EXTRA_PARAMS+= --with-sb-thread --with-sb-futex
Index: distinfo
===================================================================
RCS file: /cvs/ports/lang/sbcl/distinfo,v
retrieving revision 1.16
diff -u -p -u -p -r1.16 distinfo
--- distinfo 24 Jun 2018 18:33:36 -0000 1.16
+++ distinfo 9 Dec 2018 10:04:13 -0000
@@ -1,2 +1,2 @@
-SHA256 (sbcl-1.4.8-source.tar.bz2) =
ziS2K4hIJiHCKKT9bIAjUDMLYmPhsbmEYO60qB1/szU=
-SIZE (sbcl-1.4.8-source.tar.bz2) = 6040563
+SHA256 (sbcl-1.4.14-source.tar.bz2) =
TfJu1E1FWAzuy/nhpunkCV3nPHaZxrlFu+jMhxAwHCw=
+SIZE (sbcl-1.4.14-source.tar.bz2) = 6187821
Index: patches/patch-src_runtime_Config_generic-openbsd
===================================================================
RCS file:
/cvs/ports/lang/sbcl/patches/patch-src_runtime_Config_generic-openbsd,v
retrieving revision 1.3
diff -u -p -u -p -r1.3 patch-src_runtime_Config_generic-openbsd
--- patches/patch-src_runtime_Config_generic-openbsd 8 Mar 2018 15:17:39
-0000 1.3
+++ patches/patch-src_runtime_Config_generic-openbsd 9 Dec 2018 10:04:13
-0000
@@ -1,23 +1,11 @@
$OpenBSD: patch-src_runtime_Config_generic-openbsd,v 1.3 2018/03/08 15:17:39
sthen Exp $
-clang-5.0.0 doesn't pass -export-dynamic to the linker, instead it
-passes a bogus -e export-dynamic to ld(1). Just use -Wl,--export-dynamic.
-
Don't try to guess (wrong) with clang. Just assume we have pie
Index: src/runtime/Config.generic-openbsd
--- src/runtime/Config.generic-openbsd.orig
+++ src/runtime/Config.generic-openbsd
-@@ -9,7 +9,7 @@
- # provided with absolutely no warranty. See the COPYING and CREDITS
- # files for more information.
-
--LINKFLAGS += -export-dynamic -Wl,-z,wxneeded
-+LINKFLAGS += -Wl,--export-dynamic -Wl,-z,wxneeded
- OS_LIBS += -lutil
-
- ifdef LISP_FEATURE_SB_THREAD
-@@ -17,9 +17,7 @@ CFLAGS += -pthread
+@@ -17,9 +17,7 @@ CFLAGS += -pthread -DOS_THREAD_STACK
OS_LIBS += -pthread
endif
Index: patches/patch-src_runtime_GNUmakefile
===================================================================
RCS file: /cvs/ports/lang/sbcl/patches/patch-src_runtime_GNUmakefile,v
retrieving revision 1.8
diff -u -p -u -p -r1.8 patch-src_runtime_GNUmakefile
--- patches/patch-src_runtime_GNUmakefile 8 Mar 2018 15:17:39 -0000
1.8
+++ patches/patch-src_runtime_GNUmakefile 9 Dec 2018 10:04:13 -0000
@@ -2,7 +2,7 @@ $OpenBSD: patch-src_runtime_GNUmakefile,
Index: src/runtime/GNUmakefile
--- src/runtime/GNUmakefile.orig
+++ src/runtime/GNUmakefile
-@@ -30,7 +30,7 @@ __LDFLAGS__ =
+@@ -34,7 +34,7 @@ __LDFLAGS__ =
include ../../output/prefix.def
Index: patches/patch-src_runtime_bsd-os_c
===================================================================
RCS file: patches/patch-src_runtime_bsd-os_c
diff -N patches/patch-src_runtime_bsd-os_c
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ patches/patch-src_runtime_bsd-os_c 9 Dec 2018 10:04:13 -0000
@@ -0,0 +1,53 @@
+$OpenBSD$
+
+add futex support for OpenBSD
+
+Index: src/runtime/bsd-os.c
+--- src/runtime/bsd-os.c.orig
++++ src/runtime/bsd-os.c
+@@ -693,7 +693,45 @@ os_dlsym(void *handle, const char *symbol)
+ return ret;
+ }
+
++#if defined(LISP_FEATURE_SB_THREAD) && defined(LISP_FEATURE_SB_FUTEX) \
++ && !defined(LISP_FEATURE_SB_PTHREAD_FUTEX)
++
++#include <sys/time.h>
++#include <sys/futex.h>
++
++int
++futex_wait(int *lock_word, int oldval, long sec, unsigned long usec)
++{
++ struct timespec timeout;
++ int t;
++
++ if (sec < 0) {
++ t = futex(lock_word, FUTEX_WAIT, oldval, NULL, NULL);
++ }
++ else {
++ timeout.tv_sec = sec;
++ timeout.tv_nsec = usec * 1000;
++ t = futex(lock_word, FUTEX_WAIT, oldval, &timeout, NULL);
++ }
++ if (t == 0)
++ return 0;
++ else if (errno == ETIMEDOUT)
++ return 1;
++ else if (errno == EINTR)
++ return 2;
++ else
++ /* EWOULDBLOCK and others, need to check the lock */
++ return -1;
++}
++
++int
++futex_wake(int *lock_word, int n)
++{
++ return (futex(lock_word, FUTEX_WAKE, n, NULL, NULL));
++}
++
+ #endif
++#endif /* __OpenBSD__ */
+
+ #if defined(LISP_FEATURE_SB_WTIMER) && !defined(LISP_FEATURE_DARWIN)
+ /*
Index: patches/patch-src_runtime_run-program_c
===================================================================
RCS file: /cvs/ports/lang/sbcl/patches/patch-src_runtime_run-program_c,v
retrieving revision 1.1
diff -u -p -u -p -r1.1 patch-src_runtime_run-program_c
--- patches/patch-src_runtime_run-program_c 8 Mar 2018 15:17:39 -0000
1.1
+++ patches/patch-src_runtime_run-program_c 9 Dec 2018 10:04:13 -0000
@@ -6,7 +6,7 @@ copy of environ than libc.
Index: src/runtime/run-program.c
--- src/runtime/run-program.c.orig
+++ src/runtime/run-program.c
-@@ -200,14 +200,18 @@ int spawn(char *program, char *argv[], int sin, int so
+@@ -202,14 +202,18 @@ int spawn(char *program, char *argv[], int sin, int so
if (pwd && chdir(pwd) < 0) {
failure_code = 3;
} else {
Index: pkg/PLIST
===================================================================
RCS file: /cvs/ports/lang/sbcl/pkg/PLIST,v
retrieving revision 1.11
diff -u -p -u -p -r1.11 PLIST
--- pkg/PLIST 8 Mar 2018 15:17:39 -0000 1.11
+++ pkg/PLIST 9 Dec 2018 10:04:13 -0000
@@ -9,6 +9,8 @@ lib/sbcl/contrib/sb-aclrepl.asd
lib/sbcl/contrib/sb-aclrepl.fasl
lib/sbcl/contrib/sb-bsd-sockets.asd
lib/sbcl/contrib/sb-bsd-sockets.fasl
+lib/sbcl/contrib/sb-capstone.asd
+lib/sbcl/contrib/sb-capstone.fasl
lib/sbcl/contrib/sb-cltl2.asd
lib/sbcl/contrib/sb-cltl2.fasl
lib/sbcl/contrib/sb-concurrency.asd
