Hi Ports, Armin, This diff makes PyPy build on OpenBSD once more, and in doing so updates to the latest version.
I only started out to mark PyPy wxneeded, but it led to an odd situation with PyPy: * The bootstrap tarball no longer works, since the contained binary is not marked wxneeded, so we have to make a new bootstrap using CPython. * However, the build system itself (under CPython) is not W^X compliant either. The diff makes the PyPy build work under CPython without requiring CPython itself to be marked wxneeded (although I see another thread where people are discussing marking CPython anyway). It also marks the PyPy binary wxneeded. Since we have to have a new bootstrap, I figured I may aswell update PyPy to the latest version too. Armin, this is a minimal distribution patch that I am going to use for now until we can figure out this rmmap refactoring. It simply duplicates alloc_hinted thus avoiding the NonConstant problem. I'm still working on the correct refactoring separately. People, note that this does not make the PyPy JIT W^X compatible. Although I hope we can be there eventually, apparently this will not be a trivial task (it's going to be hard to know all the places where we need to flip the page protection flags, and when we get it wrong and get a crash, it's going to be hard to know where the crash came from). Anyway, this is the first step. OK anyone? Bootstrap tarball is here (please could you host this Stuart?): http://theunixzoo.co.uk/random/pypy-bootstrap-amd64-5.3.1.tar.xz Index: Makefile =================================================================== RCS file: /cvs/ports/lang/pypy/Makefile,v retrieving revision 1.24 diff -u -p -r1.24 Makefile --- Makefile 13 Jun 2016 08:44:17 -0000 1.24 +++ Makefile 11 Aug 2016 20:19:10 -0000 @@ -18,7 +18,7 @@ COMMENT = fast implementation of the Py # You can use the no_bootstrap FLAVOR to make a new bootstrap. This uses # CPython to build PyPy instead. Note this is slower. -V = 5.3.0 +V = 5.3.1 BOOTSTRAP_V = ${V} DISTNAME = pypy2-v${V}-src PKGNAME = pypy-${V} @@ -95,7 +95,8 @@ do-build: pypy --jit loop_longevity=300 \ ../../rpython/bin/rpython --source --opt=jit .endif - cd ${WRKDIR}/usession/testing_1 && ${SETENV} ${MAKE_ENV} ${MAKE_PROGRAM} + cd ${WRKDIR}/usession/testing_1 && ${SETENV} ${MAKE_ENV} \ + ${MAKE_PROGRAM} LDFLAGS_LINK="-Wl,-z,wxneeded -pthread" PYPY_INST_LIB_DIR = ${PREFIX}/pypy/lib PYPY_LD_LIBRARY_PATH = LD_LIBRARY_PATH=${PYPY_INST_LIB_DIR} Index: distinfo =================================================================== RCS file: /cvs/ports/lang/pypy/distinfo,v retrieving revision 1.16 diff -u -p -r1.16 distinfo --- distinfo 13 Jun 2016 08:44:17 -0000 1.16 +++ distinfo 11 Aug 2016 20:19:10 -0000 @@ -1,4 +1,4 @@ -SHA256 (pypy/pypy-bootstrap-amd64-5.3.0.tar.xz) = edy2uGOHyc3D9NndGeSZ86szEvpiJmFSQdAC/wtLBpg= -SHA256 (pypy/pypy2-v5.3.0-src.tar.bz2) = QULrj0A4ELyIpJEXkrtaUC4VLflYBuM+aQUMgozRYNU= -SIZE (pypy/pypy-bootstrap-amd64-5.3.0.tar.xz) = 21976516 -SIZE (pypy/pypy2-v5.3.0-src.tar.bz2) = 17361429 +SHA256 (pypy/pypy-bootstrap-amd64-5.3.1.tar.xz) = J8lRGufV48uBbJu302zRSAf3RdXxjEokruugShyOJ0U= +SHA256 (pypy/pypy2-v5.3.1-src.tar.bz2) = MaUrq1hKvzoPDe/Rv5opEx2rCN9DiF5+7d/H3Jtxg24= +SIZE (pypy/pypy-bootstrap-amd64-5.3.1.tar.xz) = 22269372 +SIZE (pypy/pypy2-v5.3.1-src.tar.bz2) = 17361760 Index: patches/patch-rpython_jit_backend_x86_detect_feature_py =================================================================== RCS file: patches/patch-rpython_jit_backend_x86_detect_feature_py diff -N patches/patch-rpython_jit_backend_x86_detect_feature_py --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ patches/patch-rpython_jit_backend_x86_detect_feature_py 11 Aug 2016 20:19:10 -0000 @@ -0,0 +1,30 @@ +$OpenBSD$ + +Make the CPython bootstrap W^X compatible. + +--- rpython/jit/backend/x86/detect_feature.py.orig Tue Jun 14 08:46:04 2016 ++++ rpython/jit/backend/x86/detect_feature.py Thu Aug 11 08:49:22 2016 +@@ -1,17 +1,20 @@ + import sys + import struct + from rpython.rtyper.lltypesystem import lltype, rffi +-from rpython.rlib.rmmap import alloc, free ++from rpython.rlib.rmmap import alloc, free, set_pages_executable + ++CPU_INFO_SZ = 4096 ++ + def cpu_info(instr): +- data = alloc(4096) ++ data = alloc(CPU_INFO_SZ, no_exec=True) + pos = 0 + for c in instr: + data[pos] = c + pos += 1 + fnptr = rffi.cast(lltype.Ptr(lltype.FuncType([], lltype.Signed)), data) ++ set_pages_executable(data, CPU_INFO_SZ) + code = fnptr() +- free(data, 4096) ++ free(data, CPU_INFO_SZ) + return code + + def detect_sse2(): Index: patches/patch-rpython_rlib_rmmap_py =================================================================== RCS file: patches/patch-rpython_rlib_rmmap_py diff -N patches/patch-rpython_rlib_rmmap_py --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ patches/patch-rpython_rlib_rmmap_py 11 Aug 2016 20:19:10 -0000 @@ -0,0 +1,113 @@ +$OpenBSD$ + +Make the CPython bootstrap W^X compatible. + +--- rpython/rlib/rmmap.py.orig Tue Jun 14 08:46:04 2016 ++++ rpython/rlib/rmmap.py Thu Aug 11 08:49:22 2016 +@@ -155,6 +155,9 @@ if _POSIX: + c_mremap, _ = external('mremap', + [PTR, size_t, size_t, rffi.ULONG], PTR) + ++ c_mprotect, _ = external('mprotect', ++ [PTR, size_t, rffi.INT], rffi.INT) ++ + # this one is always safe + _pagesize = rffi_platform.getintegerfunctionresult('getpagesize', + includes=includes) +@@ -694,11 +697,29 @@ if _POSIX: + def alloc_hinted(hintp, map_size): + flags = MAP_PRIVATE | MAP_ANONYMOUS + prot = PROT_EXEC | PROT_READ | PROT_WRITE ++ + if we_are_translated(): + flags = NonConstant(flags) + prot = NonConstant(prot) + return c_mmap_safe(hintp, map_size, prot, flags, -1, 0) + ++ def alloc_hinted_noexec(hintp, map_size): ++ """Same as alloc_hinted, but allocates pages non-executable. ++ Duplicated because of constancy constraints on prot.""" ++ ++ flags = MAP_PRIVATE | MAP_ANONYMOUS ++ prot = PROT_READ | PROT_WRITE ++ ++ if we_are_translated(): ++ flags = NonConstant(flags) ++ prot = NonConstant(prot) ++ return c_mmap_safe(hintp, map_size, prot, flags, -1, 0) ++ ++ def set_pages_executable(addr, size): ++ rv = c_mprotect(addr, size, PROT_EXEC) ++ if rv < 0: ++ debug.fatalerror_notb("set_pages_executable failed") ++ + def clear_large_memory_chunk_aligned(addr, map_size): + addr = rffi.cast(PTR, addr) + flags = MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS +@@ -714,10 +735,10 @@ if _POSIX: + pos = -0x4fff0000 # for reproducible results + hint = Hint() + +- def alloc(map_size): ++ def alloc(map_size, no_exec=False): + """Allocate memory. This is intended to be used by the JIT, +- so the memory has the executable bit set and gets allocated +- internally in case of a sandboxed process. ++ so the memory has the executable bit set (unless no_exec=True) ++ and gets allocated internally in case of a sandboxed process. + """ + from errno import ENOMEM + from rpython.rlib import debug +@@ -730,11 +751,17 @@ if _POSIX: + if res == rffi.cast(PTR, 0): + raise MemoryError + return res +- res = alloc_hinted(rffi.cast(PTR, hint.pos), map_size) ++ if no_exec: ++ res = alloc_hinted_noexec(rffi.cast(PTR, hint.pos), map_size) ++ else: ++ res = alloc_hinted(rffi.cast(PTR, hint.pos), map_size) + if res == rffi.cast(PTR, -1): + # some systems (some versions of OS/X?) complain if they + # are passed a non-zero address. Try again. +- res = alloc_hinted(rffi.cast(PTR, 0), map_size) ++ if no_exec: ++ res = alloc_hinted_noexec(rffi.cast(PTR, 0), map_size) ++ else: ++ res = alloc_hinted(rffi.cast(PTR, 0), map_size) + if res == rffi.cast(PTR, -1): + # ENOMEM simply raises MemoryError, but other errors are fatal + if rposix.get_saved_errno() != ENOMEM: +@@ -748,7 +775,7 @@ if _POSIX: + else: + hint.pos += map_size + return res +- alloc._annenforceargs_ = (int,) ++ alloc._annenforceargs_ = (int, bool) + + if _CYGWIN: + free = c_free_safe +@@ -886,11 +913,13 @@ elif _MS_WINDOWS: + hint = Hint() + # XXX this has no effect on windows + +- def alloc(map_size): ++ def alloc(map_size, no_exec=False): + """Allocate memory. This is intended to be used by the JIT, + so the memory has the executable bit set. + XXX implement me: it should get allocated internally in + case of a sandboxed process ++ ++ XXX no_exec ignored on windows. + """ + null = lltype.nullptr(rffi.VOIDP.TO) + res = VirtualAlloc_safe(null, map_size, MEM_COMMIT | MEM_RESERVE, +@@ -902,7 +931,7 @@ elif _MS_WINDOWS: + lltype.free(arg, flavor='raw') + # ignore errors, just try + return res +- alloc._annenforceargs_ = (int,) ++ alloc._annenforceargs_ = (int, bool) + + def free(ptr, map_size): + VirtualFree_safe(ptr, 0, MEM_RELEASE) Index: patches/patch-rpython_rlib_rvmprof_src_vmprof_config_h =================================================================== RCS file: patches/patch-rpython_rlib_rvmprof_src_vmprof_config_h diff -N patches/patch-rpython_rlib_rvmprof_src_vmprof_config_h --- patches/patch-rpython_rlib_rvmprof_src_vmprof_config_h 13 Jun 2016 08:44:17 -0000 1.2 +++ /dev/null 1 Jan 1970 00:00:00 -0000 @@ -1,18 +0,0 @@ -$OpenBSD: patch-rpython_rlib_rvmprof_src_vmprof_config_h,v 1.2 2016/06/13 08:44:17 edd Exp $ ---- rpython/rlib/rvmprof/src/vmprof_config.h.orig Sun Jun 5 16:45:04 2016 -+++ rpython/rlib/rvmprof/src/vmprof_config.h Sun Jun 5 16:47:15 2016 -@@ -1,10 +1,14 @@ -+#if !defined(__OpenBSD__) - #define HAVE_SYS_UCONTEXT_H -+#endif - #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) - #ifdef __i386__ - #define PC_FROM_UCONTEXT uc_mcontext.mc_eip - #else - #define PC_FROM_UCONTEXT uc_mcontext.mc_rip - #endif -+#elif defined(__OpenBSD__) -+#define PC_FROM_UCONTEXT sc_rip - #elif defined( __APPLE__) - #if ((ULONG_MAX) == (UINT_MAX)) - #define PC_FROM_UCONTEXT uc_mcontext->__ss.__eip Index: patches/patch-rpython_rlib_rvmprof_src_vmprof_getpc_h =================================================================== RCS file: patches/patch-rpython_rlib_rvmprof_src_vmprof_getpc_h diff -N patches/patch-rpython_rlib_rvmprof_src_vmprof_getpc_h --- patches/patch-rpython_rlib_rvmprof_src_vmprof_getpc_h 13 Jun 2016 08:44:17 -0000 1.2 +++ /dev/null 1 Jan 1970 00:00:00 -0000 @@ -1,12 +0,0 @@ -$OpenBSD: patch-rpython_rlib_rvmprof_src_vmprof_getpc_h,v 1.2 2016/06/13 08:44:17 edd Exp $ ---- rpython/rlib/rvmprof/src/vmprof_getpc.h.orig Sun Apr 24 05:08:33 2016 -+++ rpython/rlib/rvmprof/src/vmprof_getpc.h Sun Jun 5 16:43:26 2016 -@@ -65,6 +65,8 @@ - #elif defined(HAVE_CYGWIN_SIGNAL_H) - #include <cygwin/signal.h> - typedef ucontext ucontext_t; -+#elif defined(__OpenBSD__) -+#include <signal.h> - #endif - - -- Best Regards Edd Barrett http://www.theunixzoo.co.uk
