Patch created by Xavier de Gaye --- ./configure.ac.orig 2022-05-17 06:12:56.000000000 -0500 +++ ./configure.ac 2022-06-13 17:24:43.097338878 -0500 @@ -83,5 +83,5 @@ fi AC_MSG_RESULT($interp) - PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$interp + PYTHON_FOR_BUILD='PYTHON_PROJECT_BASE=$(abs_builddir) '$interp fi elif test "$cross_compiling" = maybe; then --- ./configure.orig 2022-05-17 06:12:56.000000000 -0500 +++ ./configure 2022-06-13 17:24:43.096338878 -0500 @@ -2992,5 +2992,5 @@ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $interp" >&5 $as_echo "$interp" >&6; } - PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$interp + PYTHON_FOR_BUILD='PYTHON_PROJECT_BASE=$(abs_builddir) '$interp fi elif test "$cross_compiling" = maybe; then --- ./Lib/distutils/command/build_ext.py.orig 2022-05-17 06:12:56.000000000 -0500 +++ ./Lib/distutils/command/build_ext.py 2022-06-13 17:24:43.093338878 -0500 @@ -238,5 +238,9 @@ else: # building python standard extensions - self.library_dirs.append('.') + if 'PYTHON_PROJECT_BASE' in os.environ: + self.library_dirs.append(os.path.realpath( + os.environ['PYTHON_PROJECT_BASE'])) + else: + self.library_dirs.append('.') # The argument parsing will result in self.define being a string, but @@ -741,5 +745,5 @@ elif sys.platform == 'cygwin': link_libpython = True - elif '_PYTHON_HOST_PLATFORM' in os.environ: + elif 'PYTHON_PROJECT_BASE' in os.environ: # We are cross-compiling for one of the relevant platforms if get_config_var('ANDROID_API_LEVEL') != 0: --- ./Lib/distutils/command/install.py.orig 2022-05-17 06:12:56.000000000 -0500 +++ ./Lib/distutils/command/install.py 2022-06-13 17:24:43.093338878 -0500 @@ -9,5 +9,5 @@ from distutils.core import Command from distutils.debug import DEBUG -from distutils.sysconfig import get_config_vars +from distutils.sysconfig import get_config_vars, cross_compiling from distutils.errors import DistutilsPlatformError from distutils.file_util import write_file @@ -282,10 +282,8 @@ py_version = sys.version.split()[0] - (prefix, exec_prefix) = get_config_vars('prefix', 'exec_prefix') - try: - abiflags = sys.abiflags - except AttributeError: - # sys.abiflags may not be defined on all platforms. - abiflags = '' + prefix, exec_prefix, abiflags = get_config_vars('prefix', + 'exec_prefix', 'ABIFLAGS') + # sys.abiflags may not be defined on all platforms. + abiflags = '' if abiflags is None else abiflags self.config_vars = {'dist_name': self.distribution.get_name(), 'dist_version': self.distribution.get_version(), @@ -420,6 +418,11 @@ "must not supply exec-prefix without prefix") - self.prefix = os.path.normpath(sys.prefix) - self.exec_prefix = os.path.normpath(sys.exec_prefix) + if cross_compiling: + prefix, exec_prefix = get_config_vars('prefix', + 'exec_prefix') + else: + prefix, exec_prefix = sys.prefix, sys.exec_prefix + self.prefix = os.path.normpath(prefix) + self.exec_prefix = os.path.normpath(exec_prefix) else: --- ./Lib/distutils/sysconfig.py.orig 2022-05-17 06:12:56.000000000 -0500 +++ ./Lib/distutils/sysconfig.py 2022-06-13 17:24:46.196339101 -0500 @@ -15,4 +15,5 @@ import sys +from sysconfig import cross_compiling, get_project_base, get_build_time_vars from .errors import DistutilsPlatformError @@ -26,6 +27,11 @@ # live in project/PCbuild/win32 or project/PCbuild/amd64. # set for cross builds -if "_PYTHON_PROJECT_BASE" in os.environ: - project_base = os.path.abspath(os.environ["_PYTHON_PROJECT_BASE"]) +if cross_compiling: + project_base = get_project_base() + build_time_vars = get_build_time_vars() + PREFIX = build_time_vars['prefix'] + BASE_PREFIX = PREFIX + EXEC_PREFIX = build_time_vars['exec_prefix'] + BASE_EXEC_PREFIX = EXEC_PREFIX else: if sys.executable: @@ -46,5 +52,5 @@ return False -_sys_home = getattr(sys, '_home', None) +_sys_home = getattr(sys, '_home', None) if not cross_compiling else None if os.name == 'nt': @@ -64,4 +70,7 @@ python_build = _python_build() +if cross_compiling and not python_build: + raise RuntimeError('PYTHON_PROJECT_BASE is not a build directory') + # Calculate the build qualifier flags if they are defined. Adding the flags @@ -263,6 +272,8 @@ lib_dir = get_python_lib(plat_specific=0, standard_lib=1) config_file = 'config-{}{}'.format(get_python_version(), build_flags) - if hasattr(sys.implementation, '_multiarch'): - config_file += '-%s' % sys.implementation._multiarch + multiarch = (get_config_var('MULTIARCH') if cross_compiling else + getattr(sys.implementation, '_multiarch', '')) + if multiarch: + config_file += '-%s' % multiarch return os.path.join(lib_dir, config_file, 'Makefile') @@ -438,13 +449,5 @@ def _init_posix(): """Initialize the module as appropriate for POSIX systems.""" - # _sysconfigdata is generated at build time, see the sysconfig module - name = os.environ.get('_PYTHON_SYSCONFIGDATA_NAME', - '_sysconfigdata_{abi}_{platform}_{multiarch}'.format( - abi=sys.abiflags, - platform=sys.platform, - multiarch=getattr(sys.implementation, '_multiarch', ''), - )) - _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) - build_time_vars = _temp.build_time_vars + build_time_vars = get_build_time_vars() global _config_vars _config_vars = {} --- ./Lib/distutils/util.py.orig 2022-05-17 06:12:56.000000000 -0500 +++ ./Lib/distutils/util.py 2022-06-13 17:24:43.093338878 -0500 @@ -15,4 +15,5 @@ from distutils import log from distutils.errors import DistutilsByteCompileError +from distutils.sysconfig import cross_compiling, get_config_var def get_host_platform(): @@ -46,6 +47,6 @@ # Set for cross builds explicitly - if "_PYTHON_HOST_PLATFORM" in os.environ: - return os.environ["_PYTHON_HOST_PLATFORM"] + if cross_compiling: + return get_config_var('PYTHON_HOST_PLATFORM') if os.name != "posix" or not hasattr(os, 'uname'): --- ./Lib/sysconfig.py.orig 2022-05-17 06:12:56.000000000 -0500 +++ ./Lib/sysconfig.py 2022-06-13 17:24:49.845339363 -0500 @@ -118,7 +118,18 @@ _PROJECT_BASE = _safe_realpath(os.path.join(_PROJECT_BASE, pardir, pardir)) +cross_compiling = False # set for cross builds -if "_PYTHON_PROJECT_BASE" in os.environ: - _PROJECT_BASE = _safe_realpath(os.environ["_PYTHON_PROJECT_BASE"]) +if "PYTHON_PROJECT_BASE" in os.environ: + cross_compiling = True + _PROJECT_BASE = _safe_realpath(os.environ["PYTHON_PROJECT_BASE"]) + + # Not used when cross compiling. + _PREFIX = None + _BASE_PREFIX = None + _EXEC_PREFIX = None + _BASE_EXEC_PREFIX = None + +def get_project_base(): + return _PROJECT_BASE def _is_python_source_dir(d): @@ -128,5 +139,36 @@ return False -_sys_home = getattr(sys, '_home', None) +_PYTHON_CONFIG = None +def _get_python_config(name): + """Return the value of a python configuration variable.""" + + assert cross_compiling + global _PYTHON_CONFIG + if _PYTHON_CONFIG is None: + # The import would fail when building a native interpreter since, + # at the time generate-posix-vars is run, the _posixsubprocess + # module has not been built yet. This does not happen when + # generate-posix-vars is run during cross-compilation as the + # native interpreter being used is a full-fledged interpreter. + import subprocess + + python_config = os.path.join(_PROJECT_BASE, 'python-config') + vars_ = ['version', 'abiflags', 'machdep', 'multiarch'] + args = ['/bin/sh', python_config] + args.extend('--' + v for v in vars_) + output = subprocess.check_output(args, universal_newlines=True) + _PYTHON_CONFIG = dict(zip(vars_, output.split('\n'))) + assert len(_PYTHON_CONFIG) == len(vars_) + + # The specification of the sysconfigdata file name may differ + # across versions, so forbid Python versions mismatch. + sys_version = '%d.%d' % sys.version_info[:2] + if _PYTHON_CONFIG['version'] != sys_version: + raise RuntimeError('the running python version (%s) does ' + 'not match the cross-compiled version (%s)' % + (sys_version, _PYTHON_CONFIG['version'])) + return _PYTHON_CONFIG.get(name) + +_sys_home = getattr(sys, '_home', None) if not cross_compiling else None if os.name == 'nt': @@ -150,4 +192,6 @@ _INSTALL_SCHEMES[scheme]['include'] = '{srcdir}/Include' _INSTALL_SCHEMES[scheme]['platinclude'] = '{projectbase}/.' +elif cross_compiling: + raise RuntimeError('PYTHON_PROJECT_BASE is not a build directory') @@ -353,11 +397,17 @@ def _get_sysconfigdata_name(): - return os.environ.get('_PYTHON_SYSCONFIGDATA_NAME', - '_sysconfigdata_{abi}_{platform}_{multiarch}'.format( - abi=sys.abiflags, - platform=sys.platform, - multiarch=getattr(sys.implementation, '_multiarch', ''), - )) - + if cross_compiling: + abiflags = _get_python_config('abiflags') + platform = _get_python_config('machdep') + multiarch = _get_python_config('multiarch') + else: + abiflags = sys.abiflags + platform = sys.platform + multiarch = getattr(sys.implementation, '_multiarch', '') + + sysconf_name = '_sysconfigdata_%s_%s' % (abiflags, platform) + if multiarch: + sysconf_name = '%s_%s' % (sysconf_name, multiarch) + return sysconf_name def _generate_posix_vars(): @@ -401,6 +451,7 @@ # This is more than sufficient for ensuring the subsequent call to # get_platform() succeeds. + # The same situation exists when cross compiling. name = _get_sysconfigdata_name() - if 'darwin' in sys.platform: + if 'darwin' in sys.platform or cross_compiling: import types module = types.ModuleType(name) @@ -409,5 +460,6 @@ pybuilddir = 'build/lib.%s-%s' % (get_platform(), _PY_VERSION_SHORT) - if hasattr(sys, "gettotalrefcount"): + if (cross_compiling and 'd' in vars['ABIFLAGS'] or + (not cross_compiling and hasattr(sys, "gettotalrefcount"))): pybuilddir += '-pydebug' os.makedirs(pybuilddir, exist_ok=True) @@ -424,10 +476,37 @@ f.write(pybuilddir) +def _get_module_attr(module, attribute): + _temp = __import__(module, globals(), locals(), [attribute], 0) + return getattr(_temp, attribute) + +_BUILD_TIME_VARS = None +def get_build_time_vars(): + global _BUILD_TIME_VARS + if _BUILD_TIME_VARS is None: + # _sysconfigdata is generated at build time, see _generate_posix_vars() + sysconfigdata = _get_sysconfigdata_name() + if cross_compiling: + if _is_python_source_dir(_PROJECT_BASE): + bdir = os.path.join(_PROJECT_BASE, 'pybuilddir.txt') + with open(bdir, encoding='ascii') as f: + libdir = f.read().strip() + libdir = os.path.join(_PROJECT_BASE, libdir) + else: + libdir = os.path.join(_PROJECT_BASE, + 'lib', 'python%s' % _get_python_config('version')) + sys.path.insert(0, libdir) + try: + _BUILD_TIME_VARS = _get_module_attr(sysconfigdata, + 'build_time_vars') + finally: + sys.path.pop(0) + else: + _BUILD_TIME_VARS = _get_module_attr(sysconfigdata, + 'build_time_vars') + return _BUILD_TIME_VARS + def _init_posix(vars): """Initialize the module as appropriate for POSIX systems.""" - # _sysconfigdata is generated at build time, see _generate_posix_vars() - name = _get_sysconfigdata_name() - _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) - build_time_vars = _temp.build_time_vars + build_time_vars = get_build_time_vars() vars.update(build_time_vars) @@ -637,4 +716,7 @@ """ + if cross_compiling: + return get_config_var('PYTHON_HOST_PLATFORM') + if os.name == 'nt': if 'amd64' in sys.version.lower(): @@ -650,8 +732,4 @@ return sys.platform - # Set for cross builds explicitly - if "_PYTHON_HOST_PLATFORM" in os.environ: - return os.environ["_PYTHON_HOST_PLATFORM"] - # Try to distinguish various flavours of Unix osname, host, release, version, machine = os.uname() --- ./Makefile.pre.in.orig 2022-05-17 06:12:56.000000000 -0500 +++ ./Makefile.pre.in 2022-06-13 17:24:44.562338984 -0500 @@ -257,5 +257,7 @@ UPDATE_FILE=@PYTHON_FOR_REGEN@ $(srcdir)/Tools/scripts/update_file.py PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@ -_PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@ +# To enable setup.py and sysconfig to pick up this AC_SUBST() var. +PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@ + BUILD_GNU_TYPE= @build@ HOST_GNU_TYPE= @host@ @@ -475,5 +477,5 @@ all: @DEF_MAKE_ALL_RULE@ build_all: check-clean-src $(BUILDPYTHON) oldsharedmods sharedmods gdbhooks \ - Programs/_testembed python-config + Programs/_testembed # Check that the source is clean when building out of source. @@ -602,5 +604,7 @@ # initialization to succeed. It will be overwritten by generate-posix-vars # or removed in case of failure. -pybuilddir.txt: $(BUILDPYTHON) +# python-config is a prerequisite as it is used when generating the posix +# variables if cross-compiling. +pybuilddir.txt: python-config $(BUILDPYTHON) @echo "none" > ./pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -S -m sysconfig --generate-posix-vars ;\ @@ -1543,6 +1547,10 @@ done; \ done - $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py \ - $(DESTDIR)$(LIBDEST); \ + sysconf_name=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP); \ + if test -n "$(MULTIARCH)"; then \ + sysconf_name=$${sysconf_name}_$(MULTIARCH); \ + fi; \ + $(INSTALL_DATA) `cat pybuilddir.txt`/$$sysconf_name.py \ + $(DESTDIR)$(LIBDEST) $(INSTALL_DATA) $(srcdir)/LICENSE $(DESTDIR)$(LIBDEST)/LICENSE.txt if test -d $(DESTDIR)$(LIBDEST)/distutils/tests; then \ @@ -1699,5 +1707,9 @@ --install-platlib=$(DESTSHARED) \ --root=$(DESTDIR)/ - -rm $(DESTDIR)$(DESTSHARED)/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py + -sysconf_name=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP); \ + if test -n "$(MULTIARCH)"; then \ + sysconf_name=$${sysconf_name}_$(MULTIARCH); \ + fi; \ + rm $(DESTDIR)$(DESTSHARED)/$$sysconf_name.py -rm -r $(DESTDIR)$(DESTSHARED)/__pycache__ --- ./Misc/python-config.sh.in.orig 2022-05-17 06:12:56.000000000 -0500 +++ ./Misc/python-config.sh.in 2022-06-13 17:24:43.094338878 -0500 @@ -5,5 +5,7 @@ exit_with_usage () { - echo "Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed" + echo "Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|\ + --ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed|\ + --version|--machdep|--multiarch" exit $1 } @@ -67,5 +69,7 @@ PY_EMBED=1 ;; - --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--abiflags|--configdir) + --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|\ + --extension-suffix|--abiflags|--configdir|\ + --version|--machdep|--multiarch) ;; *) @@ -113,4 +117,13 @@ echo "$LIBPL" ;; + --version) + echo "$VERSION" + ;; + --machdep) + echo "@MACHDEP@" + ;; + --multiarch) + echo "@MULTIARCH@" + ;; esac done --- ./setup.py.orig 2022-05-17 06:12:56.000000000 -0500 +++ ./setup.py 2022-06-13 17:24:43.097338878 -0500 @@ -31,4 +31,5 @@ +from sysconfig import cross_compiling as CROSS_COMPILING from distutils import log from distutils.command.build_ext import build_ext @@ -49,7 +50,6 @@ def get_platform(): - # Cross compiling - if "_PYTHON_HOST_PLATFORM" in os.environ: - return os.environ["_PYTHON_HOST_PLATFORM"] + if CROSS_COMPILING: + return sysconfig.get_config_var('PYTHON_HOST_PLATFORM') # Get value of sys.platform @@ -59,5 +59,4 @@ -CROSS_COMPILING = ("_PYTHON_HOST_PLATFORM" in os.environ) HOST_PLATFORM = get_platform() MS_WINDOWS = (HOST_PLATFORM == 'win32') --- /dev/null 2022-06-07 13:19:00.445169466 -0500 +++ ./Misc/NEWS.d/next/Build/2019-11-30-21-53-38.bpo-28833.14IeXH.rst 2022-06-13 17:28:23.273354712 -0500 @@ -0,0 +1,11 @@ +Fix cross-compilation of third-party extension modules. The +``PYTHON_PROJECT_BASE`` environment variable is the path to the directory +where Python has been cross-compiled. It is used by the native python +interpreter to find the target ``sysconfigdata`` module. For example the +following command builds a wheel file to be transfered and installed with +pip on the target platform, provided the native python interpreter and the +cross-compiled one both have the wheel package installed: + +.. code-block:: shell + + $ PYTHON_PROJECT_BASE=/path/to/builddir python setup.py bdist_wheel