This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 4a80df4ff61 Fix, cleanup and refactor adding apt dependencies when
building image (#55151)
4a80df4ff61 is described below
commit 4a80df4ff617e334c9e1cea423d7579e09e83b7c
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon Sep 1 23:16:20 2025 +0200
Fix, cleanup and refactor adding apt dependencies when building image
(#55151)
After migrating to python built from sources, it turned out that we
had a weird mixture of system and source build python in our CI images
images. There were some dependencies installed that installed
python3 debian dependencies and they were breaking the way how source
build python 3.11 interacted with system one. Namely - debian Python 3.11
had internal ABI incompatibility between 3.11.4 and 3.11.5 that caused
the ssl build with latest 3.11.5 fail when 3.11.5 system python has been
installed while the new python was being built - because system includes
interfered with the build process.
The system python was pulled by "software-properties-common" and "gdb" -
we do not need to use either of them during the build process, and user
in
In order to avoid this, we now make sure that we do not install any
debian dependencies that pull any of the python3 system packages, before
we install Python from sources.
We also reviewed and cleaned the way how we configure list of packages
installed during image building:
* packages are reviewed and unnecessary ones removed
* packages that are pulling python3 system packages are moved to
additional apt dev dependencies
* additional apt dev dependencies are installed AFTER python is
installed
* the packages listed in scripts are now one-per-line
* they are alphabetically sorted
* finally LD_LIBRARY_PATH is set to have the "source-installed"
python libraries to be always prioritised against the system
libraries if system libraries are installed to the main "/usr/lib"
directories.
---
Dockerfile | 146 ++++++++++++++++++++++++++---
Dockerfile.ci | 148 ++++++++++++++++++++++++++----
docker-stack-docs/changelog.rst | 5 +
scripts/docker/install_os_dependencies.sh | 141 +++++++++++++++++++++++++---
4 files changed, 394 insertions(+), 46 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index a20fcaaebc6..cd8d37bc80b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -97,7 +97,9 @@ COPY <<"EOF" /install_os_dependencies.sh
set -euo pipefail
if [[ "$#" != 1 ]]; then
+ echo
echo "ERROR! There should be 'runtime', 'ci' or 'dev' parameter passed as
argument.".
+ echo
exit 1
fi
@@ -111,20 +113,72 @@ elif [[ "${1}" == "dev" ]]; then
elif [[ "${1}" == "ci" ]]; then
INSTALLATION_TYPE="CI"
else
+ echo
echo "ERROR! Wrong argument. Passed ${1} and it should be one of
'runtime', 'ci' or 'dev'.".
+ echo
exit 1
fi
function get_dev_apt_deps() {
if [[ "${DEV_APT_DEPS=}" == "" ]]; then
- DEV_APT_DEPS="apt-transport-https apt-utils build-essential dirmngr \
-freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils libev4
libev-dev libffi-dev libgeos-dev \
-libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev
libsasl2-modules \
-libssl-dev libxmlsec1 libxmlsec1-dev locales lsb-release openssh-client
pkgconf sasl2-bin \
-software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev wget \
-gdb lcov pkg-config libbz2-dev libgdbm-dev libgdbm-compat-dev liblzma-dev \
-libncurses5-dev libreadline6-dev libsqlite3-dev lzma lzma-dev tk-dev uuid-dev \
-libzstd-dev"
+ DEV_APT_DEPS="\
+apt-transport-https \
+apt-utils \
+build-essential \
+dirmngr \
+freetds-bin \
+freetds-dev \
+git \
+graphviz \
+graphviz-dev \
+krb5-user \
+lcov \
+ldap-utils \
+libbluetooth-dev \
+libbz2-dev \
+libc6-dev \
+libdb-dev \
+libev-dev \
+libev4 \
+libffi-dev \
+libgdbm-compat-dev \
+libgdbm-dev \
+libgdbm-dev \
+libgeos-dev \
+libkrb5-dev \
+libldap2-dev \
+libleveldb-dev \
+libleveldb1d \
+liblzma-dev \
+libncurses5-dev \
+libreadline6-dev \
+libsasl2-2 \
+libsasl2-dev \
+libsasl2-modules \
+libsqlite3-dev \
+libssl-dev \
+libxmlsec1 \
+libxmlsec1-dev \
+libzstd-dev \
+locales \
+lsb-release \
+lzma \
+lzma-dev \
+openssh-client \
+openssl \
+pkg-config \
+pkgconf \
+sasl2-bin \
+sqlite3 \
+sudo \
+tk-dev \
+unixodbc \
+unixodbc-dev \
+uuid-dev \
+wget \
+xz-utils \
+zlib1g-dev \
+"
export DEV_APT_DEPS
fi
}
@@ -138,15 +192,43 @@ function get_runtime_apt_deps() {
echo
echo "DEBIAN CODENAME: ${debian_version}"
echo
- debian_version_apt_deps="libffi8 libldap-2.5-0 libssl3 netcat-openbsd"
+ debian_version_apt_deps="\
+libffi8 \
+libldap-2.5-0 \
+libssl3 \
+netcat-openbsd\
+"
echo
echo "APPLIED INSTALLATION CONFIGURATION FOR DEBIAN VERSION:
${debian_version}"
echo
if [[ "${RUNTIME_APT_DEPS=}" == "" ]]; then
- RUNTIME_APT_DEPS="apt-transport-https apt-utils \
-curl dumb-init freetds-bin git krb5-user libev4 libgeos-dev \
-ldap-utils libsasl2-2 libsasl2-modules libxmlsec1 locales
${debian_version_apt_deps} \
-lsb-release openssh-client python3-selinux rsync sasl2-bin sqlite3 sudo
unixodbc wget"
+ RUNTIME_APT_DEPS="\
+${debian_version_apt_deps} \
+apt-transport-https \
+apt-utils \
+curl \
+dumb-init \
+freetds-bin \
+git \
+gnupg \
+iputils-ping \
+krb5-user \
+ldap-utils \
+libev4 \
+libgeos-dev \
+libsasl2-2 \
+libsasl2-modules \
+libxmlsec1 \
+locales \
+lsb-release \
+openssh-client \
+rsync \
+sasl2-bin \
+sqlite3 \
+sudo \
+unixodbc \
+wget\
+"
export RUNTIME_APT_DEPS
fi
}
@@ -188,9 +270,15 @@ function install_debian_dev_dependencies() {
echo "DEBIAN CODENAME: ${debian_version}"
echo
# shellcheck disable=SC2086
- apt-get install -y --no-install-recommends ${DEV_APT_DEPS}
${ADDITIONAL_DEV_APT_DEPS}
+ apt-get install -y --no-install-recommends ${DEV_APT_DEPS}
}
+function install_additional_dev_dependencies() {
+ if [[ "${ADDITIONAL_DEV_APT_DEPS=}" != "" ]]; then
+ # shellcheck disable=SC2086
+ apt-get install -y --no-install-recommends ${ADDITIONAL_DEV_APT_DEPS}
+ fi
+}
function link_python() {
# link python binaries to /usr/local/bin and /usr/python/bin with and
without 3 suffix
@@ -242,6 +330,27 @@ function install_debian_runtime_dependencies() {
}
function install_python() {
+ # If system python (3.11 in bookworm) is installed (via automatic
installation of some dependencies for example), we need
+ # to fail and make sure that it is not there, because there can be strange
interactions if we install
+ # newer version and system libraries are installed, because
+ # when you create a virtualenv part of the shared libraries of Python can
be taken from the system
+ # Installation leading to weird errors when you want to install some
modules - for example when you install ssl:
+ #
/usr/python/lib/python3.11/lib-dynload/_ssl.cpython-311-aarch64-linux-gnu.so:
undefined symbol: _PyModule_Add
+ if dpkg -l | grep '^ii' | grep '^ii libpython' >/dev/null; then
+ echo
+ echo "ERROR! System python is installed by one of the previous steps"
+ echo
+ echo "Please make sure that no python packages are installed by
default. Displaying the reason why libpython3.11 is installed:"
+ echo
+ apt-get install -yqq aptitude >/dev/null
+ aptitude why libpython3.11
+ echo
+ exit 1
+ else
+ echo
+ echo "GOOD! System python is not installed - OK"
+ echo
+ fi
wget -O python.tar.xz
"https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz";
wget -O python.tar.xz.asc
"https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.asc";;
declare -A keys=(
@@ -281,7 +390,8 @@ function install_python() {
LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"
LDFLAGS="${LDFLAGS:--Wl},--strip-all"
./configure --enable-optimizations --prefix=/usr/python/ --with-ensurepip
--build="$gnuArch" \
- --enable-loadable-sqlite-extensions --enable-option-checking=fatal
--enable-shared --with-lto
+ --enable-loadable-sqlite-extensions --enable-option-checking=fatal \
+ --enable-shared --with-lto
make -s -j "$(nproc)" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \
"LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" python
make -s -j "$(nproc)" install
@@ -314,6 +424,7 @@ else
get_dev_apt_deps
install_debian_dev_dependencies
install_python
+ install_additional_dev_dependencies
if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then
install_golang
fi
@@ -1590,6 +1701,11 @@ ENV AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION}
COPY --from=scripts install_os_dependencies.sh /scripts/docker/
RUN bash /scripts/docker/install_os_dependencies.sh dev
+# In case system python is installed, setting LD_LIBRARY_PATH prevents any
case the system python
+# libraries will be accidentally used before the library installed from
sources (which is newer and
+# python interpreter might break if accidentally the old system libraries are
used.
+ENV LD_LIBRARY_PATH="/usr/python/lib"
+
ARG INSTALL_MYSQL_CLIENT="true"
ARG INSTALL_MYSQL_CLIENT_TYPE="mariadb"
ARG INSTALL_MSSQL_CLIENT="true"
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 77776120374..232adc1e333 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -37,7 +37,9 @@ COPY <<"EOF" /install_os_dependencies.sh
set -euo pipefail
if [[ "$#" != 1 ]]; then
+ echo
echo "ERROR! There should be 'runtime', 'ci' or 'dev' parameter passed as
argument.".
+ echo
exit 1
fi
@@ -51,20 +53,72 @@ elif [[ "${1}" == "dev" ]]; then
elif [[ "${1}" == "ci" ]]; then
INSTALLATION_TYPE="CI"
else
+ echo
echo "ERROR! Wrong argument. Passed ${1} and it should be one of
'runtime', 'ci' or 'dev'.".
+ echo
exit 1
fi
function get_dev_apt_deps() {
if [[ "${DEV_APT_DEPS=}" == "" ]]; then
- DEV_APT_DEPS="apt-transport-https apt-utils build-essential dirmngr \
-freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils libev4
libev-dev libffi-dev libgeos-dev \
-libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev
libsasl2-modules \
-libssl-dev libxmlsec1 libxmlsec1-dev locales lsb-release openssh-client
pkgconf sasl2-bin \
-software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev wget \
-gdb lcov pkg-config libbz2-dev libgdbm-dev libgdbm-compat-dev liblzma-dev \
-libncurses5-dev libreadline6-dev libsqlite3-dev lzma lzma-dev tk-dev uuid-dev \
-libzstd-dev"
+ DEV_APT_DEPS="\
+apt-transport-https \
+apt-utils \
+build-essential \
+dirmngr \
+freetds-bin \
+freetds-dev \
+git \
+graphviz \
+graphviz-dev \
+krb5-user \
+lcov \
+ldap-utils \
+libbluetooth-dev \
+libbz2-dev \
+libc6-dev \
+libdb-dev \
+libev-dev \
+libev4 \
+libffi-dev \
+libgdbm-compat-dev \
+libgdbm-dev \
+libgdbm-dev \
+libgeos-dev \
+libkrb5-dev \
+libldap2-dev \
+libleveldb-dev \
+libleveldb1d \
+liblzma-dev \
+libncurses5-dev \
+libreadline6-dev \
+libsasl2-2 \
+libsasl2-dev \
+libsasl2-modules \
+libsqlite3-dev \
+libssl-dev \
+libxmlsec1 \
+libxmlsec1-dev \
+libzstd-dev \
+locales \
+lsb-release \
+lzma \
+lzma-dev \
+openssh-client \
+openssl \
+pkg-config \
+pkgconf \
+sasl2-bin \
+sqlite3 \
+sudo \
+tk-dev \
+unixodbc \
+unixodbc-dev \
+uuid-dev \
+wget \
+xz-utils \
+zlib1g-dev \
+"
export DEV_APT_DEPS
fi
}
@@ -78,15 +132,43 @@ function get_runtime_apt_deps() {
echo
echo "DEBIAN CODENAME: ${debian_version}"
echo
- debian_version_apt_deps="libffi8 libldap-2.5-0 libssl3 netcat-openbsd"
+ debian_version_apt_deps="\
+libffi8 \
+libldap-2.5-0 \
+libssl3 \
+netcat-openbsd\
+"
echo
echo "APPLIED INSTALLATION CONFIGURATION FOR DEBIAN VERSION:
${debian_version}"
echo
if [[ "${RUNTIME_APT_DEPS=}" == "" ]]; then
- RUNTIME_APT_DEPS="apt-transport-https apt-utils \
-curl dumb-init freetds-bin git krb5-user libev4 libgeos-dev \
-ldap-utils libsasl2-2 libsasl2-modules libxmlsec1 locales
${debian_version_apt_deps} \
-lsb-release openssh-client python3-selinux rsync sasl2-bin sqlite3 sudo
unixodbc wget"
+ RUNTIME_APT_DEPS="\
+${debian_version_apt_deps} \
+apt-transport-https \
+apt-utils \
+curl \
+dumb-init \
+freetds-bin \
+git \
+gnupg \
+iputils-ping \
+krb5-user \
+ldap-utils \
+libev4 \
+libgeos-dev \
+libsasl2-2 \
+libsasl2-modules \
+libxmlsec1 \
+locales \
+lsb-release \
+openssh-client \
+rsync \
+sasl2-bin \
+sqlite3 \
+sudo \
+unixodbc \
+wget\
+"
export RUNTIME_APT_DEPS
fi
}
@@ -128,9 +210,15 @@ function install_debian_dev_dependencies() {
echo "DEBIAN CODENAME: ${debian_version}"
echo
# shellcheck disable=SC2086
- apt-get install -y --no-install-recommends ${DEV_APT_DEPS}
${ADDITIONAL_DEV_APT_DEPS}
+ apt-get install -y --no-install-recommends ${DEV_APT_DEPS}
}
+function install_additional_dev_dependencies() {
+ if [[ "${ADDITIONAL_DEV_APT_DEPS=}" != "" ]]; then
+ # shellcheck disable=SC2086
+ apt-get install -y --no-install-recommends ${ADDITIONAL_DEV_APT_DEPS}
+ fi
+}
function link_python() {
# link python binaries to /usr/local/bin and /usr/python/bin with and
without 3 suffix
@@ -182,6 +270,27 @@ function install_debian_runtime_dependencies() {
}
function install_python() {
+ # If system python (3.11 in bookworm) is installed (via automatic
installation of some dependencies for example), we need
+ # to fail and make sure that it is not there, because there can be strange
interactions if we install
+ # newer version and system libraries are installed, because
+ # when you create a virtualenv part of the shared libraries of Python can
be taken from the system
+ # Installation leading to weird errors when you want to install some
modules - for example when you install ssl:
+ #
/usr/python/lib/python3.11/lib-dynload/_ssl.cpython-311-aarch64-linux-gnu.so:
undefined symbol: _PyModule_Add
+ if dpkg -l | grep '^ii' | grep '^ii libpython' >/dev/null; then
+ echo
+ echo "ERROR! System python is installed by one of the previous steps"
+ echo
+ echo "Please make sure that no python packages are installed by
default. Displaying the reason why libpython3.11 is installed:"
+ echo
+ apt-get install -yqq aptitude >/dev/null
+ aptitude why libpython3.11
+ echo
+ exit 1
+ else
+ echo
+ echo "GOOD! System python is not installed - OK"
+ echo
+ fi
wget -O python.tar.xz
"https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz";
wget -O python.tar.xz.asc
"https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.asc";;
declare -A keys=(
@@ -221,7 +330,8 @@ function install_python() {
LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"
LDFLAGS="${LDFLAGS:--Wl},--strip-all"
./configure --enable-optimizations --prefix=/usr/python/ --with-ensurepip
--build="$gnuArch" \
- --enable-loadable-sqlite-extensions --enable-option-checking=fatal
--enable-shared --with-lto
+ --enable-loadable-sqlite-extensions --enable-option-checking=fatal \
+ --enable-shared --with-lto
make -s -j "$(nproc)" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \
"LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" python
make -s -j "$(nproc)" install
@@ -254,6 +364,7 @@ else
get_dev_apt_deps
install_debian_dev_dependencies
install_python
+ install_additional_dev_dependencies
if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then
install_golang
fi
@@ -1390,7 +1501,7 @@ RUN echo "Base image version: ${BASE_IMAGE}"
ARG DEV_APT_COMMAND=""
ARG ADDITIONAL_DEV_APT_COMMAND=""
ARG ADDITIONAL_DEV_ENV_VARS=""
-ARG ADDITIONAL_DEV_APT_DEPS="bash-completion dumb-init git graphviz krb5-user \
+ARG ADDITIONAL_DEV_APT_DEPS="bash-completion dumb-init git gdb graphviz
krb5-user \
less libenchant-2-2 libgcc-11-dev libgeos-dev libpq-dev net-tools
netcat-openbsd \
openssh-server postgresql-client software-properties-common rsync tmux
unzip vim xxd"
@@ -1408,6 +1519,11 @@ COPY --from=scripts install_os_dependencies.sh
/scripts/docker/
RUN bash /scripts/docker/install_os_dependencies.sh ci
+# In case system python is installed, setting LD_LIBRARY_PATH prevents any
case the system python
+# libraries will be accidentally used before the library installed from
sources (which is newer and
+# python interpreter might break if accidentally the old system libraries are
used.
+ENV LD_LIBRARY_PATH="/usr/python/lib"
+
COPY --from=scripts common.sh /scripts/docker/
# Only copy mysql/mssql installation scripts for now - so that changing the
other
diff --git a/docker-stack-docs/changelog.rst b/docker-stack-docs/changelog.rst
index d4b054b5a18..c70b16c3a3c 100644
--- a/docker-stack-docs/changelog.rst
+++ b/docker-stack-docs/changelog.rst
@@ -57,6 +57,11 @@ command:
location has been hard-coded to - for example - detect ``site-packages``
dir, the folders moved to
the ``/usr/python`` directory.
+There might be other subtle changes in the image due to the change of the base
image, some packages (
+especially dev libraries installed with ``apt`` might not be installed by
default if they were installed
+as a side-effect in the original image, however that should only affect those
who want to customise the image.
+They should be able to install in their custom images following the
:doc:`Building the image <build>`
+
Airflow 3.0.2
~~~~~~~~~~~~~
diff --git a/scripts/docker/install_os_dependencies.sh
b/scripts/docker/install_os_dependencies.sh
index 9f341789630..0488f83ec89 100644
--- a/scripts/docker/install_os_dependencies.sh
+++ b/scripts/docker/install_os_dependencies.sh
@@ -19,7 +19,9 @@
set -euo pipefail
if [[ "$#" != 1 ]]; then
+ echo
echo "ERROR! There should be 'runtime', 'ci' or 'dev' parameter passed as
argument.".
+ echo
exit 1
fi
@@ -33,20 +35,72 @@ elif [[ "${1}" == "dev" ]]; then
elif [[ "${1}" == "ci" ]]; then
INSTALLATION_TYPE="CI"
else
+ echo
echo "ERROR! Wrong argument. Passed ${1} and it should be one of
'runtime', 'ci' or 'dev'.".
+ echo
exit 1
fi
function get_dev_apt_deps() {
if [[ "${DEV_APT_DEPS=}" == "" ]]; then
- DEV_APT_DEPS="apt-transport-https apt-utils build-essential dirmngr \
-freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils libev4
libev-dev libffi-dev libgeos-dev \
-libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev
libsasl2-modules \
-libssl-dev libxmlsec1 libxmlsec1-dev locales lsb-release openssh-client
pkgconf sasl2-bin \
-software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev wget \
-gdb lcov pkg-config libbz2-dev libgdbm-dev libgdbm-compat-dev liblzma-dev \
-libncurses5-dev libreadline6-dev libsqlite3-dev lzma lzma-dev tk-dev uuid-dev \
-libzstd-dev"
+ DEV_APT_DEPS="\
+apt-transport-https \
+apt-utils \
+build-essential \
+dirmngr \
+freetds-bin \
+freetds-dev \
+git \
+graphviz \
+graphviz-dev \
+krb5-user \
+lcov \
+ldap-utils \
+libbluetooth-dev \
+libbz2-dev \
+libc6-dev \
+libdb-dev \
+libev-dev \
+libev4 \
+libffi-dev \
+libgdbm-compat-dev \
+libgdbm-dev \
+libgdbm-dev \
+libgeos-dev \
+libkrb5-dev \
+libldap2-dev \
+libleveldb-dev \
+libleveldb1d \
+liblzma-dev \
+libncurses5-dev \
+libreadline6-dev \
+libsasl2-2 \
+libsasl2-dev \
+libsasl2-modules \
+libsqlite3-dev \
+libssl-dev \
+libxmlsec1 \
+libxmlsec1-dev \
+libzstd-dev \
+locales \
+lsb-release \
+lzma \
+lzma-dev \
+openssh-client \
+openssl \
+pkg-config \
+pkgconf \
+sasl2-bin \
+sqlite3 \
+sudo \
+tk-dev \
+unixodbc \
+unixodbc-dev \
+uuid-dev \
+wget \
+xz-utils \
+zlib1g-dev \
+"
export DEV_APT_DEPS
fi
}
@@ -60,15 +114,43 @@ function get_runtime_apt_deps() {
echo
echo "DEBIAN CODENAME: ${debian_version}"
echo
- debian_version_apt_deps="libffi8 libldap-2.5-0 libssl3 netcat-openbsd"
+ debian_version_apt_deps="\
+libffi8 \
+libldap-2.5-0 \
+libssl3 \
+netcat-openbsd\
+"
echo
echo "APPLIED INSTALLATION CONFIGURATION FOR DEBIAN VERSION:
${debian_version}"
echo
if [[ "${RUNTIME_APT_DEPS=}" == "" ]]; then
- RUNTIME_APT_DEPS="apt-transport-https apt-utils \
-curl dumb-init freetds-bin git krb5-user libev4 libgeos-dev \
-ldap-utils libsasl2-2 libsasl2-modules libxmlsec1 locales
${debian_version_apt_deps} \
-lsb-release openssh-client python3-selinux rsync sasl2-bin sqlite3 sudo
unixodbc wget"
+ RUNTIME_APT_DEPS="\
+${debian_version_apt_deps} \
+apt-transport-https \
+apt-utils \
+curl \
+dumb-init \
+freetds-bin \
+git \
+gnupg \
+iputils-ping \
+krb5-user \
+ldap-utils \
+libev4 \
+libgeos-dev \
+libsasl2-2 \
+libsasl2-modules \
+libxmlsec1 \
+locales \
+lsb-release \
+openssh-client \
+rsync \
+sasl2-bin \
+sqlite3 \
+sudo \
+unixodbc \
+wget\
+"
export RUNTIME_APT_DEPS
fi
}
@@ -110,9 +192,15 @@ function install_debian_dev_dependencies() {
echo "DEBIAN CODENAME: ${debian_version}"
echo
# shellcheck disable=SC2086
- apt-get install -y --no-install-recommends ${DEV_APT_DEPS}
${ADDITIONAL_DEV_APT_DEPS}
+ apt-get install -y --no-install-recommends ${DEV_APT_DEPS}
}
+function install_additional_dev_dependencies() {
+ if [[ "${ADDITIONAL_DEV_APT_DEPS=}" != "" ]]; then
+ # shellcheck disable=SC2086
+ apt-get install -y --no-install-recommends ${ADDITIONAL_DEV_APT_DEPS}
+ fi
+}
function link_python() {
# link python binaries to /usr/local/bin and /usr/python/bin with and
without 3 suffix
@@ -164,6 +252,27 @@ function install_debian_runtime_dependencies() {
}
function install_python() {
+ # If system python (3.11 in bookworm) is installed (via automatic
installation of some dependencies for example), we need
+ # to fail and make sure that it is not there, because there can be strange
interactions if we install
+ # newer version and system libraries are installed, because
+ # when you create a virtualenv part of the shared libraries of Python can
be taken from the system
+ # Installation leading to weird errors when you want to install some
modules - for example when you install ssl:
+ #
/usr/python/lib/python3.11/lib-dynload/_ssl.cpython-311-aarch64-linux-gnu.so:
undefined symbol: _PyModule_Add
+ if dpkg -l | grep '^ii' | grep '^ii libpython' >/dev/null; then
+ echo
+ echo "ERROR! System python is installed by one of the previous steps"
+ echo
+ echo "Please make sure that no python packages are installed by
default. Displaying the reason why libpython3.11 is installed:"
+ echo
+ apt-get install -yqq aptitude >/dev/null
+ aptitude why libpython3.11
+ echo
+ exit 1
+ else
+ echo
+ echo "GOOD! System python is not installed - OK"
+ echo
+ fi
wget -O python.tar.xz
"https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz";
wget -O python.tar.xz.asc
"https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.asc";;
declare -A keys=(
@@ -203,7 +312,8 @@ function install_python() {
LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"
LDFLAGS="${LDFLAGS:--Wl},--strip-all"
./configure --enable-optimizations --prefix=/usr/python/ --with-ensurepip
--build="$gnuArch" \
- --enable-loadable-sqlite-extensions --enable-option-checking=fatal
--enable-shared --with-lto
+ --enable-loadable-sqlite-extensions --enable-option-checking=fatal \
+ --enable-shared --with-lto
make -s -j "$(nproc)" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \
"LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" python
make -s -j "$(nproc)" install
@@ -236,6 +346,7 @@ else
get_dev_apt_deps
install_debian_dev_dependencies
install_python
+ install_additional_dev_dependencies
if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then
install_golang
fi
