This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 92483818515 HDDS-13778. Use lean multi-arch image for Hadoop (#9134)
92483818515 is described below
commit 9248381851515f2585acb82c0507053c49036853
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Fri Oct 10 21:19:50 2025 +0200
HDDS-13778. Use lean multi-arch image for Hadoop (#9134)
---
hadoop-ozone/dev-support/checks/acceptance.sh | 6 +++---
hadoop-ozone/dist/pom.xml | 6 +++++-
hadoop-ozone/dist/src/main/compose/common/hadoop-test.sh | 6 +++---
hadoop-ozone/dist/src/main/compose/ozone-ha/.env | 2 +-
hadoop-ozone/dist/src/main/compose/ozone/.env | 2 +-
hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env | 4 ++--
hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env | 4 ++--
hadoop-ozone/dist/src/main/compose/ozonesecure/.env | 4 ++--
hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env | 4 ++--
hadoop-ozone/dist/src/main/compose/xcompat/.env | 4 ++--
10 files changed, 23 insertions(+), 19 deletions(-)
diff --git a/hadoop-ozone/dev-support/checks/acceptance.sh
b/hadoop-ozone/dev-support/checks/acceptance.sh
index b9d6f9a3b4e..ac007337587 100755
--- a/hadoop-ozone/dev-support/checks/acceptance.sh
+++ b/hadoop-ozone/dev-support/checks/acceptance.sh
@@ -51,7 +51,7 @@ if [[ "${OZONE_ACCEPTANCE_SUITE}" == "s3a" ]]; then
OZONE_ACCEPTANCE_TEST_TYPE="maven"
if [[ -z "${HADOOP_AWS_DIR}" ]]; then
- HADOOP_VERSION=$(mvn help:evaluate -Dexpression=hadoop.version -q
-DforceStdout -Dscan=false)
+ hadoop_version=$(mvn help:evaluate -Dexpression=hadoop.version -q
-DforceStdout -Dscan=false)
export HADOOP_AWS_DIR=${OZONE_ROOT}/target/hadoop-src
fi
@@ -66,7 +66,7 @@ if [[ "${OZONE_ACCEPTANCE_SUITE}" == "s3a" ]]; then
if [[ ! -e "${dir}" ]] || [[ ! -d "${dir}"/src/test/resources ]]; then
mkdir -p "${dir}"
if [[ ! -f "${dir}.tar.gz" ]]; then
- local
url="https://www.apache.org/dyn/closer.lua?action=download&filename=hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}-src.tar.gz";
+ local
url="https://www.apache.org/dyn/closer.lua?action=download&filename=hadoop/common/hadoop-${hadoop_version}/hadoop-${hadoop_version}-src.tar.gz";
echo "Downloading Hadoop from ${url}"
curl -LSs --fail -o "${dir}.tar.gz" "$url" || return 1
fi
@@ -75,7 +75,7 @@ if [[ "${OZONE_ACCEPTANCE_SUITE}" == "s3a" ]]; then
}
if ! download_hadoop_aws "${HADOOP_AWS_DIR}"; then
- echo "Failed to download Hadoop ${HADOOP_VERSION}" > "${REPORT_FILE}"
+ echo "Failed to download Hadoop ${hadoop_version}" > "${REPORT_FILE}"
exit 1
fi
fi
diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml
index d01c1321842..b5a62d4835d 100644
--- a/hadoop-ozone/dist/pom.xml
+++ b/hadoop-ozone/dist/pom.xml
@@ -24,10 +24,14 @@
<packaging>jar</packaging>
<name>Apache Ozone Distribution</name>
<properties>
- <!-- suffix appended to Ozone version to get Docker image version -->
+ <!-- Hadoop Docker image for current version; multi-arch Hadoop 3.4.2-lean
only available from ghcr.io currently -->
+ <docker.hadoop.image>ghcr.io/apache/hadoop</docker.hadoop.image>
+ <!-- suffix appended to Hadoop version to get Docker image version -->
+ <docker.hadoop.image.flavor>-lean</docker.hadoop.image.flavor>
<docker.ozone-runner.version>20250625-2-jdk21</docker.ozone-runner.version>
<docker.ozone-testkr5b.image>ghcr.io/apache/ozone-testkrb5:20241129-1</docker.ozone-testkr5b.image>
<docker.ozone.image>apache/ozone</docker.ozone.image>
+ <!-- suffix appended to Ozone version to get Docker image version; only
used in xcompat test, to avoid old centos-based images -->
<docker.ozone.image.flavor>-rocky</docker.ozone.image.flavor>
<downloadSources>true</downloadSources>
<file.encoding>UTF-8</file.encoding>
diff --git a/hadoop-ozone/dist/src/main/compose/common/hadoop-test.sh
b/hadoop-ozone/dist/src/main/compose/common/hadoop-test.sh
index 6c95470c792..288f2dfac04 100755
--- a/hadoop-ozone/dist/src/main/compose/common/hadoop-test.sh
+++ b/hadoop-ozone/dist/src/main/compose/common/hadoop-test.sh
@@ -21,13 +21,13 @@ if [[ ${SECURITY_ENABLED} == "true" ]]; then
fi
export
COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.yaml}":../common/${extra_compose_file}
-: ${HADOOP_IMAGE:="apache/hadoop"}
+: ${HADOOP_IMAGE:="${docker.hadoop.image}"}
: ${HADOOP_TEST_IMAGES:=""}
if [[ -z "${HADOOP_TEST_IMAGES}" ]]; then
# hadoop2 image is only available from Docker Hub
HADOOP_TEST_IMAGES="${HADOOP_TEST_IMAGES} apache/hadoop:${hadoop2.version}"
- HADOOP_TEST_IMAGES="${HADOOP_TEST_IMAGES} ${HADOOP_IMAGE}:${hadoop.version}"
+ HADOOP_TEST_IMAGES="${HADOOP_TEST_IMAGES}
${HADOOP_IMAGE}:${hadoop.version}${docker.hadoop.image.flavor}"
fi
export HADOOP_MAJOR_VERSION=3
@@ -53,7 +53,7 @@ source "$COMPOSE_DIR/../testlib.sh"
for HADOOP_TEST_IMAGE in $HADOOP_TEST_IMAGES; do
export HADOOP_TEST_IMAGE
- hadoop_version="${HADOOP_TEST_IMAGE##*:}"
+ hadoop_version=$(docker run --rm "${HADOOP_TEST_IMAGE}" bash -c "hadoop
version | grep -m1 '^Hadoop' | cut -f2 -d' '")
export HADOOP_MAJOR_VERSION=${hadoop_version%%.*}
docker-compose --ansi never --profile hadoop up -d nm rm
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/.env
b/hadoop-ozone/dist/src/main/compose/ozone-ha/.env
index 6507664fad7..2186e05a270 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-ha/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/.env
@@ -15,7 +15,7 @@
# limitations under the License.
HDDS_VERSION=${hdds.version}
-HADOOP_IMAGE=apache/hadoop
+HADOOP_IMAGE=${docker.hadoop.image}
OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
OZONE_RUNNER_IMAGE=apache/ozone-runner
OZONE_OPTS=
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/.env
b/hadoop-ozone/dist/src/main/compose/ozone/.env
index 6507664fad7..2186e05a270 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozone/.env
@@ -15,7 +15,7 @@
# limitations under the License.
HDDS_VERSION=${hdds.version}
-HADOOP_IMAGE=apache/hadoop
+HADOOP_IMAGE=${docker.hadoop.image}
OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
OZONE_RUNNER_IMAGE=apache/ozone-runner
OZONE_OPTS=
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
index 729a0a399a3..1571c3aa824 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
@@ -15,8 +15,8 @@
# limitations under the License.
HDDS_VERSION=${hdds.version}
-HADOOP_IMAGE=apache/hadoop
-HADOOP_VERSION=${hadoop.version}
+HADOOP_IMAGE=${docker.hadoop.image}
+HADOOP_VERSION=${hadoop.version}${docker.hadoop.image.flavor}
OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
OZONE_RUNNER_IMAGE=apache/ozone-runner
OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env
b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env
index c3a2c5329aa..c260913a2f5 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env
@@ -15,8 +15,8 @@
# limitations under the License.
HDDS_VERSION=${hdds.version}
-HADOOP_IMAGE=apache/hadoop
-HADOOP_VERSION=${hadoop.version}
+HADOOP_IMAGE=${docker.hadoop.image}
+HADOOP_VERSION=${hadoop.version}${docker.hadoop.image.flavor}
OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
OZONE_RUNNER_IMAGE=apache/ozone-runner
OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
index c3a2c5329aa..c260913a2f5 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
@@ -15,8 +15,8 @@
# limitations under the License.
HDDS_VERSION=${hdds.version}
-HADOOP_IMAGE=apache/hadoop
-HADOOP_VERSION=${hadoop.version}
+HADOOP_IMAGE=${docker.hadoop.image}
+HADOOP_VERSION=${hadoop.version}${docker.hadoop.image.flavor}
OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
OZONE_RUNNER_IMAGE=apache/ozone-runner
OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env
b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env
index 2625c4fbe90..d7c817854ba 100644
--- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env
@@ -14,8 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-HADOOP_IMAGE=apache/hadoop
-HADOOP_VERSION=${hadoop.version}
+HADOOP_IMAGE=${docker.hadoop.image}
+HADOOP_VERSION=${hadoop.version}${docker.hadoop.image.flavor}
HDDS_VERSION=${hdds.version}
OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
OZONE_RUNNER_IMAGE=apache/ozone-runner
diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/.env
b/hadoop-ozone/dist/src/main/compose/xcompat/.env
index 11979d34326..b99900a5852 100644
--- a/hadoop-ozone/dist/src/main/compose/xcompat/.env
+++ b/hadoop-ozone/dist/src/main/compose/xcompat/.env
@@ -17,8 +17,8 @@
HDDS_VERSION=${hdds.version}
OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
OZONE_RUNNER_IMAGE=apache/ozone-runner
-HADOOP_IMAGE=apache/hadoop
-HADOOP_VERSION=${hadoop.version}
+HADOOP_IMAGE=${docker.hadoop.image}
+HADOOP_VERSION=${hadoop.version}${docker.hadoop.image.flavor}
OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}
OZONE_IMAGE=${docker.ozone.image}
OZONE_IMAGE_FLAVOR="${docker.ozone.image.flavor}"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
