Repository: spark Updated Branches: refs/heads/master e80f18dbd -> 15524c41b
[SPARK-25682][K8S] Package example jars in same target for dev and distro images. This way the image generated from both environments has the same layout, with just a difference in contents that should not affect functionality. Also added some minor error checking to the image script. Closes #22681 from vanzin/SPARK-25682. Authored-by: Marcelo Vanzin <van...@cloudera.com> Signed-off-by: Marcelo Vanzin <van...@cloudera.com> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/15524c41 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/15524c41 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/15524c41 Branch: refs/heads/master Commit: 15524c41b27697c478981cb2df7d5d7df02f3ba4 Parents: e80f18d Author: Marcelo Vanzin <van...@cloudera.com> Authored: Thu Oct 18 10:21:37 2018 -0700 Committer: Marcelo Vanzin <van...@cloudera.com> Committed: Thu Oct 18 10:21:37 2018 -0700 ---------------------------------------------------------------------- bin/docker-image-tool.sh | 16 ++++++++++++++++ .../docker/src/main/dockerfiles/spark/Dockerfile | 5 ++++- 2 files changed, 20 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/15524c41/bin/docker-image-tool.sh ---------------------------------------------------------------------- diff --git a/bin/docker-image-tool.sh b/bin/docker-image-tool.sh index 228494d..f17791a 100755 --- a/bin/docker-image-tool.sh +++ b/bin/docker-image-tool.sh @@ -47,6 +47,11 @@ function build { if [ ! -f "$SPARK_HOME/RELEASE" ]; then # Set image build arguments accordingly if this is a source repo and not a distribution archive. + # + # Note that this will copy all of the example jars directory into the image, and that will + # contain a lot of duplicated jars with the main Spark directory. In a proper distribution, + # the examples directory is cleaned up before generating the distribution tarball, so this + # issue does not occur. IMG_PATH=resource-managers/kubernetes/docker/src/main/dockerfiles BUILD_ARGS=( ${BUILD_PARAMS} @@ -55,6 +60,8 @@ function build { --build-arg spark_jars=assembly/target/scala-$SPARK_SCALA_VERSION/jars --build-arg + example_jars=examples/target/scala-$SPARK_SCALA_VERSION/jars + --build-arg k8s_tests=resource-managers/kubernetes/integration-tests/tests ) else @@ -78,14 +85,23 @@ function build { docker build $NOCACHEARG "${BUILD_ARGS[@]}" \ -t $(image_ref spark) \ -f "$BASEDOCKERFILE" . + if [[ $? != 0 ]]; then + error "Failed to build Spark docker image." + fi docker build $NOCACHEARG "${BINDING_BUILD_ARGS[@]}" \ -t $(image_ref spark-py) \ -f "$PYDOCKERFILE" . + if [[ $? != 0 ]]; then + error "Failed to build PySpark docker image." + fi docker build $NOCACHEARG "${BINDING_BUILD_ARGS[@]}" \ -t $(image_ref spark-r) \ -f "$RDOCKERFILE" . + if [[ $? != 0 ]]; then + error "Failed to build SparkR docker image." + fi } function push { http://git-wip-us.apache.org/repos/asf/spark/blob/15524c41/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile ---------------------------------------------------------------------- diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile index 4bada0d..5f469c3 100644 --- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile +++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile @@ -18,6 +18,7 @@ FROM openjdk:8-alpine ARG spark_jars=jars +ARG example_jars=examples/jars ARG img_path=kubernetes/dockerfiles ARG k8s_tests=kubernetes/tests @@ -32,6 +33,7 @@ RUN set -ex && \ apk upgrade --no-cache && \ apk add --no-cache bash tini libc6-compat linux-pam krb5 krb5-libs && \ mkdir -p /opt/spark && \ + mkdir -p /opt/spark/examples && \ mkdir -p /opt/spark/work-dir && \ touch /opt/spark/RELEASE && \ rm /bin/sh && \ @@ -43,7 +45,8 @@ COPY ${spark_jars} /opt/spark/jars COPY bin /opt/spark/bin COPY sbin /opt/spark/sbin COPY ${img_path}/spark/entrypoint.sh /opt/ -COPY examples /opt/spark/examples +COPY ${example_jars} /opt/spark/examples/jars +COPY examples/src /opt/spark/examples/src COPY ${k8s_tests} /opt/spark/tests COPY data /opt/spark/data --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
