This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 92035e3a983b [SPARK-46757][INFRA] Reduce the number of layers of
testing dockerfile
92035e3a983b is described below
commit 92035e3a983b4f1105dcf040fd041ac990f5a83d
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Thu Jan 18 15:41:17 2024 +0800
[SPARK-46757][INFRA] Reduce the number of layers of testing dockerfile
### What changes were proposed in this pull request?
Reduce the number of layers of testing dockerfile
### Why are the changes needed?
to address
https://github.com/apache/spark/pull/44768#pullrequestreview-1827558821
### Does this PR introduce _any_ user-facing change?
no, infra-only
### How was this patch tested?
ci
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #44781 from zhengruifeng/infra_docker_layers.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Kent Yao <[email protected]>
---
dev/infra/Dockerfile | 50 +++++++++++++++++++++++---------------------------
1 file changed, 23 insertions(+), 27 deletions(-)
diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile
index 54f62bbc8202..976f94251d7a 100644
--- a/dev/infra/Dockerfile
+++ b/dev/infra/Dockerfile
@@ -69,15 +69,14 @@ RUN gpg --keyserver hkps://keyserver.ubuntu.com --recv-key
E298A3A825C0D65DFD57C
RUN gpg -a --export E084DAB9 | apt-key add -
RUN add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu
focal-cran40/'
+# See more in SPARK-39959, roxygen2 < 7.2.1
RUN Rscript -e "install.packages(c('devtools', 'knitr', 'markdown', \
'rmarkdown', 'testthat', 'devtools', 'e1071', 'survival', 'arrow', \
- 'ggplot2', 'mvtnorm', 'statmod', 'xml2'),
repos='https://cloud.r-project.org/')"
-
-# See more in SPARK-39959, roxygen2 < 7.2.1
-RUN Rscript -e "devtools::install_version('roxygen2', version='7.2.0',
repos='https://cloud.r-project.org')"
-RUN Rscript -e "devtools::install_version('lintr', version='2.0.1',
repos='https://cloud.r-project.org')"
-RUN Rscript -e "devtools::install_version('pkgdown', version='2.0.1',
repos='https://cloud.r-project.org')"
-RUN Rscript -e "devtools::install_version('preferably', version='0.4',
repos='https://cloud.r-project.org')"
+ 'ggplot2', 'mvtnorm', 'statmod', 'xml2'),
repos='https://cloud.r-project.org/')" && \
+ Rscript -e "devtools::install_version('roxygen2', version='7.2.0',
repos='https://cloud.r-project.org')" && \
+ Rscript -e "devtools::install_version('lintr', version='2.0.1',
repos='https://cloud.r-project.org')" && \
+ Rscript -e "devtools::install_version('pkgdown', version='2.0.1',
repos='https://cloud.r-project.org')" && \
+ Rscript -e "devtools::install_version('preferably', version='0.4',
repos='https://cloud.r-project.org')"
# See more in SPARK-39735
ENV R_LIBS_SITE
"/usr/local/lib/R/site-library:${R_LIBS_SITE}:/usr/lib/R/library"
@@ -99,12 +98,11 @@ ARG BASIC_PIP_PKGS="numpy pyarrow>=14.0.0 six==1.16.0
pandas<=2.1.4 scipy plotly
# Python deps for Spark Connect
ARG CONNECT_PIP_PKGS="grpcio==1.59.3 grpcio-status==1.59.3 protobuf==4.25.1
googleapis-common-protos==1.56.4"
-
-RUN python3.9 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting
$CONNECT_PIP_PKGS
# Add torch as a testing dependency for TorchDistributor and
DeepspeedTorchDistributor
-RUN python3.9 -m pip install 'torch<=2.0.1' torchvision --index-url
https://download.pytorch.org/whl/cpu
-RUN python3.9 -m pip install deepspeed torcheval
-RUN python3.9 -m pip cache purge
+RUN python3.9 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting
$CONNECT_PIP_PKGS && \
+ python3.9 -m pip install 'torch<=2.0.1' torchvision --index-url
https://download.pytorch.org/whl/cpu && \
+ python3.9 -m pip install deepspeed torcheval && \
+ python3.9 -m pip cache purge
# Install Python 3.10 at the last stage to avoid breaking Python 3.9
RUN add-apt-repository ppa:deadsnakes/ppa
@@ -112,10 +110,10 @@ RUN apt-get update && apt-get install -y \
python3.10 python3.10-distutils \
&& rm -rf /var/lib/apt/lists/*
RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10
-RUN python3.10 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting
$CONNECT_PIP_PKGS
-RUN python3.10 -m pip install 'torch<=2.0.1' torchvision --index-url
https://download.pytorch.org/whl/cpu
-RUN python3.10 -m pip install deepspeed torcheval
-RUN python3.10 -m pip cache purge
+RUN python3.10 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting
$CONNECT_PIP_PKGS && \
+ python3.10 -m pip install 'torch<=2.0.1' torchvision --index-url
https://download.pytorch.org/whl/cpu && \
+ python3.10 -m pip install deepspeed torcheval && \
+ python3.10 -m pip cache purge
# Install Python 3.11 at the last stage to avoid breaking the existing Python
installations
RUN add-apt-repository ppa:deadsnakes/ppa
@@ -123,10 +121,10 @@ RUN apt-get update && apt-get install -y \
python3.11 python3.11-distutils \
&& rm -rf /var/lib/apt/lists/*
RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11
-RUN python3.11 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting
$CONNECT_PIP_PKGS
-RUN python3.11 -m pip install 'torch<=2.0.1' torchvision --index-url
https://download.pytorch.org/whl/cpu
-RUN python3.11 -m pip install deepspeed torcheval
-RUN python3.11 -m pip cache purge
+RUN python3.11 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting
$CONNECT_PIP_PKGS && \
+ python3.11 -m pip install 'torch<=2.0.1' torchvision --index-url
https://download.pytorch.org/whl/cpu && \
+ python3.11 -m pip install deepspeed torcheval && \
+ python3.11 -m pip cache purge
# Install Python 3.12 at the last stage to avoid breaking the existing Python
installations
RUN add-apt-repository ppa:deadsnakes/ppa
@@ -135,11 +133,9 @@ RUN apt-get update && apt-get install -y \
&& rm -rf /var/lib/apt/lists/*
RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12
# TODO(SPARK-46647) Add unittest-xml-reporting into Python 3.12 image when it
supports Python 3.12
-RUN python3.12 -m pip install $BASIC_PIP_PKGS $CONNECT_PIP_PKGS lxml
# TODO(SPARK-46078) Use official one instead of nightly build when it's ready
-RUN python3.12 -m pip install --pre torch --index-url
https://download.pytorch.org/whl/nightly/cpu
-RUN python3.12 -m pip install torchvision --index-url
https://download.pytorch.org/whl/cpu
-RUN python3.12 -m pip install torcheval
-RUN python3.12 -m pip cache purge
-
-RUN df -h
+RUN python3.12 -m pip install $BASIC_PIP_PKGS $CONNECT_PIP_PKGS lxml && \
+ python3.12 -m pip install --pre torch --index-url
https://download.pytorch.org/whl/nightly/cpu && \
+ python3.12 -m pip install torchvision --index-url
https://download.pytorch.org/whl/cpu && \
+ python3.12 -m pip install torcheval && \
+ python3.12 -m pip cache purge
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
