This is an automated email from the ASF dual-hosted git repository.
peter-toth pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new 4596d256475f [SPARK-57962][PYTHON] Guard against path traversal in
install_spark tar extraction
4596d256475f is described below
commit 4596d256475f2c6cf21d285de10b23097a9a6d0a
Author: Peter Toth <[email protected]>
AuthorDate: Fri Jul 10 12:16:50 2026 +0200
[SPARK-57962][PYTHON] Guard against path traversal in install_spark tar
extraction
`install_spark()` in `python/pyspark/install.py` downloads a Spark release
archive and extracts it by rewriting each tar member's name with
`os.path.relpath(member.name, package_name + os.path.sep)` and calling
`tar.extract(member, dest)`. `os.path.relpath` does not strip `..` segments (it
can produce them), so a crafted archive member could resolve to a path outside
`dest` (a "zip slip" / path traversal) when extracted.
This change extracts the loop into a helper `_extract_tar` that, before
extracting each member, resolves its destination
(`os.path.realpath(os.path.join(dest, member.name))`) and rejects any member
that would land outside `dest`. The behavior for legitimate archives is
unchanged.
tarfile's `filter='data'` (PEP 706) rejects such members natively and would
replace this manual check, but it is only generally available from Python
3.12.0 (backported to 3.11.4+), so we keep the explicit check while Spark still
supports Python 3.11, and leave a note to revisit once the minimum supported
Python is >= 3.12.
`install_spark()` runs at `pip install pyspark` time (when
`PYSPARK_HADOOP_VERSION`/`PYSPARK_HIVE_VERSION` is set) and downloads from an
Apache mirror or a user-supplied `PYSPARK_RELEASE_MIRROR`. Extracting archive
members without a containment check means a member path containing `..` would
be written outside the intended destination directory. Adding an explicit
resolved-path check is a straightforward robustness improvement that works on
all supported Python versions.
No.
New unit test `test_extract_tar` in
`python/pyspark/tests/test_install_spark.py` that (1) extracts a benign archive
with the top-level package directory stripped, and (2) asserts a member whose
path escapes the destination directory is rejected and nothing is written.
Generated-by: Claude Code (Opus 4.8)
Closes #57040 from
peter-toth/SPARK-57962-install-spark-path-traversal-guard.
Authored-by: Peter Toth <[email protected]>
Signed-off-by: Peter Toth <[email protected]>
(cherry picked from commit d61f2507c00b2d5578d583ccbb867295831a1b85)
Signed-off-by: Peter Toth <[email protected]>
---
python/pyspark/install.py | 37 +++++++++++++++++++++++++-----
python/pyspark/tests/test_install_spark.py | 37 ++++++++++++++++++++++++++++++
2 files changed, 68 insertions(+), 6 deletions(-)
diff --git a/python/pyspark/install.py b/python/pyspark/install.py
index 90b0150b0a8c..dd69f5a53f78 100644
--- a/python/pyspark/install.py
+++ b/python/pyspark/install.py
@@ -147,12 +147,7 @@ def install_spark(dest, spark_version, hadoop_version,
hive_version):
print("Installing to %s" % dest)
tar = tarfile.open(package_local_path, "r:gz")
- for member in tar.getmembers():
- if member.name == package_name:
- # Skip the root directory.
- continue
- member.name = os.path.relpath(member.name, package_name +
os.path.sep)
- tar.extract(member, dest)
+ _extract_tar(tar, package_name, dest)
return
except Exception:
print("Failed to download %s from %s:" % (pretty_pkg_name, url))
@@ -166,6 +161,36 @@ def install_spark(dest, spark_version, hadoop_version,
hive_version):
raise IOError("Unable to download %s." % pretty_pkg_name)
+def _extract_tar(tar: tarfile.TarFile, package_name: str, dest: str) -> None:
+ """
+ Extract the members of ``tar`` into ``dest``, stripping the top-level
+ ``package_name`` directory from each member path.
+
+ Guards against path traversal ("zip slip"): ``os.path.relpath`` does not
+ strip ``..`` segments, so a crafted member could otherwise resolve outside
+ ``dest``. Any member whose resolved destination escapes ``dest`` is
+ rejected instead of extracted.
+
+ Note: tarfile's ``filter="data"`` (PEP 706) rejects such members natively
and
+ would replace this manual check, but it is only generally available from
+ Python 3.12.0 (backported to 3.11.4+), so we keep the explicit check while
+ Spark still supports Python 3.11.
+ """
+ dest_root = os.path.realpath(dest)
+ for member in tar.getmembers():
+ if member.name == package_name:
+ # Skip the root directory.
+ continue
+ member.name = os.path.relpath(member.name, package_name + os.path.sep)
+ resolved = os.path.realpath(os.path.join(dest, member.name))
+ if resolved != dest_root and not resolved.startswith(dest_root +
os.sep):
+ raise ValueError(
+ "Archive member '%s' would extract outside of the destination "
+ "directory; refusing to extract." % member.name
+ )
+ tar.extract(member, dest)
+
+
def get_preferred_mirrors():
mirror_urls = []
for _ in range(3):
diff --git a/python/pyspark/tests/test_install_spark.py
b/python/pyspark/tests/test_install_spark.py
index e3d603f372a8..aebec186df8b 100644
--- a/python/pyspark/tests/test_install_spark.py
+++ b/python/pyspark/tests/test_install_spark.py
@@ -14,12 +14,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+import io
import os
+import tarfile
import tempfile
import unittest
from pyspark.install import (
install_spark,
+ _extract_tar,
DEFAULT_HADOOP,
DEFAULT_HIVE,
UNSUPPORTED_COMBINATIONS,
@@ -46,6 +49,40 @@ class SparkInstallationTestCase(unittest.TestCase):
self.assertTrue(os.path.exists("%s/bin/spark-submit" % tmp_dir))
self.assertTrue(os.path.exists("%s/RELEASE" % tmp_dir))
+ def test_extract_tar(self):
+ # A benign member is extracted with the top-level package directory
+ # stripped, while a member whose path escapes the destination
+ # directory (zip slip) is rejected rather than extracted.
+ package_name = "spark-4.1.1-bin-hadoop3"
+
+ def make_tar(path, member_name):
+ with tarfile.open(path, "w") as tar:
+ data = b"content"
+ info = tarfile.TarInfo(name=member_name)
+ info.size = len(data)
+ tar.addfile(info, io.BytesIO(data))
+
+ with tempfile.TemporaryDirectory(prefix="test_install_spark") as
tmp_dir:
+ # Benign archive extracts into dest with the package prefix
removed.
+ safe_tar = os.path.join(tmp_dir, "safe.tar")
+ make_tar(safe_tar, "%s/bin/spark-submit" % package_name)
+ safe_dest = os.path.join(tmp_dir, "safe_dest")
+ os.makedirs(safe_dest)
+ with tarfile.open(safe_tar, "r") as tar:
+ _extract_tar(tar, package_name, safe_dest)
+ self.assertTrue(os.path.exists(os.path.join(safe_dest, "bin",
"spark-submit")))
+
+ # Malicious archive member escaping dest is rejected, and nothing
+ # is written into the destination directory.
+ evil_tar = os.path.join(tmp_dir, "evil.tar")
+ make_tar(evil_tar, "%s/../../evil" % package_name)
+ evil_dest = os.path.join(tmp_dir, "evil_dest")
+ os.makedirs(evil_dest)
+ with tarfile.open(evil_tar, "r") as tar:
+ with self.assertRaisesRegex(ValueError, "outside of the
destination"):
+ _extract_tar(tar, package_name, evil_dest)
+ self.assertEqual([], os.listdir(evil_dest))
+
def test_package_name(self):
self.assertEqual(
"spark-3.0.0-bin-hadoop3.2", checked_package_name("spark-3.0.0",
"hadoop3.2", "hive2.3")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
