This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 0567dcdf0c3a [SPARK-54885][PYTHON] Remove unreachable code after
upgrading pyarrow minimum version to 18.0.0
0567dcdf0c3a is described below
commit 0567dcdf0c3a04a514d995e015a136d49c1e7754
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Sat Jan 3 10:44:46 2026 +0800
[SPARK-54885][PYTHON] Remove unreachable code after upgrading pyarrow
minimum version to 18.0.0
### What changes were proposed in this pull request?
Remove unreachable code after upgrading pyarrow minimum version to 18.0.0
### Why are the changes needed?
Code clean up
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
CI
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #53663 from zhengruifeng/del_old_path.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
python/pyspark/sql/pandas/types.py | 5 +----
python/pyspark/sql/tests/arrow/test_arrow.py | 14 +-------------
2 files changed, 2 insertions(+), 17 deletions(-)
diff --git a/python/pyspark/sql/pandas/types.py
b/python/pyspark/sql/pandas/types.py
index 2e23d304d8cb..a9f45cc45020 100644
--- a/python/pyspark/sql/pandas/types.py
+++ b/python/pyspark/sql/pandas/types.py
@@ -26,7 +26,6 @@ from decimal import Decimal
from typing import Any, Callable, Iterable, List, Optional, Union,
TYPE_CHECKING
from pyspark.errors import PySparkTypeError, UnsupportedOperationException,
PySparkValueError
-from pyspark.loose_version import LooseVersion
from pyspark.sql.types import (
cast,
BooleanType,
@@ -539,9 +538,7 @@ def _check_arrow_array_timestamps_localize(
a.items, mt.valueType, truncate, timezone
),
}
- # SPARK-48302: PyArrow added support for mask argument to
pa.MapArray.from_arrays in
- # version 17.0.0
- if a.null_count and LooseVersion(pa.__version__) >=
LooseVersion("17.0.0"):
+ if a.null_count:
params["mask"] = a.is_null()
return pa.MapArray.from_arrays(**params)
diff --git a/python/pyspark/sql/tests/arrow/test_arrow.py
b/python/pyspark/sql/tests/arrow/test_arrow.py
index e410f7df711b..db5c7d18a0fc 100644
--- a/python/pyspark/sql/tests/arrow/test_arrow.py
+++ b/python/pyspark/sql/tests/arrow/test_arrow.py
@@ -59,7 +59,6 @@ from pyspark.testing.sqlutils import (
pyarrow_requirement_message,
)
from pyspark.errors import ArithmeticException, PySparkTypeError,
UnsupportedOperationException
-from pyspark.loose_version import LooseVersion
from pyspark.util import is_remote_only
if have_pandas:
@@ -1658,18 +1657,7 @@ class ArrowTestsMixin:
)
df = self.spark.createDataFrame(origin)
t = df.toArrow()
-
- # SPARK-48302: PyArrow versions before 17.0.0 replaced nulls with
empty lists when
- # reconstructing MapArray columns to localize timestamps
- if LooseVersion(pa.__version__) >= LooseVersion("17.0.0"):
- expected = origin
- else:
- expected = pa.table(
- [[dict(ts=datetime.datetime(2023, 1, 1, 8, 0, 0)), []]],
- schema=origin_schema,
- )
-
- self.assertTrue(t.equals(expected))
+ self.assertTrue(t.equals(origin))
def test_createDataFrame_udt(self):
for arrow_enabled in [True, False]:
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
