This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 80b4486a6fdc [SPARK-54769][PYTHON] Remove dead code in conversion.py
80b4486a6fdc is described below
commit 80b4486a6fdcede57053fc2d8b9189f2bf066884
Author: Tian Gao <[email protected]>
AuthorDate: Fri Dec 19 17:24:42 2025 +0900
[SPARK-54769][PYTHON] Remove dead code in conversion.py
### What changes were proposed in this pull request?
Dead code is removed and replaced with assertions.
### Why are the changes needed?
There are two kinds of dead code in this PR:
1. The else case. We already confirmed that we need a converter for this
`dataType`, `else` case should not ever happen, unless we made a mistake in our
code.
2. If `ArrayType` has an `elementType` that does not need a converter, the
`ArrayType` itself should not need a converter and we should never ask for one.
assertion has clear semantics that this should never happen, but also
provides a reasonable and reproducible fallback behavior when unexpected
happens. We can catch our code failure faster.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
CI
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #53536 from gaogaotiantian/remove-dead-code.
Authored-by: Tian Gao <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
python/pyspark/sql/conversion.py | 39 +++++++++++++--------------------------
1 file changed, 13 insertions(+), 26 deletions(-)
diff --git a/python/pyspark/sql/conversion.py b/python/pyspark/sql/conversion.py
index 359dd32cc889..e035f259c836 100644
--- a/python/pyspark/sql/conversion.py
+++ b/python/pyspark/sql/conversion.py
@@ -436,11 +436,8 @@ class LocalDataToArrowConversion:
return value
return convert_other
- else:
- if none_on_identity:
- return None
- else:
- return lambda value: value
+ else: # pragma: no cover
+ assert False, f"Need converter for {dataType} but failed to find
one."
@staticmethod
def convert(data: Sequence[Any], schema: StructType, use_large_var_types:
bool) -> "pa.Table":
@@ -613,23 +610,16 @@ class ArrowTableToRowsConversion:
dataType.elementType, none_on_identity=True,
binary_as_bytes=binary_as_bytes
)
- if element_conv is None:
-
- def convert_array(value: Any) -> Any:
- if value is None:
- return None
- else:
- assert isinstance(value, list)
- return value
-
- else:
+ assert (
+ element_conv is not None
+ ), f"_need_converter() returned True for ArrayType of
{dataType.elementType}"
- def convert_array(value: Any) -> Any:
- if value is None:
- return None
- else:
- assert isinstance(value, list)
- return [element_conv(v) for v in value]
+ def convert_array(value: Any) -> Any:
+ if value is None:
+ return None
+ else:
+ assert isinstance(value, list)
+ return [element_conv(v) for v in value]
return convert_array
@@ -793,11 +783,8 @@ class ArrowTableToRowsConversion:
return convert_geometry
- else:
- if none_on_identity:
- return None
- else:
- return lambda value: value
+ else: # pragma: no cover
+ assert False, f"Need converter for {dataType} but failed to find
one."
@overload
@staticmethod
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
