This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new e284b2ce1ec2 [SPARK-50525][SQL][TESTS][FOLLOWUP] Fix
`DataFrameSuite.repartition by MapType` test assumption
e284b2ce1ec2 is described below
commit e284b2ce1ec2f622c62d233b1837e37196fa2e33
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sun Jan 12 18:29:40 2025 -0800
[SPARK-50525][SQL][TESTS][FOLLOWUP] Fix `DataFrameSuite.repartition by
MapType` test assumption
### What changes were proposed in this pull request?
This is a follow-up to recover the NON-ANSI mode CI failure by adding a
test assumption clearly.
- #49144
### Why are the changes needed?
**BEFORE**
```
$ SPARK_ANSI_SQL_MODE=false build/sbt "sql/testOnly *.DataFrameSuite -- -z
MapType"
[info] *** 1 TEST FAILED ***
[error] Failed tests:
[error] org.apache.spark.sql.DataFrameSuite
```
**AFTER**
```
$ SPARK_ANSI_SQL_MODE=false build/sbt "sql/testOnly *.DataFrameSuite -- -z
MapType"
[info] All tests passed.
```
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Manually test with `SPARK_ANSI_SQL_MODE=false`.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #49457 from dongjoon-hyun/SPARK-50525.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../org/apache/spark/sql/DataFrameSuite.scala | 32 ++++++++++++----------
1 file changed, 17 insertions(+), 15 deletions(-)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 317a88edf8e9..71d55b007aa1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -430,21 +430,23 @@ class DataFrameSuite extends QueryTest
test("repartition by MapType") {
Seq("int", "long", "float", "double", "decimal(10, 2)", "string",
"varchar(6)").foreach { dt =>
- val df = spark.range(20)
- .withColumn("c1",
- when(col("id") % 3 === 1, typedLit(Map(1 -> 1)))
- .when(col("id") % 3 === 2, typedLit(Map(1 -> 1, 2 -> 2)))
- .otherwise(typedLit(Map(2 -> 2, 1 -> 1))).cast(s"map<$dt, $dt>"))
- .withColumn("c2", typedLit(Map(1 -> null)).cast(s"map<$dt, $dt>"))
- .withColumn("c3", lit(null).cast(s"map<$dt, $dt>"))
-
- assertPartitionNumber(df.repartition(4, col("c1")), 2)
- assertPartitionNumber(df.repartition(4, col("c2")), 1)
- assertPartitionNumber(df.repartition(4, col("c3")), 1)
- assertPartitionNumber(df.repartition(4, col("c1"), col("c2")), 2)
- assertPartitionNumber(df.repartition(4, col("c1"), col("c3")), 2)
- assertPartitionNumber(df.repartition(4, col("c1"), col("c2"),
col("c3")), 2)
- assertPartitionNumber(df.repartition(4, col("c2"), col("c3")), 2)
+ withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
+ val df = spark.range(20)
+ .withColumn("c1",
+ when(col("id") % 3 === 1, typedLit(Map(1 -> 1)))
+ .when(col("id") % 3 === 2, typedLit(Map(1 -> 1, 2 -> 2)))
+ .otherwise(typedLit(Map(2 -> 2, 1 -> 1))).cast(s"map<$dt, $dt>"))
+ .withColumn("c2", typedLit(Map(1 -> null)).cast(s"map<$dt, $dt>"))
+ .withColumn("c3", lit(null).cast(s"map<$dt, $dt>"))
+
+ assertPartitionNumber(df.repartition(4, col("c1")), 2)
+ assertPartitionNumber(df.repartition(4, col("c2")), 1)
+ assertPartitionNumber(df.repartition(4, col("c3")), 1)
+ assertPartitionNumber(df.repartition(4, col("c1"), col("c2")), 2)
+ assertPartitionNumber(df.repartition(4, col("c1"), col("c3")), 2)
+ assertPartitionNumber(df.repartition(4, col("c1"), col("c2"),
col("c3")), 2)
+ assertPartitionNumber(df.repartition(4, col("c2"), col("c3")), 2)
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
