This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 7775dd66748b [SPARK-52233][SQL] Fix map_zip_with for Floating Point
Types
7775dd66748b is described below
commit 7775dd66748bd58d8553a633892e7e51358d9de6
Author: Mihailo Milosevic <mihailo.milose...@databricks.com>
AuthorDate: Mon May 26 21:29:02 2025 +0800
[SPARK-52233][SQL] Fix map_zip_with for Floating Point Types
### What changes were proposed in this pull request?
Fix to `map_zip_with` expression while handling floating point numbers.
### Why are the changes needed?
Previously we would run `getKeysWithIndexesFast` which would use faulty
scala.collections.mutable.LinkedHashMap implementation, which does not use
proper equality on keys for floating point numbers. All NaNs would be treated
in a different way. This PR aims to fix this behaviour, by using
`java.utils.LinkedHashMap` instead, which uses boxed `Type.equals()` instead of
primitive type equality `==`.
Example:
```
select map_zip_with(map(float('NaN'), 1), map(float('NaN'), 2), (k, v1, v2)
-> (v1, v2))
```
Output before:
```
{"NaN":{"v1":1,"v2":null},"NaN":{"v1":null,"v2":2}}
```
Output after:
```
{"NaN":{"v1":1,"v2":2}}
```
### Does this PR introduce _any_ user-facing change?
Yes, fixing the way expression works.
### How was this patch tested?
Added tests to golden files for both edge cases, `NaN` and `Infinity`.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #50967 from mihailom-db/FixMapZipWith.
Authored-by: Mihailo Milosevic <mihailo.milose...@databricks.com>
Signed-off-by: Wenchen Fan <wenc...@databricks.com>
---
.../expressions/higherOrderFunctions.scala | 34 ++++-
.../org/apache/spark/sql/internal/SQLConf.scala | 11 ++
.../typeCoercion/native/mapZipWith.sql.out | 138 +++++++++++++--------
.../inputs/typeCoercion/native/mapZipWith.sql | 16 ++-
.../results/typeCoercion/native/mapZipWith.sql.out | 28 ++++-
5 files changed, 170 insertions(+), 57 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
index 7635690a4605..0ce23b576e96 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
@@ -21,6 +21,7 @@ import java.util.Comparator
import java.util.concurrent.atomic.{AtomicInteger, AtomicReference}
import scala.collection.mutable
+import scala.jdk.CollectionConverters.MapHasAsScala
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, TypeCoercion,
UnresolvedException}
@@ -1109,8 +1110,10 @@ case class MapZipWith(left: Expression, right:
Expression, function: Expression)
*/
@transient private lazy val getKeysWithValueIndexes:
(ArrayData, ArrayData) => mutable.Iterable[(Any, Array[Option[Int]])] = {
- if (TypeUtils.typeWithProperEquals(keyType)) {
- getKeysWithIndexesFast
+ if (TypeUtils.typeWithProperEquals(keyType) &&
SQLConf.get.mapZipWithUsesJavaCollections) {
+ getKeysWithIndexesFastAsJava
+ } else if (TypeUtils.typeWithProperEquals(keyType)) {
+ getKeysWithIndexesFastUsingScala
} else {
getKeysWithIndexesBruteForce
}
@@ -1122,7 +1125,7 @@ case class MapZipWith(left: Expression, right:
Expression, function: Expression)
}
}
- private def getKeysWithIndexesFast(keys1: ArrayData, keys2: ArrayData) = {
+ private def getKeysWithIndexesFastUsingScala(keys1: ArrayData, keys2:
ArrayData) = {
val hashMap = new mutable.LinkedHashMap[Any, Array[Option[Int]]]
for ((z, array) <- Array((0, keys1), (1, keys2))) {
var i = 0
@@ -1144,6 +1147,31 @@ case class MapZipWith(left: Expression, right:
Expression, function: Expression)
hashMap
}
+ private def getKeysWithIndexesFastAsJava(
+ keys1: ArrayData,
+ keys2: ArrayData
+ ): scala.collection.mutable.LinkedHashMap[Any, Array[Option[Int]]] = {
+ val hashMap = new java.util.LinkedHashMap[Any, Array[Option[Int]]]
+ for ((z, array) <- Array((0, keys1), (1, keys2))) {
+ var i = 0
+ while (i < array.numElements()) {
+ val key = array.get(i, keyType)
+ Option(hashMap.get(key)) match {
+ case Some(indexes) =>
+ if (indexes(z).isEmpty) {
+ indexes(z) = Some(i)
+ }
+ case None =>
+ val indexes = Array[Option[Int]](None, None)
+ indexes(z) = Some(i)
+ hashMap.put(key, indexes)
+ }
+ i += 1
+ }
+ }
+ scala.collection.mutable.LinkedHashMap(hashMap.asScala.toSeq: _*)
+ }
+
private def getKeysWithIndexesBruteForce(keys1: ArrayData, keys2: ArrayData)
= {
val arrayBuffer = new mutable.ArrayBuffer[(Any, Array[Option[Int]])]
for ((z, array) <- Array((0, keys1), (1, keys2))) {
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 7335d525c2a2..3147b44f4dd0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1109,6 +1109,14 @@ object SQLConf {
.stringConf
.createOptional
+ val MAP_ZIP_WITH_USES_JAVA_COLLECTIONS =
+ buildConf("spark.sql.mapZipWithUsesJavaCollections")
+ .doc("When true, the `map_zip_with` function uses Java collections
instead of Scala " +
+ "collections. This is useful for avoiding NaN equality issues.")
+ .version("4.1.0")
+ .booleanConf
+ .createWithDefault(true)
+
val SUBEXPRESSION_ELIMINATION_ENABLED =
buildConf("spark.sql.subexpressionElimination.enabled")
.internal()
@@ -6383,6 +6391,9 @@ class SQLConf extends Serializable with Logging with
SqlApiConf {
*/
def hintErrorHandler: HintErrorHandler = HintErrorLogger
+ def mapZipWithUsesJavaCollections: Boolean =
+ getConf(MAP_ZIP_WITH_USES_JAVA_COLLECTIONS)
+
def subexpressionEliminationEnabled: Boolean =
getConf(SUBEXPRESSION_ELIMINATION_ENABLED)
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/typeCoercion/native/mapZipWith.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/typeCoercion/native/mapZipWith.sql.out
index 13cca708b8cb..a54ff050e46e 100644
---
a/sql/core/src/test/resources/sql-tests/analyzer-results/typeCoercion/native/mapZipWith.sql.out
+++
b/sql/core/src/test/resources/sql-tests/analyzer-results/typeCoercion/native/mapZipWith.sql.out
@@ -17,7 +17,11 @@ CREATE TEMPORARY VIEW various_maps AS SELECT * FROM VALUES (
map('2016-11-15 20:54:00.000', '2016-11-12 20:54:00.000'),
map('922337203685477897945456575809789456', 'text'),
map(array(1L, 2L), array(1L, 2L)), map(array(1, 2), array(1, 2)),
- map(struct(1S, 2L), struct(1S, 2L)), map(struct(1, 2), struct(1, 2))
+ map(struct(1S, 2L), struct(1S, 2L)), map(struct(1, 2), struct(1, 2)),
+ map(float('NaN'), 1),
+ map(float('NaN'), 2),
+ map(float('Infinity'), 1),
+ map(float('Infinity'), 2)
) AS various_maps(
boolean_map,
tinyint_map,
@@ -31,7 +35,9 @@ CREATE TEMPORARY VIEW various_maps AS SELECT * FROM VALUES (
timestamp_map,
string_map1, string_map2, string_map3, string_map4,
array_map1, array_map2,
- struct_map1, struct_map2
+ struct_map1, struct_map2,
+ float_nan_map1, float_nan_map2,
+ float_infinity_map1, float_infinity_map2
)
-- !query analysis
CreateViewCommand `various_maps`, SELECT * FROM VALUES (
@@ -51,7 +57,11 @@ CreateViewCommand `various_maps`, SELECT * FROM VALUES (
map('2016-11-15 20:54:00.000', '2016-11-12 20:54:00.000'),
map('922337203685477897945456575809789456', 'text'),
map(array(1L, 2L), array(1L, 2L)), map(array(1, 2), array(1, 2)),
- map(struct(1S, 2L), struct(1S, 2L)), map(struct(1, 2), struct(1, 2))
+ map(struct(1S, 2L), struct(1S, 2L)), map(struct(1, 2), struct(1, 2)),
+ map(float('NaN'), 1),
+ map(float('NaN'), 2),
+ map(float('Infinity'), 1),
+ map(float('Infinity'), 2)
) AS various_maps(
boolean_map,
tinyint_map,
@@ -65,11 +75,13 @@ CreateViewCommand `various_maps`, SELECT * FROM VALUES (
timestamp_map,
string_map1, string_map2, string_map3, string_map4,
array_map1, array_map2,
- struct_map1, struct_map2
+ struct_map1, struct_map2,
+ float_nan_map1, float_nan_map2,
+ float_infinity_map1, float_infinity_map2
), false, false, LocalTempView, UNSUPPORTED, true
- +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x]
+ +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x,
float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
+- SubqueryAlias various_maps
- +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x]
+ +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
-- !query
@@ -78,11 +90,11 @@ FROM various_maps
-- !query analysis
Project [map_zip_with(cast(tinyint_map#x as map<smallint,tinyint>),
smallint_map#x, lambdafunction(struct(k, lambda k#x, v1, lambda v1#x, v2,
lambda v2#x), lambda k#x, lambda v1#x, lambda v2#x, false)) AS m#x]
+- SubqueryAlias various_maps
- +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x])
- +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
- +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x]
+ +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x])
+ +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
+ +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x,
float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
+- SubqueryAlias various_maps
- +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x]
+ +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
-- !query
@@ -91,11 +103,11 @@ FROM various_maps
-- !query analysis
Project [map_zip_with(cast(smallint_map#x as map<int,smallint>), int_map#x,
lambdafunction(struct(k, lambda k#x, v1, lambda v1#x, v2, lambda v2#x), lambda
k#x, lambda v1#x, lambda v2#x, false)) AS m#x]
+- SubqueryAlias various_maps
- +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x])
- +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
- +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x]
+ +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x])
+ +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
+ +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x,
float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
+- SubqueryAlias various_maps
- +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x]
+ +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
-- !query
@@ -104,11 +116,11 @@ FROM various_maps
-- !query analysis
Project [map_zip_with(cast(int_map#x as map<bigint,int>), bigint_map#x,
lambdafunction(struct(k, lambda k#xL, v1, lambda v1#x, v2, lambda v2#xL),
lambda k#xL, lambda v1#x, lambda v2#xL, false)) AS m#x]
+- SubqueryAlias various_maps
- +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x])
- +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
- +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x]
+ +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x])
+ +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
+ +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x,
float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
+- SubqueryAlias various_maps
- +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x]
+ +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
-- !query
@@ -117,11 +129,11 @@ FROM various_maps
-- !query analysis
Project [map_zip_with(double_map#x, cast(float_map#x as map<double,float>),
lambdafunction(struct(k, lambda k#x, v1, lambda v1#x, v2, lambda v2#x), lambda
k#x, lambda v1#x, lambda v2#x, false)) AS m#x]
+- SubqueryAlias various_maps
- +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x])
- +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
- +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x]
+ +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x])
+ +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
+ +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x,
float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
+- SubqueryAlias various_maps
- +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x]
+ +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
-- !query
@@ -130,11 +142,11 @@ FROM various_maps
-- !query analysis
Project [map_zip_with(cast(decimal_map1#x as
map<decimal(38,2),decimal(36,0)>), cast(decimal_map2#x as
map<decimal(38,2),decimal(36,35)>), lambdafunction(struct(k, lambda k#x, v1,
lambda v1#x, v2, lambda v2#x), lambda k#x, lambda v1#x, lambda v2#x, false)) AS
m#x]
+- SubqueryAlias various_maps
- +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x])
- +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
- +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x]
+ +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x])
+ +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
+ +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x,
float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
+- SubqueryAlias various_maps
- +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x]
+ +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
-- !query
@@ -143,11 +155,11 @@ FROM various_maps
-- !query analysis
Project [map_zip_with(decimal_map1#x, cast(int_map#x as
map<decimal(36,0),int>), lambdafunction(struct(k, lambda k#x, v1, lambda v1#x,
v2, lambda v2#x), lambda k#x, lambda v1#x, lambda v2#x, false)) AS m#x]
+- SubqueryAlias various_maps
- +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x])
- +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
- +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x]
+ +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x])
+ +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
+ +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x,
float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
+- SubqueryAlias various_maps
- +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x]
+ +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
-- !query
@@ -156,11 +168,11 @@ FROM various_maps
-- !query analysis
Project [map_zip_with(cast(decimal_map1#x as map<double,decimal(36,0)>),
double_map#x, lambdafunction(struct(k, lambda k#x, v1, lambda v1#x, v2, lambda
v2#x), lambda k#x, lambda v1#x, lambda v2#x, false)) AS m#x]
+- SubqueryAlias various_maps
- +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x])
- +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
- +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x]
+ +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x])
+ +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
+ +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x,
float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
+- SubqueryAlias various_maps
- +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x]
+ +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
-- !query
@@ -169,11 +181,11 @@ FROM various_maps
-- !query analysis
Project [map_zip_with(cast(decimal_map2#x as
map<decimal(38,28),decimal(36,35)>), cast(int_map#x as
map<decimal(38,28),int>), lambdafunction(struct(k, lambda k#x, v1, lambda v1#x,
v2, lambda v2#x), lambda k#x, lambda v1#x, lambda v2#x, false)) AS m#x]
+- SubqueryAlias various_maps
- +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x])
- +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
- +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x]
+ +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x])
+ +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
+ +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x,
float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
+- SubqueryAlias various_maps
- +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x]
+ +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
-- !query
@@ -182,11 +194,11 @@ FROM various_maps
-- !query analysis
Project [map_zip_with(cast(decimal_map2#x as map<double,decimal(36,35)>),
double_map#x, lambdafunction(struct(k, lambda k#x, v1, lambda v1#x, v2, lambda
v2#x), lambda k#x, lambda v1#x, lambda v2#x, false)) AS m#x]
+- SubqueryAlias various_maps
- +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x])
- +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
- +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x]
+ +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x])
+ +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
+ +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x,
float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
+- SubqueryAlias various_maps
- +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x]
+ +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
-- !query
@@ -291,11 +303,11 @@ FROM various_maps
-- !query analysis
Project [map_zip_with(array_map1#x, cast(array_map2#x as
map<array<bigint>,array<int>>), lambdafunction(struct(k, lambda k#x, v1, lambda
v1#x, v2, lambda v2#x), lambda k#x, lambda v1#x, lambda v2#x, false)) AS m#x]
+- SubqueryAlias various_maps
- +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x])
- +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
- +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x]
+ +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x])
+ +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
+ +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x,
float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
+- SubqueryAlias various_maps
- +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x]
+ +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
-- !query
@@ -304,8 +316,34 @@ FROM various_maps
-- !query analysis
Project [map_zip_with(cast(struct_map1#x as
map<struct<col1:int,col2:bigint>,struct<col1:smallint,col2:bigint>>),
cast(struct_map2#x as
map<struct<col1:int,col2:bigint>,struct<col1:int,col2:int>>),
lambdafunction(struct(k, lambda k#x, v1, lambda v1#x, v2, lambda v2#x), lambda
k#x, lambda v1#x, lambda v2#x, false)) AS m#x]
+- SubqueryAlias various_maps
- +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x])
- +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
- +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x]
+ +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x])
+ +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
+ +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x,
float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
+- SubqueryAlias various_maps
- +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x]
+ +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
+
+
+-- !query
+SELECT map_zip_with(float_nan_map1, float_nan_map2, (k, v1, v2) -> (v1, v2)) m
+FROM various_maps
+-- !query analysis
+Project [map_zip_with(float_nan_map1#x, float_nan_map2#x,
lambdafunction(named_struct(v1, lambda v1#x, v2, lambda v2#x), lambda k#x,
lambda v1#x, lambda v2#x, false)) AS m#x]
++- SubqueryAlias various_maps
+ +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x])
+ +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
+ +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x,
float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
+ +- SubqueryAlias various_maps
+ +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
+
+
+-- !query
+SELECT map_zip_with(float_infinity_map1, float_infinity_map2, (k, v1, v2) ->
(v1, v2)) m
+FROM various_maps
+-- !query analysis
+Project [map_zip_with(float_infinity_map1#x, float_infinity_map2#x,
lambdafunction(named_struct(v1, lambda v1#x, v2, lambda v2#x), lambda k#x,
lambda v1#x, lambda v2#x, false)) AS m#x]
++- SubqueryAlias various_maps
+ +- View (`various_maps`, [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x])
+ +- Project [cast(boolean_map#x as map<boolean,boolean>) AS
boolean_map#x, cast(tinyint_map#x as map<tinyint,tinyint>) AS tinyint_map#x,
cast(smallint_map#x as map<smallint,smallint>) AS smallint_map#x,
cast(int_map#x as map<int,int>) AS int_map#x, cast(bigint_map#x as
map<bigint,bigint>) AS bigint_map#x, cast(decimal_map1#x as
map<decimal(36,0),decimal(36,0)>) AS decimal_map1#x, cast(decimal_map2#x as
map<decimal(36,35),decimal(36,35)>) AS decimal_map2#x, cast(double_map#x as
map<d [...]
+ +- Project [boolean_map#x, tinyint_map#x, smallint_map#x, int_map#x,
bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x, float_map#x,
date_map#x, timestamp_map#x, string_map1#x, string_map2#x, string_map3#x,
string_map4#x, array_map1#x, array_map2#x, struct_map1#x, struct_map2#x,
float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
+ +- SubqueryAlias various_maps
+ +- LocalRelation [boolean_map#x, tinyint_map#x, smallint_map#x,
int_map#x, bigint_map#x, decimal_map1#x, decimal_map2#x, double_map#x,
float_map#x, date_map#x, timestamp_map#x, string_map1#x, string_map2#x,
string_map3#x, string_map4#x, array_map1#x, array_map2#x, struct_map1#x,
struct_map2#x, float_nan_map1#x, float_nan_map2#x, float_infinity_map1#x,
float_infinity_map2#x]
diff --git
a/sql/core/src/test/resources/sql-tests/inputs/typeCoercion/native/mapZipWith.sql
b/sql/core/src/test/resources/sql-tests/inputs/typeCoercion/native/mapZipWith.sql
index 1727ee725db2..a0c0e10606f9 100644
---
a/sql/core/src/test/resources/sql-tests/inputs/typeCoercion/native/mapZipWith.sql
+++
b/sql/core/src/test/resources/sql-tests/inputs/typeCoercion/native/mapZipWith.sql
@@ -15,7 +15,11 @@ CREATE TEMPORARY VIEW various_maps AS SELECT * FROM VALUES (
map('2016-11-15 20:54:00.000', '2016-11-12 20:54:00.000'),
map('922337203685477897945456575809789456', 'text'),
map(array(1L, 2L), array(1L, 2L)), map(array(1, 2), array(1, 2)),
- map(struct(1S, 2L), struct(1S, 2L)), map(struct(1, 2), struct(1, 2))
+ map(struct(1S, 2L), struct(1S, 2L)), map(struct(1, 2), struct(1, 2)),
+ map(float('NaN'), 1),
+ map(float('NaN'), 2),
+ map(float('Infinity'), 1),
+ map(float('Infinity'), 2)
) AS various_maps(
boolean_map,
tinyint_map,
@@ -29,7 +33,9 @@ CREATE TEMPORARY VIEW various_maps AS SELECT * FROM VALUES (
timestamp_map,
string_map1, string_map2, string_map3, string_map4,
array_map1, array_map2,
- struct_map1, struct_map2
+ struct_map1, struct_map2,
+ float_nan_map1, float_nan_map2,
+ float_infinity_map1, float_infinity_map2
);
SELECT map_zip_with(tinyint_map, smallint_map, (k, v1, v2) -> struct(k, v1,
v2)) m
@@ -76,3 +82,9 @@ FROM various_maps;
SELECT map_zip_with(struct_map1, struct_map2, (k, v1, v2) -> struct(k, v1,
v2)) m
FROM various_maps;
+
+SELECT map_zip_with(float_nan_map1, float_nan_map2, (k, v1, v2) -> (v1, v2)) m
+FROM various_maps;
+
+SELECT map_zip_with(float_infinity_map1, float_infinity_map2, (k, v1, v2) ->
(v1, v2)) m
+FROM various_maps;
diff --git
a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapZipWith.sql.out
b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapZipWith.sql.out
index a0bd111f6ba6..4359853e6f5c 100644
---
a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapZipWith.sql.out
+++
b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapZipWith.sql.out
@@ -17,7 +17,11 @@ CREATE TEMPORARY VIEW various_maps AS SELECT * FROM VALUES (
map('2016-11-15 20:54:00.000', '2016-11-12 20:54:00.000'),
map('922337203685477897945456575809789456', 'text'),
map(array(1L, 2L), array(1L, 2L)), map(array(1, 2), array(1, 2)),
- map(struct(1S, 2L), struct(1S, 2L)), map(struct(1, 2), struct(1, 2))
+ map(struct(1S, 2L), struct(1S, 2L)), map(struct(1, 2), struct(1, 2)),
+ map(float('NaN'), 1),
+ map(float('NaN'), 2),
+ map(float('Infinity'), 1),
+ map(float('Infinity'), 2)
) AS various_maps(
boolean_map,
tinyint_map,
@@ -31,7 +35,9 @@ CREATE TEMPORARY VIEW various_maps AS SELECT * FROM VALUES (
timestamp_map,
string_map1, string_map2, string_map3, string_map4,
array_map1, array_map2,
- struct_map1, struct_map2
+ struct_map1, struct_map2,
+ float_nan_map1, float_nan_map2,
+ float_infinity_map1, float_infinity_map2
)
-- !query schema
struct<>
@@ -240,3 +246,21 @@ FROM various_maps
struct<m:map<struct<col1:int,col2:bigint>,struct<k:struct<col1:int,col2:bigint>,v1:struct<col1:smallint,col2:bigint>,v2:struct<col1:int,col2:int>>>>
-- !query output
{{"col1":1,"col2":2}:{"k":{"col1":1,"col2":2},"v1":{"col1":1,"col2":2},"v2":{"col1":1,"col2":2}}}
+
+
+-- !query
+SELECT map_zip_with(float_nan_map1, float_nan_map2, (k, v1, v2) -> (v1, v2)) m
+FROM various_maps
+-- !query schema
+struct<m:map<float,struct<v1:int,v2:int>>>
+-- !query output
+{NaN:{"v1":1,"v2":2}}
+
+
+-- !query
+SELECT map_zip_with(float_infinity_map1, float_infinity_map2, (k, v1, v2) ->
(v1, v2)) m
+FROM various_maps
+-- !query schema
+struct<m:map<float,struct<v1:int,v2:int>>>
+-- !query output
+{Infinity:{"v1":1,"v2":2}}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
