This is an automated email from the ASF dual-hosted git repository.
beliefer pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new d8dbb6674d0 [SPARK-45514][SQL][MLLIB] Replace
`scala.runtime.Tuple3Zipped` to `scala.collection.LazyZip3`
d8dbb6674d0 is described below
commit d8dbb6674d0f3087f072dc08263f86e02a263693
Author: Jiaan Geng <[email protected]>
AuthorDate: Mon Oct 16 10:00:26 2023 +0800
[SPARK-45514][SQL][MLLIB] Replace `scala.runtime.Tuple3Zipped` to
`scala.collection.LazyZip3`
### What changes were proposed in this pull request?
Since scala 2.13.0, `scala.runtime.Tuple3Zipped` marked as deprecated and
`scala.collection.LazyZip3` recommended.
### Why are the changes needed?
Replace `scala.runtime.Tuple3Zipped` to `scala.collection.LazyZip3`.
### Does this PR introduce _any_ user-facing change?
'No'.
### How was this patch tested?
Exists test cases.
### Was this patch authored or co-authored using generative AI tooling?
'No'.
Closes #43363 from beliefer/SPARK-45514.
Authored-by: Jiaan Geng <[email protected]>
Signed-off-by: Jiaan Geng <[email protected]>
---
.../spark/mllib/feature/ElementwiseProductSuite.scala | 2 +-
.../org/apache/spark/mllib/feature/NormalizerSuite.scala | 6 +++---
.../apache/spark/mllib/feature/StandardScalerSuite.scala | 16 ++++++++--------
.../spark/sql/execution/arrow/ArrowConvertersSuite.scala | 2 +-
.../spark/sql/hive/execution/HiveComparisonTest.scala | 2 +-
5 files changed, 14 insertions(+), 14 deletions(-)
diff --git
a/mllib/src/test/scala/org/apache/spark/mllib/feature/ElementwiseProductSuite.scala
b/mllib/src/test/scala/org/apache/spark/mllib/feature/ElementwiseProductSuite.scala
index 9eca2d682d6..a764ac8551f 100644
---
a/mllib/src/test/scala/org/apache/spark/mllib/feature/ElementwiseProductSuite.scala
+++
b/mllib/src/test/scala/org/apache/spark/mllib/feature/ElementwiseProductSuite.scala
@@ -48,7 +48,7 @@ class ElementwiseProductSuite extends SparkFunSuite with
MLlibTestSparkContext {
val data2 = sparseData.map(transformer.transform)
val data2RDD = transformer.transform(dataRDD)
- assert((sparseData, data2, data2RDD.collect()).zipped.forall {
+ assert(sparseData.lazyZip(data2).lazyZip(data2RDD.collect()).forall {
case (v1: DenseVector, v2: DenseVector, v3: DenseVector) => true
case (v1: SparseVector, v2: SparseVector, v3: SparseVector) => true
case _ => false
diff --git
a/mllib/src/test/scala/org/apache/spark/mllib/feature/NormalizerSuite.scala
b/mllib/src/test/scala/org/apache/spark/mllib/feature/NormalizerSuite.scala
index 71ce26360b8..77eade64db9 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/feature/NormalizerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/feature/NormalizerSuite.scala
@@ -43,7 +43,7 @@ class NormalizerSuite extends SparkFunSuite with
MLlibTestSparkContext {
val data1 = data.map(l1Normalizer.transform)
val data1RDD = l1Normalizer.transform(dataRDD)
- assert((data, data1, data1RDD.collect()).zipped.forall {
+ assert(data.lazyZip(data1).lazyZip(data1RDD.collect()).forall {
case (v1: DenseVector, v2: DenseVector, v3: DenseVector) => true
case (v1: SparseVector, v2: SparseVector, v3: SparseVector) => true
case _ => false
@@ -70,7 +70,7 @@ class NormalizerSuite extends SparkFunSuite with
MLlibTestSparkContext {
val data2 = data.map(l2Normalizer.transform)
val data2RDD = l2Normalizer.transform(dataRDD)
- assert((data, data2, data2RDD.collect()).zipped.forall {
+ assert(data.lazyZip(data2).lazyZip(data2RDD.collect()).forall {
case (v1: DenseVector, v2: DenseVector, v3: DenseVector) => true
case (v1: SparseVector, v2: SparseVector, v3: SparseVector) => true
case _ => false
@@ -97,7 +97,7 @@ class NormalizerSuite extends SparkFunSuite with
MLlibTestSparkContext {
val dataInf = data.map(lInfNormalizer.transform)
val dataInfRDD = lInfNormalizer.transform(dataRDD)
- assert((data, dataInf, dataInfRDD.collect()).zipped.forall {
+ assert(data.lazyZip(dataInf).lazyZip(dataInfRDD.collect()).forall {
case (v1: DenseVector, v2: DenseVector, v3: DenseVector) => true
case (v1: SparseVector, v2: SparseVector, v3: SparseVector) => true
case _ => false
diff --git
a/mllib/src/test/scala/org/apache/spark/mllib/feature/StandardScalerSuite.scala
b/mllib/src/test/scala/org/apache/spark/mllib/feature/StandardScalerSuite.scala
index a2c72de4231..2fad944344b 100644
---
a/mllib/src/test/scala/org/apache/spark/mllib/feature/StandardScalerSuite.scala
+++
b/mllib/src/test/scala/org/apache/spark/mllib/feature/StandardScalerSuite.scala
@@ -87,19 +87,19 @@ class StandardScalerSuite extends SparkFunSuite with
MLlibTestSparkContext {
val summary2 = computeSummary(data2RDD)
val summary3 = computeSummary(data3RDD)
- assert((denseData, data1, data1RDD.collect()).zipped.forall {
+ assert(denseData.lazyZip(data1).lazyZip(data1RDD.collect()).forall {
case (v1: DenseVector, v2: DenseVector, v3: DenseVector) => true
case (v1: SparseVector, v2: SparseVector, v3: SparseVector) => true
case _ => false
}, "The vector type should be preserved after standardization.")
- assert((denseData, data2, data2RDD.collect()).zipped.forall {
+ assert(denseData.lazyZip(data2).lazyZip(data2RDD.collect()).forall {
case (v1: DenseVector, v2: DenseVector, v3: DenseVector) => true
case (v1: SparseVector, v2: SparseVector, v3: SparseVector) => true
case _ => false
}, "The vector type should be preserved after standardization.")
- assert((denseData, data3, data3RDD.collect()).zipped.forall {
+ assert(denseData.lazyZip(data3).lazyZip(data3RDD.collect()).forall {
case (v1: DenseVector, v2: DenseVector, v3: DenseVector) => true
case (v1: SparseVector, v2: SparseVector, v3: SparseVector) => true
case _ => false
@@ -151,19 +151,19 @@ class StandardScalerSuite extends SparkFunSuite with
MLlibTestSparkContext {
val summary2 = computeSummary(data2RDD)
val summary3 = computeSummary(data3RDD)
- assert((denseData, data1, data1RDD.collect()).zipped.forall {
+ assert(denseData.lazyZip(data1).lazyZip(data1RDD.collect()).forall {
case (v1: DenseVector, v2: DenseVector, v3: DenseVector) => true
case (v1: SparseVector, v2: SparseVector, v3: SparseVector) => true
case _ => false
}, "The vector type should be preserved after standardization.")
- assert((denseData, data2, data2RDD.collect()).zipped.forall {
+ assert(denseData.lazyZip(data2).lazyZip(data2RDD.collect()).forall {
case (v1: DenseVector, v2: DenseVector, v3: DenseVector) => true
case (v1: SparseVector, v2: SparseVector, v3: SparseVector) => true
case _ => false
}, "The vector type should be preserved after standardization.")
- assert((denseData, data3, data3RDD.collect()).zipped.forall {
+ assert(denseData.lazyZip(data3).lazyZip(data3RDD.collect()).forall {
case (v1: DenseVector, v2: DenseVector, v3: DenseVector) => true
case (v1: SparseVector, v2: SparseVector, v3: SparseVector) => true
case _ => false
@@ -219,7 +219,7 @@ class StandardScalerSuite extends SparkFunSuite with
MLlibTestSparkContext {
val summary2 = computeSummary(data2RDD)
val summary3 = computeSummary(data3RDD)
- assert((sparseData, data2, data2RDD.collect()).zipped.forall {
+ assert(sparseData.lazyZip(data2).lazyZip(data2RDD.collect()).forall {
case (v1: DenseVector, v2: DenseVector, v3: DenseVector) => true
case (v1: SparseVector, v2: SparseVector, v3: SparseVector) => true
case _ => false
@@ -268,7 +268,7 @@ class StandardScalerSuite extends SparkFunSuite with
MLlibTestSparkContext {
val summary2 = computeSummary(data2RDD)
val summary3 = computeSummary(data3RDD)
- assert((sparseData, data2, data2RDD.collect()).zipped.forall {
+ assert(sparseData.lazyZip(data2).lazyZip(data2RDD.collect()).forall {
case (v1: DenseVector, v2: DenseVector, v3: DenseVector) => true
case (v1: SparseVector, v2: SparseVector, v3: SparseVector) => true
case _ => false
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala
index 4a947f18532..4b95b1f634b 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala
@@ -550,7 +550,7 @@ class ArrowConvertersSuite extends SharedSparkSession {
val upperCase = Seq("A", "B", "C")
val lowerCase = Seq("a", "b", "c")
val nullStr = Seq("ab", "CDE", null)
- val df = (upperCase, lowerCase, nullStr).zipped.toList
+ val df = upperCase.lazyZip(lowerCase).lazyZip(nullStr).toList
.toDF("upper_case", "lower_case", "null_str")
collectAndValidate(df, json, "stringData.json")
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
index 67b51f93478..f0feccb4f49 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
@@ -364,7 +364,7 @@ abstract class HiveComparisonTest extends SparkFunSuite
with BeforeAndAfterAll {
}
}
- (queryList, hiveResults, catalystResults).zipped.foreach {
+ queryList.lazyZip(hiveResults).lazyZip(catalystResults).foreach {
case (query, hive, (hiveQuery, catalyst)) =>
// Check that the results match unless its an EXPLAIN query.
val preparedHive = prepareAnswer(hiveQuery, hive)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
