Repository: spark Updated Branches: refs/heads/branch-2.0 399597b04 -> 8b99e204a
[SPARK-18189][SQL][FOLLOWUP] Move test from ReplSuite to prevent java.lang.ClassCircularityError ## What changes were proposed in this pull request? Move the test which is causing java.lang.ClassCircularityError from ReplSuite to DatasetSuite. ## How was this patch tested? > build/mvn -DskipTests -Phadoop-2.3 -Pyarn -Phive -Phive-thriftserver > -Pkinesis-asl -Pmesos clean package > build/mvn -Dtest=none -DwildcardSuites=org.apache.spark.repl.ReplSuite test Author: Ergin Seyfe <[email protected]> Closes #15774 from seyfe/fix_replsuite_test_error_branch2.0. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8b99e204 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8b99e204 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8b99e204 Branch: refs/heads/branch-2.0 Commit: 8b99e204a9a056fd071f9bd75f3e0a29f90bccc0 Parents: 399597b Author: Ergin Seyfe <[email protected]> Authored: Fri Nov 4 23:29:20 2016 -0700 Committer: Reynold Xin <[email protected]> Committed: Fri Nov 4 23:29:20 2016 -0700 ---------------------------------------------------------------------- .../scala/org/apache/spark/repl/ReplSuite.scala | 17 ----------------- .../scala/org/apache/spark/sql/DatasetSuite.scala | 13 +++++++++++++ 2 files changed, 13 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/8b99e204/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala ---------------------------------------------------------------------- diff --git a/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala b/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala index 8deafe3..f7d7a4f 100644 --- a/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala +++ b/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala @@ -473,21 +473,4 @@ class ReplSuite extends SparkFunSuite { assertDoesNotContain("AssertionError", output) assertDoesNotContain("Exception", output) } - - test("SPARK-18189: Fix serialization issue in KeyValueGroupedDataset") { - val resultValue = 12345 - val output = runInterpreter("local", - s""" - |val keyValueGrouped = Seq((1, 2), (3, 4)).toDS().groupByKey(_._1) - |val mapGroups = keyValueGrouped.mapGroups((k, v) => (k, 1)) - |val broadcasted = sc.broadcast($resultValue) - | - |// Using broadcast triggers serialization issue in KeyValueGroupedDataset - |val dataset = mapGroups.map(_ => broadcasted.value) - |dataset.collect() - """.stripMargin) - assertDoesNotContain("error:", output) - assertDoesNotContain("Exception", output) - assertContains(s": Array[Int] = Array($resultValue, $resultValue)", output) - } } http://git-wip-us.apache.org/repos/asf/spark/blob/8b99e204/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala index f897cfb..6113e5d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala @@ -882,6 +882,19 @@ class DatasetSuite extends QueryTest with SharedSQLContext { df.withColumn("b", expr("0")).as[ClassData] .groupByKey(_.a).flatMapGroups { case (x, iter) => List[Int]() }) } + + // This is moved from ReplSuite to prevent java.lang.ClassCircularityError. + test("SPARK-18189: Fix serialization issue in KeyValueGroupedDataset") { + val resultValue = 12345 + val keyValueGrouped = Seq((1, 2), (3, 4)).toDS().groupByKey(_._1) + val mapGroups = keyValueGrouped.mapGroups((k, v) => (k, 1)) + val broadcasted = spark.sparkContext.broadcast(resultValue) + + // Using broadcast triggers serialization issue in KeyValueGroupedDataset + val dataset = mapGroups.map(_ => broadcasted.value) + + assert(dataset.collect() sameElements Array(resultValue, resultValue)) + } } case class Generic[T](id: T, value: Double) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
