This is an automated email from the ASF dual-hosted git repository.
lixiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 6692bac [SPARK-26409][SQL][TESTS] SQLConf should be serializable in
test sessions
6692bac is described below
commit 6692bacf3e74e7a17d8e676e8a06ab198f85d328
Author: Gengliang Wang <[email protected]>
AuthorDate: Thu Dec 20 10:05:56 2018 -0800
[SPARK-26409][SQL][TESTS] SQLConf should be serializable in test sessions
## What changes were proposed in this pull request?
`SQLConf` is supposed to be serializable. However, currently it is not
serializable in `WithTestConf`. `WithTestConf` uses the method `overrideConfs`
in closure, while the classes which implements it
(`TestHiveSessionStateBuilder` and `TestSQLSessionStateBuilder`) are not
serializable.
This PR is to use a local variable to fix it.
## How was this patch tested?
Add unit test.
Closes #23352 from gengliangwang/serializableSQLConf.
Authored-by: Gengliang Wang <[email protected]>
Signed-off-by: gatorsmile <[email protected]>
---
.../org/apache/spark/sql/internal/BaseSessionStateBuilder.scala | 3 ++-
.../src/test/scala/org/apache/spark/sql/SerializationSuite.scala | 5 +++++
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
index ac07e1f..319c264 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
@@ -309,13 +309,14 @@ private[sql] trait WithTestConf { self:
BaseSessionStateBuilder =>
def overrideConfs: Map[String, String]
override protected lazy val conf: SQLConf = {
+ val overrideConfigurations = overrideConfs
val conf = parentState.map(_.conf.clone()).getOrElse {
new SQLConf {
clear()
override def clear(): Unit = {
super.clear()
// Make sure we start with the default test configs even after clear
- overrideConfs.foreach { case (key, value) => setConfString(key,
value) }
+ overrideConfigurations.foreach { case (key, value) =>
setConfString(key, value) }
}
}
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/SerializationSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/SerializationSuite.scala
index cd6b264..1a1c956 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SerializationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SerializationSuite.scala
@@ -27,4 +27,9 @@ class SerializationSuite extends SparkFunSuite with
SharedSQLContext {
val spark = SparkSession.builder.getOrCreate()
new JavaSerializer(new
SparkConf()).newInstance().serialize(spark.sqlContext)
}
+
+ test("[SPARK-26409] SQLConf should be serializable") {
+ val spark = SparkSession.builder.getOrCreate()
+ new JavaSerializer(new
SparkConf()).newInstance().serialize(spark.sessionState.conf)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
