This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 7b1fba2 [SPARK-31242][SQL][TEST] mergeSparkConf in WithTestConf
should also respect spark.sql.legacy.sessionInitWithConfigDefaults
7b1fba2 is described below
commit 7b1fba2978e6e1fb7eacf693be2a5ad8b9a859b2
Author: yi.wu <[email protected]>
AuthorDate: Thu Mar 26 18:52:56 2020 +0800
[SPARK-31242][SQL][TEST] mergeSparkConf in WithTestConf should also respect
spark.sql.legacy.sessionInitWithConfigDefaults
### What changes were proposed in this pull request?
Make `mergeSparkConf` in `WithTestConf` respects
`spark.sql.legacy.sessionInitWithConfigDefaults`.
### Why are the changes needed?
Without the fix, conf specified by `withSQLConf` can be reverted to
original value in a cloned SparkSession. For example, you will fail test below
without the fix:
```
withSQLConf(SQLConf.CODEGEN_FALLBACK.key -> "true") {
val cloned = spark.cloneSession()
SparkSession.setActiveSession(cloned)
assert(SQLConf.get.getConf(SQLConf.CODEGEN_FALLBACK) === true)
}
```
So we should fix it just as #24540 did before.
### Does this PR introduce any user-facing change?
No.
### How was this patch tested?
Added tests.
Closes #28014 from Ngone51/sparksession_clone.
Authored-by: yi.wu <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit 8b798c1bc501f0d2b1c2c80ab64ffd764bc72987)
Signed-off-by: Wenchen Fan <[email protected]>
---
.../spark/sql/internal/BaseSessionStateBuilder.scala | 14 ++++++++++----
.../test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 10 ++++++++++
2 files changed, 20 insertions(+), 4 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
index 9556d4d..3bbdbb0 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
@@ -347,8 +347,14 @@ private[sql] trait WithTestConf { self:
BaseSessionStateBuilder =>
override protected lazy val conf: SQLConf = {
val overrideConfigurations = overrideConfs
- val conf = parentState.map(_.conf.clone()).getOrElse {
- new SQLConf {
+ parentState.map { s =>
+ val cloned = s.conf.clone()
+ if
(session.sparkContext.conf.get(StaticSQLConf.SQL_LEGACY_SESSION_INIT_WITH_DEFAULTS))
{
+ mergeSparkConf(conf, session.sparkContext.conf)
+ }
+ cloned
+ }.getOrElse {
+ val conf = new SQLConf {
clear()
override def clear(): Unit = {
super.clear()
@@ -356,8 +362,8 @@ private[sql] trait WithTestConf { self:
BaseSessionStateBuilder =>
overrideConfigurations.foreach { case (key, value) =>
setConfString(key, value) }
}
}
+ mergeSparkConf(conf, session.sparkContext.conf)
+ conf
}
- mergeSparkConf(conf, session.sparkContext.conf)
- conf
}
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index feb1450..42f0d96 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -3462,6 +3462,16 @@ class SQLQuerySuite extends QueryTest with
SharedSparkSession with AdaptiveSpark
sql("(SELECT map()) UNION ALL (SELECT map(1, 2))"),
Seq(Row(Map[Int, Int]()), Row(Map(1 -> 2))))
}
+
+ test("SPARK-31242: clone SparkSession should respect
sessionInitWithConfigDefaults") {
+ // Note, only the conf explicitly set in SparkConf(e.g. in
SharedSparkSessionBase) would cause
+ // problem before the fix.
+ withSQLConf(SQLConf.CODEGEN_FALLBACK.key -> "true") {
+ val cloned = spark.cloneSession()
+ SparkSession.setActiveSession(cloned)
+ assert(SQLConf.get.getConf(SQLConf.CODEGEN_FALLBACK) === true)
+ }
+ }
}
case class Foo(bar: Option[String])
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
