This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 1f1fc8b [SPARK-32564][SQL][TEST][FOLLOWUP] Re-enable TPCDSQuerySuite
with empty tables
1f1fc8b is described below
commit 1f1fc8ba0873974c0013e82cdf420b733f772564
Author: Takeshi Yamamuro <[email protected]>
AuthorDate: Sat Aug 8 16:33:25 2020 -0700
[SPARK-32564][SQL][TEST][FOLLOWUP] Re-enable TPCDSQuerySuite with empty
tables
### What changes were proposed in this pull request?
This is the follow-up PR of #29384 to address the cloud-fan comment:
https://github.com/apache/spark/pull/29384#issuecomment-670595111
This PR re-enables `TPCDSQuerySuite` with empty tables for better test
coverages.
### Why are the changes needed?
For better test coverage.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing tests.
Closes #29391 from maropu/SPARK-32564-FOLLOWUP.
Authored-by: Takeshi Yamamuro <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit 1df855bef2b2dbe330cafb0d10e0b4af813a311a)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../org/apache/spark/sql/TPCDSQuerySuite.scala | 27 ++++++++++++++++++----
.../scala/org/apache/spark/sql/TPCDSSchema.scala | 6 -----
2 files changed, 22 insertions(+), 11 deletions(-)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
index 89f09ee..e72c31f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
@@ -17,6 +17,7 @@
package org.apache.spark.sql
+import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.util.resourceToString
import org.apache.spark.sql.internal.SQLConf
@@ -46,11 +47,7 @@ class TPCDSQuerySuite extends BenchmarkQueryTest with
TPCDSSchema {
"q81", "q82", "q83", "q84", "q85", "q86", "q87", "q88", "q89", "q90",
"q91", "q92", "q93", "q94", "q95", "q96", "q97", "q98", "q99")
- val sqlConfgs = Seq(
- SQLConf.CBO_ENABLED.key -> "true",
- SQLConf.PLAN_STATS_ENABLED.key -> "true",
- SQLConf.JOIN_REORDER_ENABLED.key -> "true"
- )
+ val sqlConfgs: Seq[(String, String)] = Nil
tpcdsQueries.foreach { name =>
val queryString = resourceToString(s"tpcds/$name.sql",
@@ -104,3 +101,23 @@ class TPCDSQuerySuite extends BenchmarkQueryTest with
TPCDSSchema {
}
}
}
+
+class TPCDSQueryWithStatsSuite extends TPCDSQuerySuite {
+
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+ for (tableName <- tableNames) {
+ // To simulate plan generation on actual TPCDS data, injects data stats
here
+ spark.sessionState.catalog.alterTableStats(
+ TableIdentifier(tableName),
Some(TPCDSTableStats.sf100TableStats(tableName)))
+ }
+ }
+
+ // Sets configurations for enabling the optimization rules that
+ // exploit data statistics.
+ override val sqlConfgs = Seq(
+ SQLConf.CBO_ENABLED.key -> "true",
+ SQLConf.PLAN_STATS_ENABLED.key -> "true",
+ SQLConf.JOIN_REORDER_ENABLED.key -> "true"
+ )
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala
b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala
index 012b7d1..43974ad 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala
@@ -17,8 +17,6 @@
package org.apache.spark.sql
-import org.apache.spark.sql.catalyst.TableIdentifier
-
trait TPCDSSchema {
private val tableColumns = Map(
@@ -257,9 +255,5 @@ trait TPCDSSchema {
|USING $format
|${options.mkString("\n")}
""".stripMargin)
-
- // To simulate plan generation on actual TPCDS data, injects data stats
here
- spark.sessionState.catalog.alterTableStats(
- TableIdentifier(tableName),
Some(TPCDSTableStats.sf100TableStats(tableName)))
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
