This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 36b3bbc0 [SPARK-36979][SQL] Add RewriteLateralSubquery rule into
nonExcludableRules
36b3bbc0 is described below
commit 36b3bbc0aa9f9c39677960cd93f32988c7d7aaca
Author: ulysses-you <[email protected]>
AuthorDate: Tue Oct 12 16:21:53 2021 +0800
[SPARK-36979][SQL] Add RewriteLateralSubquery rule into nonExcludableRules
### What changes were proposed in this pull request?
Add RewriteLateralSubquery rule into nonExcludableRules.
### Why are the changes needed?
Lateral Join has no meaning without rule `RewriteLateralSubquery`. So now
if we set
`spark.sql.optimizer.excludedRules=org.apache.spark.sql.catalyst.optimizer.RewriteLateralSubquery`,
the lateral join query will fail with:
```
java.lang.AssertionError: assertion failed: No plan for LateralJoin
lateral-subquery#218
```
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
add test
Closes #34249 from ulysses-you/SPARK-36979.
Authored-by: ulysses-you <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
---
.../scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala | 3 ++-
sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 7 +++++++
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index b8c7fe7..73be790 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -284,7 +284,8 @@ abstract class Optimizer(catalogManager: CatalogManager)
NormalizeFloatingNumbers.ruleName ::
ReplaceUpdateFieldsExpression.ruleName ::
PullOutGroupingExpressions.ruleName ::
- RewriteAsOfJoin.ruleName :: Nil
+ RewriteAsOfJoin.ruleName ::
+ RewriteLateralSubquery.ruleName :: Nil
/**
* Optimize all the subqueries inside expression.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 3d5b911..11b7ee6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -4204,6 +4204,13 @@ class SQLQuerySuite extends QueryTest with
SharedSparkSession with AdaptiveSpark
checkAnswer(sql("""SELECT from_json(r'{"a": "\\"}', 'a string')"""),
Row(Row("\\")))
checkAnswer(sql("""SELECT from_json(R'{"a": "\\"}', 'a string')"""),
Row(Row("\\")))
}
+
+ test("SPARK-36979: Add RewriteLateralSubquery rule into nonExcludableRules")
{
+ withSQLConf(SQLConf.OPTIMIZER_EXCLUDED_RULES.key ->
+ "org.apache.spark.sql.catalyst.optimizer.RewriteLateralSubquery") {
+ sql("SELECT * FROM testData, LATERAL (SELECT * FROM testData)").collect()
+ }
+ }
}
case class Foo(bar: Option[String])
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
