This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 7b9190bb93f0 [SPARK-51135][SQL] Fix ViewResolverSuite for ANSI modes
7b9190bb93f0 is described below
commit 7b9190bb93f00af02fd9e6990e3d846f023c4d64
Author: Vladimir Golubev <[email protected]>
AuthorDate: Mon Feb 10 10:46:34 2025 +0800
[SPARK-51135][SQL] Fix ViewResolverSuite for ANSI modes
### What changes were proposed in this pull request?
Fix `ViewResolverSuite` for non-ANSI mode. View column `Cast`s have have
ANSI evaluation in non-ANSI mode when view schema `COMPENSATION` is the default
view schema mode:
https://github.com/apache/spark/blob/301b666a1fcbd4c59d96c53fe3a547ea1512f397/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala#L965
Also, use `dsl` package to simplify test code.
### Why are the changes needed?
To fix tests in non-ANSI mode:
https://github.com/apache/spark/pull/49658#discussion_r1944266007
### Does this PR introduce _any_ user-facing change?
Just a test suite change.
### How was this patch tested?
Fixed the `ViewResolverSuite`.
### Was this patch authored or co-authored using generative AI tooling?
copilot.nvim.
Closes #49857 from vladimirg-db/vladimirg-db/fix-ansi-view-resolution-suite.
Authored-by: Vladimir Golubev <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit d0848aaaeda2444f6e686e1e61e76885061799b0)
Signed-off-by: Wenchen Fan <[email protected]>
---
.../sql/analysis/resolver/ViewResolverSuite.scala | 101 ++++++++++-----------
1 file changed, 47 insertions(+), 54 deletions(-)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/analysis/resolver/ViewResolverSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/analysis/resolver/ViewResolverSuite.scala
index 285074f1e57c..d50b0a321184 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/analysis/resolver/ViewResolverSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/analysis/resolver/ViewResolverSuite.scala
@@ -17,76 +17,62 @@
package org.apache.spark.sql.analysis.resolver
-import org.apache.spark.sql.{AnalysisException, QueryTest}
-import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
+import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
-import org.apache.spark.sql.catalyst.analysis.resolver.{MetadataResolver,
Resolver}
-import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference,
Cast}
-import org.apache.spark.sql.catalyst.plans.logical.{
- LocalRelation,
- LogicalPlan,
- OneRowRelation,
- Project,
- SubqueryAlias,
- View
-}
+import org.apache.spark.sql.catalyst.analysis.resolver.{MetadataResolver,
Resolver, ResolverRunner}
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.test.SharedSparkSession
-import org.apache.spark.sql.types.{IntegerType, StringType}
+import org.apache.spark.sql.types._
class ViewResolverSuite extends QueryTest with SharedSparkSession {
private val catalogName =
"spark_catalog"
- private val col1Integer =
- AttributeReference(name = "col1", dataType = IntegerType, nullable =
false)()
- private val col2String =
- AttributeReference(name = "col2", dataType = StringType, nullable =
false)()
+ private val col1Integer = "col1".attr.int.withNullability(false)
+ private val col2String = "col2".attr.string.withNullability(false)
+ private val testTable = LocalRelation.fromExternalRows(
+ Seq(col1Integer, col2String),
+ Seq(Row(1, "a"))
+ )
test("Temporary view") {
- withView("temporary_view") {
- spark.sql("CREATE TEMPORARY VIEW temporary_view AS SELECT col1, col2
FROM VALUES (1, 'a');")
+ withView("v1") {
+ spark.sql("CREATE TEMPORARY VIEW v1 AS SELECT col1, col2 FROM VALUES (1,
'a')")
checkViewResolution(
- "SELECT * FROM temporary_view",
- expectedChild = Project(
- projectList = Seq(
- Alias(Cast(col1Integer,
IntegerType).withTimeZone(conf.sessionLocalTimeZone), "col1")(),
- Alias(Cast(col2String,
StringType).withTimeZone(conf.sessionLocalTimeZone), "col2")()
- ),
- child = Project(
- projectList = Seq(col1Integer, col2String),
- child = LocalRelation(
- output = Seq(col1Integer, col2String),
- data = Seq(
- InternalRow.fromSeq(Seq(1,
"a").map(CatalystTypeConverters.convertToCatalyst))
- )
- )
+ "SELECT * FROM v1",
+ expectedChild = testTable
+ .select(col1Integer, col2String)
+ .select(
+ cast(col1Integer, IntegerType).as("col1"),
+ cast(col2String, StringType).as("col2")
)
- )
)
}
}
test("Persistent view") {
- withView("persistent_view") {
- spark.sql("CREATE VIEW persistent_view AS SELECT col1, col2 FROM VALUES
(1, 'a');")
+ withView("v1") {
+ spark.sql("CREATE VIEW v1 AS SELECT col1, col2 FROM VALUES (1, 'a')")
checkViewResolution(
- "SELECT * FROM persistent_view",
- expectedChild = Project(
- projectList = Seq(
- Alias(Cast(col1Integer,
IntegerType).withTimeZone(conf.sessionLocalTimeZone), "col1")(),
- Alias(Cast(col2String,
StringType).withTimeZone(conf.sessionLocalTimeZone), "col2")()
- ),
- child = Project(
- projectList = Seq(col1Integer, col2String),
- child = LocalRelation(
- output = Seq(col1Integer, col2String),
- data = Seq(
- InternalRow.fromSeq(Seq(1,
"a").map(CatalystTypeConverters.convertToCatalyst))
- )
- )
+ "SELECT * FROM v1",
+ expectedChild = testTable
+ .select(col1Integer, col2String)
+ .select(
+ cast(
+ col1Integer,
+ IntegerType,
+ ansiEnabled = conf.ansiEnabled || conf.viewSchemaCompensation
+ ).as("col1"),
+ cast(
+ col2String,
+ StringType,
+ ansiEnabled = conf.ansiEnabled || conf.viewSchemaCompensation
+ ).as("col2")
)
- )
)
}
}
@@ -173,10 +159,10 @@ class ViewResolverSuite extends QueryTest with
SharedSparkSession {
.child
.asInstanceOf[View]
- val resolver = new Resolver(spark.sessionState.catalogManager)
+ val resolverRunner = new ResolverRunner(new
Resolver(spark.sessionState.catalogManager))
- val resolvedView = resolver
- .lookupMetadataAndResolve(unresolvedPlan)
+ val resolvedView = resolverRunner
+ .resolve(unresolvedPlan)
.asInstanceOf[Project]
.child
.asInstanceOf[SubqueryAlias]
@@ -185,4 +171,11 @@ class ViewResolverSuite extends QueryTest with
SharedSparkSession {
assert(resolvedView.isTempView == unresolvedView.isTempView)
assert(normalizeExprIds(resolvedView.child) ==
normalizeExprIds(expectedChild))
}
+
+ private def cast(
+ child: Expression,
+ dataType: DataType,
+ ansiEnabled: Boolean = conf.ansiEnabled): Expression = {
+ Cast(child, dataType, ansiEnabled).withTimeZone(conf.sessionLocalTimeZone)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
