This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new eb1e6ad13aab [SPARK-46388][SQL] HiveAnalysis misses the pattern guard
of `query.resolved`
eb1e6ad13aab is described below
commit eb1e6ad13aab3960f1543b75bf3b75b3a7d62746
Author: Kent Yao <[email protected]>
AuthorDate: Wed Dec 13 18:04:38 2023 +0800
[SPARK-46388][SQL] HiveAnalysis misses the pattern guard of `query.resolved`
### What changes were proposed in this pull request?
This PR adds `query.resolved` as a pattern guard when HiveAnalysis converts
InsertIntoStatement to InsertIntoHiveTable.
### Why are the changes needed?
Due to
https://github.com/apache/spark/pull/41262/files#diff-ed19f376a63eba52eea59ca71f3355d4495fad4fad4db9a3324aade0d4986a47R1080,
the `table` field is resolved regardless of the query field. Before, it never
got a chance to be resolved as `HiveTableRelation` and then match any rule of
HiveAnalysis. But now, it gets the chance always and results in a spark-kernel
bug - `Invalid call to toAttribute on unresolved object.`
```
insert into t2 select cast(a as short) from t where b=1;
Invalid call to toAttribute on unresolved object
```
### Does this PR introduce _any_ user-facing change?
no, bugfix for 3.5 and later
### How was this patch tested?
added new test
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #44326 from yaooqinn/SPARK-46388.
Authored-by: Kent Yao <[email protected]>
Signed-off-by: Kent Yao <[email protected]>
(cherry picked from commit ccc436d829cd0b07088e2864cb1ecc55ab97a491)
Signed-off-by: Kent Yao <[email protected]>
---
.../org/apache/spark/sql/hive/HiveStrategies.scala | 2 +-
.../spark/sql/hive/execution/SQLQuerySuite.scala | 26 ++++++++++++++++++++++
2 files changed, 27 insertions(+), 1 deletion(-)
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala
index 3da3d4a0eb5c..c53a6c378d45 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala
@@ -161,7 +161,7 @@ object HiveAnalysis extends Rule[LogicalPlan] {
override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
case InsertIntoStatement(
r: HiveTableRelation, partSpec, _, query, overwrite,
ifPartitionNotExists, _)
- if DDLUtils.isHiveTable(r.tableMeta) =>
+ if DDLUtils.isHiveTable(r.tableMeta) && query.resolved =>
InsertIntoHiveTable(r.tableMeta, partSpec, query, overwrite,
ifPartitionNotExists, query.output.map(_.name))
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index 9308d1eda146..6160c3e5f6c6 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -2660,6 +2660,32 @@ abstract class SQLQuerySuiteBase extends QueryTest with
SQLTestUtils with TestHi
checkAnswer(df, Seq.empty[Row])
}
}
+
+ test("SPARK-46388: HiveAnalysis convert InsertIntoStatement to
InsertIntoHiveTable " +
+ "iff child resolved") {
+ withTable("t") {
+ sql("CREATE TABLE t (a STRING)")
+ checkError(
+ exception = intercept[AnalysisException](sql("INSERT INTO t SELECT a*2
FROM t where b=1")),
+ errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION",
+ sqlState = None,
+ parameters = Map("objectName" -> "`b`", "proposal" -> "`a`"),
+ context = ExpectedContext(
+ fragment = "b",
+ start = 38,
+ stop = 38) )
+ checkError(
+ exception = intercept[AnalysisException](
+ sql("INSERT INTO t SELECT cast(a as short) FROM t where b=1")),
+ errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION",
+ sqlState = None,
+ parameters = Map("objectName" -> "`b`", "proposal" -> "`a`"),
+ context = ExpectedContext(
+ fragment = "b",
+ start = 51,
+ stop = 51))
+ }
+ }
}
@SlowHiveTest
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
