This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new f581d965e60b [SPARK-55811][SQL] Catch `NonFatal` instead of
`UnresolvedException` when calling `nodeWithOutputColumnsString`
f581d965e60b is described below
commit f581d965e60bc3f8b38bef9a25d6f19128e929cc
Author: mihailoale-db <[email protected]>
AuthorDate: Thu Mar 5 01:52:58 2026 +0800
[SPARK-55811][SQL] Catch `NonFatal` instead of `UnresolvedException` when
calling `nodeWithOutputColumnsString`
### What changes were proposed in this pull request?
```
SELECT 1L UNION SELECT 1
```
fails with `CANNOT_MERGE_INCOMPATIBLE_DATA_TYPE` when
`spark.sql.planChangeLog.level` is set to `info` because we call `Union.output`
too early (before type coercion is performed). In this PR I propose to fix it
by catching `NonFatal` instead of `UnresolvedException` when calling
`nodeWithOutputColumnsString`.
### Why are the changes needed?
To fix an issue.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Added + existing tests.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #54600 from mihailoale-db/fixunioninfoissue.
Authored-by: mihailoale-db <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
---
.../org/apache/spark/sql/catalyst/plans/QueryPlan.scala | 15 +++++++++++----
.../test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 7 +++++++
2 files changed, 18 insertions(+), 4 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
index 6a085d714ddf..8ca3f76c3b88 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
@@ -21,10 +21,10 @@ import java.lang.{Boolean => JBoolean}
import java.util.IdentityHashMap
import scala.collection.mutable
+import scala.util.control.NonFatal
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.SQLConfHelper
-import org.apache.spark.sql.catalyst.analysis.UnresolvedException
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.rules.RuleId
import org.apache.spark.sql.catalyst.rules.UnknownRuleId
@@ -56,6 +56,15 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]]
def output: Seq[Attribute]
+ /**
+ * Returns a string representation of this node with output column
information appended,
+ * including each column's nullability. If `output` has more than
`maxColumns` entries, only the
+ * first `maxColumns` are shown with a count of the remaining ones.
+ * If we encounter a [[NonFatal]], it's high likely that the call of
`this.output`
+ * ([[UnresolvedException]] by calling e.g. `dataType` on unresolved
expression or
+ * [[CANNOT_MERGE_INCOMPATIBLE_DATA_TYPE]] by calling `Union.output` before
type coercing it)
+ * throws it. In this case, falls back to showing just the node name.
+ */
override def nodeWithOutputColumnsString(maxColumns: Int): String = {
try {
nodeName + {
@@ -75,9 +84,7 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]]
}
}
} catch {
- case _: UnresolvedException =>
- // If we encounter an UnresolvedException, it's high likely that the
call of `this.output`
- // throws it. In this case, we may have to give up and only show the
nodeName.
+ case NonFatal(_) =>
nodeName + " <output='Unresolved'>"
}
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 74cdee49e55a..5a78e0519eb1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -5088,6 +5088,13 @@ class SQLQuerySuite extends QueryTest with
SharedSparkSession with AdaptiveSpark
checkAnswer(sql(query), Row(1, 2))
}
}
+
+ gridTest("SPARK-55811: Catch NonFatal instead of UnresolvedException when
calling " +
+ "nodeWithOutputColumnsString")(Seq("TRACE", "DEBUG", "INFO", "WARN",
"ERROR")) { level =>
+ withSQLConf(SQLConf.PLAN_CHANGE_LOG_LEVEL.key -> level) {
+ checkAnswer(sql("SELECT 1L UNION SELECT 1"), Row(1L))
+ }
+ }
}
case class Foo(bar: Option[String])
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
