This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new f927c0f24c5 [SPARK-45920][SQL][3.4] group by ordinal should be
idempotent
f927c0f24c5 is described below
commit f927c0f24c51f10e4e56a09a15795fe4df0e007a
Author: Wenchen Fan <[email protected]>
AuthorDate: Thu Nov 16 08:18:40 2023 -0800
[SPARK-45920][SQL][3.4] group by ordinal should be idempotent
backport https://github.com/apache/spark/pull/43797
### What changes were proposed in this pull request?
GROUP BY ordinal is not idempotent today. If the ordinal points to another
integer literal and the plan get analyzed again, we will re-do the ordinal
resolution which can lead to wrong result or index out-of-bound error. This PR
fixes it by using a hack: if the ordinal points to another integer literal,
don't replace the ordinal.
### Why are the changes needed?
For advanced users or Spark plugins, they may manipulate the logical plans
directly. We need to make the framework more reliable.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
new test
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #43838 from cloud-fan/3.4-port.
Authored-by: Wenchen Fan <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../spark/sql/catalyst/analysis/Analyzer.scala | 14 ++++++++++++-
.../SubstituteUnresolvedOrdinalsSuite.scala | 23 ++++++++++++++++++++--
2 files changed, 34 insertions(+), 3 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index b7d174089bc..c2efac4c84f 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -1993,7 +1993,19 @@ class Analyzer(override val catalogManager:
CatalogManager) extends RuleExecutor
throw
QueryCompilationErrors.groupByPositionRefersToAggregateFunctionError(
index, ordinalExpr)
} else {
- ordinalExpr
+ trimAliases(ordinalExpr) match {
+ // HACK ALERT: If the ordinal expression is also an integer
literal, don't use it
+ // but still keep the ordinal literal. The reason
is we may repeatedly
+ // analyze the plan. Using a different integer
literal may lead to
+ // a repeat GROUP BY ordinal resolution which is
wrong. GROUP BY
+ // constant is meaningless so whatever value does
not matter here.
+ // TODO: (SPARK-45932) GROUP BY ordinal should pull out
grouping expressions to
+ // a Project, then the resolved ordinal expression is
always
+ // `AttributeReference`.
+ case Literal(_: Int, IntegerType) =>
+ Literal(index)
+ case _ => ordinalExpr
+ }
}
} else {
throw QueryCompilationErrors.groupByPositionRangeError(index,
aggs.size)
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/SubstituteUnresolvedOrdinalsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/SubstituteUnresolvedOrdinalsSuite.scala
index b0d7ace646e..953b2c8bb10 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/SubstituteUnresolvedOrdinalsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/SubstituteUnresolvedOrdinalsSuite.scala
@@ -17,10 +17,11 @@
package org.apache.spark.sql.catalyst.analysis
-import org.apache.spark.sql.catalyst.analysis.TestRelations.testRelation2
+import org.apache.spark.sql.catalyst.analysis.TestRelations.{testRelation,
testRelation2}
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
-import org.apache.spark.sql.catalyst.expressions.Literal
+import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, Literal}
+import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
import org.apache.spark.sql.internal.SQLConf
class SubstituteUnresolvedOrdinalsSuite extends AnalysisTest {
@@ -67,4 +68,22 @@ class SubstituteUnresolvedOrdinalsSuite extends AnalysisTest
{
testRelation2.groupBy(Literal(1), Literal(2))($"a", $"b"))
}
}
+
+ test("SPARK-45920: group by ordinal repeated analysis") {
+ val plan = testRelation.groupBy(Literal(1))(Literal(100).as("a")).analyze
+ comparePlans(
+ plan,
+ testRelation.groupBy(Literal(1))(Literal(100).as("a"))
+ )
+
+ val testRelationWithData = testRelation.copy(data = Seq(new
GenericInternalRow(Array(1: Any))))
+ // Copy the plan to reset its `analyzed` flag, so that analyzer rules will
re-apply.
+ val copiedPlan = plan.transform {
+ case _: LocalRelation => testRelationWithData
+ }
+ comparePlans(
+ copiedPlan.analyze, // repeated analysis
+ testRelationWithData.groupBy(Literal(1))(Literal(100).as("a"))
+ )
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
