This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 6eb4d3c9d38f [SPARK-53865][SQL] Extract common logic from
ResolveGenerate rule
6eb4d3c9d38f is described below
commit 6eb4d3c9d38f6849b0acfcffdbadce03c8f49ac6
Author: Mikhail Nikoliukin <[email protected]>
AuthorDate: Sat Oct 11 00:02:03 2025 +0800
[SPARK-53865][SQL] Extract common logic from ResolveGenerate rule
### What changes were proposed in this pull request?
Small refactoring, extracting `makeGeneratorOutput` to separate the object.
### Why are the changes needed?
In the future, it will be reused in the single-pass analyzer. It's a much
clearer approach than having a direct dependency on the legacy rule in the new
analyzer.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Existing tests
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code v2.0.13
Closes #52571 from mikhailnik-db/refactor-resolve-generate.
Authored-by: Mikhail Nikoliukin <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
---
.../spark/sql/catalyst/analysis/Analyzer.scala | 26 ++---------
.../catalyst/analysis/GeneratorResolution.scala | 51 ++++++++++++++++++++++
2 files changed, 54 insertions(+), 23 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 489f6da231f8..efbf86c76af1 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -3188,7 +3188,7 @@ class Analyzer(override val catalogManager:
CatalogManager) extends RuleExecutor
unrequiredChildIndex = Nil,
outer = outer,
qualifier = None,
- generatorOutput =
ResolveGenerate.makeGeneratorOutput(generator, names),
+ generatorOutput =
GeneratorResolution.makeGeneratorOutput(generator, names),
child)
(Some(g), res._2 ++ g.nullableOutput)
case other =>
@@ -3236,28 +3236,8 @@ class Analyzer(override val catalogManager:
CatalogManager) extends RuleExecutor
if (g.generator.children.exists(ExtractGenerator.hasGenerator)) {
throw QueryCompilationErrors.nestedGeneratorError(g.generator)
}
- g.copy(generatorOutput = makeGeneratorOutput(g.generator,
g.generatorOutput.map(_.name)))
- }
- }
-
- /**
- * Construct the output attributes for a [[Generator]], given a list of
names. If the list of
- * names is empty names are assigned from field names in generator.
- */
- private[analysis] def makeGeneratorOutput(
- generator: Generator,
- names: Seq[String]): Seq[Attribute] = {
- val elementAttrs = DataTypeUtils.toAttributes(generator.elementSchema)
-
- if (names.length == elementAttrs.length) {
- names.zip(elementAttrs).map {
- case (name, attr) => attr.withName(name)
- }
- } else if (names.isEmpty) {
- elementAttrs
- } else {
- throw QueryCompilationErrors.aliasesNumberNotMatchUDTFOutputError(
- elementAttrs.size, names.mkString(","))
+ g.copy(generatorOutput =
+ GeneratorResolution.makeGeneratorOutput(g.generator,
g.generatorOutput.map(_.name)))
}
}
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/GeneratorResolution.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/GeneratorResolution.scala
new file mode 100644
index 000000000000..de826facdf43
--- /dev/null
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/GeneratorResolution.scala
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.catalyst.expressions.{Attribute, Generator}
+import org.apache.spark.sql.catalyst.types.DataTypeUtils
+import org.apache.spark.sql.errors.QueryCompilationErrors
+
+/**
+ * Common utilities for generator resolution.
+ */
+object GeneratorResolution {
+ /**
+ * Construct the output attributes for a [[Generator]], given a list of
names. If the list of
+ * names is empty names are assigned from field names in generator.
+ *
+ * @throws AnalysisException UDTF_ALIAS_NUMBER_MISMATCH if the number of
names does not match
+ * the number of output attributes from the
generator
+ */
+ def makeGeneratorOutput(
+ generator: Generator,
+ names: Seq[String]): Seq[Attribute] = {
+ val elementAttrs = DataTypeUtils.toAttributes(generator.elementSchema)
+
+ if (names.length == elementAttrs.length) {
+ names.zip(elementAttrs).map {
+ case (name, attr) => attr.withName(name)
+ }
+ } else if (names.isEmpty) {
+ elementAttrs
+ } else {
+ throw QueryCompilationErrors.aliasesNumberNotMatchUDTFOutputError(
+ elementAttrs.size, names.mkString(","))
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
