This is an automated email from the ASF dual-hosted git repository.
yangjie01 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 441922044a2b [SPARK-52485][SQL] Declarative Pipelines code cleanup
441922044a2b is described below
commit 441922044a2bafe683b11008743c151e45a65614
Author: Sandy Ryza <[email protected]>
AuthorDate: Mon Jun 16 15:00:39 2025 +0800
[SPARK-52485][SQL] Declarative Pipelines code cleanup
### What changes were proposed in this pull request?
A few small code cleanups for Declarative Pipelines code. Descriptions are
inline.
### Why are the changes needed?
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Existing tests
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #51180 from sryza/dbr-comment.
Authored-by: Sandy Ryza <[email protected]>
Signed-off-by: yangjie01 <[email protected]>
---
.../spark/sql/pipelines/graph/PipelineUpdateContextImpl.scala | 6 ------
.../org/apache/spark/sql/pipelines/utils/PipelineTest.scala | 10 +---------
2 files changed, 1 insertion(+), 15 deletions(-)
diff --git
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/PipelineUpdateContextImpl.scala
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/PipelineUpdateContextImpl.scala
index 30f56d7a8e2c..b5428e52c1d0 100644
---
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/PipelineUpdateContextImpl.scala
+++
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/PipelineUpdateContextImpl.scala
@@ -17,8 +17,6 @@
package org.apache.spark.sql.pipelines.graph
-import scala.annotation.unused
-
import org.apache.spark.sql.classic.SparkSession
import org.apache.spark.sql.pipelines.logging.{
FlowProgressEventLogger,
@@ -31,10 +29,6 @@ import org.apache.spark.sql.pipelines.logging.{
* @param unresolvedGraph The graph (unresolved) to be executed in this update.
* @param eventCallback A callback function to be called when an event is
added to the event buffer.
*/
-@unused(
- "TODO(SPARK-51727) construct this spark connect server when we expose APIs
for users " +
- "to interact with a pipeline"
-)
class PipelineUpdateContextImpl(
override val unresolvedGraph: DataflowGraph,
eventCallback: PipelineEvent => Unit
diff --git
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/utils/PipelineTest.scala
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/utils/PipelineTest.scala
index 48f2f26fb450..4c868e56a0a6 100644
---
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/utils/PipelineTest.scala
+++
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/utils/PipelineTest.scala
@@ -319,15 +319,7 @@ abstract class PipelineTest
/**
* Helper method to verify unresolved column error message. We expect three
elements to be present
- * in the message: error class, unresolved column name, list of suggested
columns. There are three
- * significant differences between different versions of DBR:
- * - Error class changed in DBR 11.3 from `MISSING_COLUMN` to
`UNRESOLVED_COLUMN.WITH_SUGGESTION`
- * - Name parts in suggested columns are escaped with backticks starting
from DBR 11.3,
- * e.g. table.column => `table`.`column`
- * - Starting from DBR 13.1 suggested columns qualification matches
unresolved column, i.e. if
- * unresolved column is a single-part identifier then suggested column
will be as well. E.g.
- * for unresolved column `x` suggested columns will omit catalog/schema or
`LIVE` qualifier. For
- * this reason we verify only last part of suggested column name.
+ * in the message: error class, unresolved column name, list of suggested
columns.
*/
protected def verifyUnresolveColumnError(
errorMessage: String,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
