This is an automated email from the ASF dual-hosted git repository.
kabhwan pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new a92c9430ede7 [SPARK-52329][SS] Remove private sql scoping tags for new
transformWithState API
a92c9430ede7 is described below
commit a92c9430ede731d9338ce7956c38c99764799976
Author: Anish Shrigondekar <anish.shrigonde...@databricks.com>
AuthorDate: Wed May 28 12:03:11 2025 +0900
[SPARK-52329][SS] Remove private sql scoping tags for new
transformWithState API
### What changes were proposed in this pull request?
Remove private sql scoping tags for new transformWithState API
### Why are the changes needed?
To allow users to use the new operator in their queries without scoping
explicitly to sql
### Does this PR introduce _any_ user-facing change?
Yes
### How was this patch tested?
Existing unit tests
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #51033 from anishshri-db/task/SPARK-52329.
Authored-by: Anish Shrigondekar <anish.shrigonde...@databricks.com>
Signed-off-by: Jungtaek Lim <kabhwan.opensou...@gmail.com>
(cherry picked from commit 62eac3ec8f93fda2d88caf22fae86a11f1602b16)
Signed-off-by: Jungtaek Lim <kabhwan.opensou...@gmail.com>
---
.../apache/spark/sql/KeyValueGroupedDataset.scala | 16 +++++++--------
.../spark/sql/connect/KeyValueGroupedDataset.scala | 24 +++++++++++-----------
.../spark/sql/classic/KeyValueGroupedDataset.scala | 16 +++++++--------
3 files changed, 28 insertions(+), 28 deletions(-)
diff --git
a/sql/api/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala
b/sql/api/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala
index 4117ab71ac67..61914bd34268 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala
@@ -566,7 +566,7 @@ abstract class KeyValueGroupedDataset[K, V] extends
Serializable {
* See [[org.apache.spark.sql.Encoder]] for more details on what types are
encodable to Spark
* SQL.
*/
- private[sql] def transformWithState[U: Encoder](
+ def transformWithState[U: Encoder](
statefulProcessor: StatefulProcessor[K, V, U],
timeMode: TimeMode,
outputMode: OutputMode): Dataset[U]
@@ -595,7 +595,7 @@ abstract class KeyValueGroupedDataset[K, V] extends
Serializable {
* See [[org.apache.spark.sql.Encoder]] for more details on what types are
encodable to Spark
* SQL.
*/
- private[sql] def transformWithState[U: Encoder](
+ def transformWithState[U: Encoder](
statefulProcessor: StatefulProcessor[K, V, U],
eventTimeColumnName: String,
outputMode: OutputMode): Dataset[U]
@@ -621,7 +621,7 @@ abstract class KeyValueGroupedDataset[K, V] extends
Serializable {
* See [[org.apache.spark.sql.Encoder]] for more details on what types are
encodable to Spark
* SQL.
*/
- private[sql] def transformWithState[U: Encoder](
+ def transformWithState[U: Encoder](
statefulProcessor: StatefulProcessor[K, V, U],
timeMode: TimeMode,
outputMode: OutputMode,
@@ -657,7 +657,7 @@ abstract class KeyValueGroupedDataset[K, V] extends
Serializable {
* See [[org.apache.spark.sql.Encoder]] for more details on what types are
encodable to Spark
* SQL.
*/
- private[sql] def transformWithState[U: Encoder](
+ def transformWithState[U: Encoder](
statefulProcessor: StatefulProcessor[K, V, U],
eventTimeColumnName: String,
outputMode: OutputMode,
@@ -686,7 +686,7 @@ abstract class KeyValueGroupedDataset[K, V] extends
Serializable {
* See [[org.apache.spark.sql.Encoder]] for more details on what types are
encodable to Spark
* SQL.
*/
- private[sql] def transformWithState[U: Encoder, S: Encoder](
+ def transformWithState[U: Encoder, S: Encoder](
statefulProcessor: StatefulProcessorWithInitialState[K, V, U, S],
timeMode: TimeMode,
outputMode: OutputMode,
@@ -719,7 +719,7 @@ abstract class KeyValueGroupedDataset[K, V] extends
Serializable {
* See [[org.apache.spark.sql.Encoder]] for more details on what types are
encodable to Spark
* SQL.
*/
- private[sql] def transformWithState[U: Encoder, S: Encoder](
+ def transformWithState[U: Encoder, S: Encoder](
statefulProcessor: StatefulProcessorWithInitialState[K, V, U, S],
eventTimeColumnName: String,
outputMode: OutputMode,
@@ -751,7 +751,7 @@ abstract class KeyValueGroupedDataset[K, V] extends
Serializable {
* See [[org.apache.spark.sql.Encoder]] for more details on what types are
encodable to Spark
* SQL.
*/
- private[sql] def transformWithState[U: Encoder, S: Encoder](
+ def transformWithState[U: Encoder, S: Encoder](
statefulProcessor: StatefulProcessorWithInitialState[K, V, U, S],
timeMode: TimeMode,
outputMode: OutputMode,
@@ -793,7 +793,7 @@ abstract class KeyValueGroupedDataset[K, V] extends
Serializable {
* See [[org.apache.spark.sql.Encoder]] for more details on what types are
encodable to Spark
* SQL.
*/
- private[sql] def transformWithState[U: Encoder, S: Encoder](
+ def transformWithState[U: Encoder, S: Encoder](
statefulProcessor: StatefulProcessorWithInitialState[K, V, U, S],
outputMode: OutputMode,
initialState: KeyValueGroupedDataset[K, S],
diff --git
a/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/KeyValueGroupedDataset.scala
b/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/KeyValueGroupedDataset.scala
index b15e8c28df74..aaf3797f98d8 100644
---
a/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/KeyValueGroupedDataset.scala
+++
b/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/KeyValueGroupedDataset.scala
@@ -137,14 +137,14 @@ class KeyValueGroupedDataset[K, V] private[sql] ()
extends sql.KeyValueGroupedDa
}
/** @inheritdoc */
- private[sql] def transformWithState[U: Encoder](
+ def transformWithState[U: Encoder](
statefulProcessor: StatefulProcessor[K, V, U],
timeMode: TimeMode,
outputMode: OutputMode): Dataset[U] =
unsupported()
/** @inheritdoc */
- private[sql] def transformWithState[U: Encoder, S: Encoder](
+ def transformWithState[U: Encoder, S: Encoder](
statefulProcessor: StatefulProcessorWithInitialState[K, V, U, S],
timeMode: TimeMode,
outputMode: OutputMode,
@@ -152,13 +152,13 @@ class KeyValueGroupedDataset[K, V] private[sql] ()
extends sql.KeyValueGroupedDa
unsupported()
/** @inheritdoc */
- override private[sql] def transformWithState[U: Encoder](
+ override def transformWithState[U: Encoder](
statefulProcessor: StatefulProcessor[K, V, U],
eventTimeColumnName: String,
outputMode: OutputMode): Dataset[U] = unsupported()
/** @inheritdoc */
- override private[sql] def transformWithState[U: Encoder, S: Encoder](
+ override def transformWithState[U: Encoder, S: Encoder](
statefulProcessor: StatefulProcessorWithInitialState[K, V, U, S],
eventTimeColumnName: String,
outputMode: OutputMode,
@@ -241,29 +241,29 @@ class KeyValueGroupedDataset[K, V] private[sql] ()
extends sql.KeyValueGroupedDa
initialState)
/** @inheritdoc */
- override private[sql] def transformWithState[U: Encoder](
+ override def transformWithState[U: Encoder](
statefulProcessor: StatefulProcessor[K, V, U],
timeMode: TimeMode,
outputMode: OutputMode,
- outputEncoder: Encoder[U]) =
+ outputEncoder: Encoder[U]): Dataset[U] =
super.transformWithState(statefulProcessor, timeMode, outputMode,
outputEncoder)
/** @inheritdoc */
- override private[sql] def transformWithState[U: Encoder](
+ override def transformWithState[U: Encoder](
statefulProcessor: StatefulProcessor[K, V, U],
eventTimeColumnName: String,
outputMode: OutputMode,
- outputEncoder: Encoder[U]) =
+ outputEncoder: Encoder[U]): Dataset[U] =
super.transformWithState(statefulProcessor, eventTimeColumnName,
outputMode, outputEncoder)
/** @inheritdoc */
- override private[sql] def transformWithState[U: Encoder, S: Encoder](
+ override def transformWithState[U: Encoder, S: Encoder](
statefulProcessor: StatefulProcessorWithInitialState[K, V, U, S],
timeMode: TimeMode,
outputMode: OutputMode,
initialState: sql.KeyValueGroupedDataset[K, S],
outputEncoder: Encoder[U],
- initialStateEncoder: Encoder[S]) = super.transformWithState(
+ initialStateEncoder: Encoder[S]): Dataset[U] = super.transformWithState(
statefulProcessor,
timeMode,
outputMode,
@@ -272,13 +272,13 @@ class KeyValueGroupedDataset[K, V] private[sql] ()
extends sql.KeyValueGroupedDa
initialStateEncoder)
/** @inheritdoc */
- override private[sql] def transformWithState[U: Encoder, S: Encoder](
+ override def transformWithState[U: Encoder, S: Encoder](
statefulProcessor: StatefulProcessorWithInitialState[K, V, U, S],
outputMode: OutputMode,
initialState: sql.KeyValueGroupedDataset[K, S],
eventTimeColumnName: String,
outputEncoder: Encoder[U],
- initialStateEncoder: Encoder[S]) = super.transformWithState(
+ initialStateEncoder: Encoder[S]): Dataset[U] = super.transformWithState(
statefulProcessor,
outputMode,
initialState,
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/classic/KeyValueGroupedDataset.scala
b/sql/core/src/main/scala/org/apache/spark/sql/classic/KeyValueGroupedDataset.scala
index 89158928dd55..31d2bcd79066 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/classic/KeyValueGroupedDataset.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/classic/KeyValueGroupedDataset.scala
@@ -444,29 +444,29 @@ class KeyValueGroupedDataset[K, V] private[sql](
initialState)
/** @inheritdoc */
- override private[sql] def transformWithState[U: Encoder](
+ override def transformWithState[U: Encoder](
statefulProcessor: StatefulProcessor[K, V, U],
timeMode: TimeMode,
outputMode: OutputMode,
- outputEncoder: Encoder[U]) =
+ outputEncoder: Encoder[U]): Dataset[U] =
super.transformWithState(statefulProcessor, timeMode, outputMode,
outputEncoder)
/** @inheritdoc */
- override private[sql] def transformWithState[U: Encoder](
+ override def transformWithState[U: Encoder](
statefulProcessor: StatefulProcessor[K, V, U],
eventTimeColumnName: String,
outputMode: OutputMode,
- outputEncoder: Encoder[U]) =
+ outputEncoder: Encoder[U]): Dataset[U] =
super.transformWithState(statefulProcessor, eventTimeColumnName,
outputMode, outputEncoder)
/** @inheritdoc */
- override private[sql] def transformWithState[U: Encoder, S: Encoder](
+ override def transformWithState[U: Encoder, S: Encoder](
statefulProcessor: StatefulProcessorWithInitialState[K, V, U, S],
timeMode: TimeMode,
outputMode: OutputMode,
initialState: sql.KeyValueGroupedDataset[K, S],
outputEncoder: Encoder[U],
- initialStateEncoder: Encoder[S]) = super.transformWithState(
+ initialStateEncoder: Encoder[S]): Dataset[U] = super.transformWithState(
statefulProcessor,
timeMode,
outputMode,
@@ -475,13 +475,13 @@ class KeyValueGroupedDataset[K, V] private[sql](
initialStateEncoder)
/** @inheritdoc */
- override private[sql] def transformWithState[U: Encoder, S: Encoder](
+ override def transformWithState[U: Encoder, S: Encoder](
statefulProcessor: StatefulProcessorWithInitialState[K, V, U, S],
outputMode: OutputMode,
initialState: sql.KeyValueGroupedDataset[K, S],
eventTimeColumnName: String,
outputEncoder: Encoder[U],
- initialStateEncoder: Encoder[S]) = super.transformWithState(
+ initialStateEncoder: Encoder[S]): Dataset[U] = super.transformWithState(
statefulProcessor,
outputMode,
initialState,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
