This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
from f96417fe0d37 [SPARK-47081][CONNECT][FOLLOW-UP] Respect
spark.connect.progress.reportInterval over timeout
add fafe43c87d8b [SPARK-49907][ML][CONNECT] Support spark.ml on Connect
No new revisions were added by this update.
Summary of changes:
.../src/main/resources/error/error-conditions.json | 18 +
dev/sparktestsupport/modules.py | 2 +
.../services/org.apache.spark.ml.Estimator | 11 +-
.../services/org.apache.spark.ml.Transformer | 11 +-
.../ml/classification/ClassificationSummary.scala | 3 +-
.../scala/org/apache/spark/ml/param/params.scala | 9 +-
.../scala/org/apache/spark/ml/util/Summary.scala | 28 ++
.../org/apache/spark/ml/param/JavaTestParams.java | 3 +-
python/pyspark/ml/classification.py | 7 +
.../pyspark/ml/remote}/__init__.py | 0
python/pyspark/ml/remote/proto.py | 76 ++++
python/pyspark/ml/remote/readwrite.py | 134 +++++++
python/pyspark/ml/remote/serialize.py | 132 +++++++
python/pyspark/ml/remote/util.py | 293 ++++++++++++++
.../test_connect_spark_ml_classification.py | 49 +++
python/pyspark/ml/tests/test_algorithms.py | 82 +---
python/pyspark/ml/tests/test_classification.py | 304 +++++++++++++++
python/pyspark/ml/tests/test_persistence.py | 23 --
python/pyspark/ml/tests/test_training_summary.py | 91 -----
python/pyspark/ml/util.py | 4 +
python/pyspark/ml/wrapper.py | 18 +-
python/pyspark/sql/connect/client/core.py | 37 ++
python/pyspark/sql/connect/proto/__init__.py | 2 +
python/pyspark/sql/connect/proto/base_pb2.py | 359 ++++++++---------
python/pyspark/sql/connect/proto/base_pb2.pyi | 11 +
python/pyspark/sql/connect/proto/commands_pb2.py | 203 +++++-----
python/pyspark/sql/connect/proto/commands_pb2.pyi | 10 +
python/pyspark/sql/connect/proto/ml_common_pb2.py | 80 ++++
python/pyspark/sql/connect/proto/ml_common_pb2.pyi | 427 +++++++++++++++++++++
python/pyspark/sql/connect/proto/ml_pb2.py | 72 ++++
python/pyspark/sql/connect/proto/ml_pb2.pyi | 393 +++++++++++++++++++
python/pyspark/sql/connect/proto/relations_pb2.py | 325 ++++++++--------
python/pyspark/sql/connect/proto/relations_pb2.pyi | 203 ++++++++++
.../src/main/protobuf/spark/connect/base.proto | 4 +
.../src/main/protobuf/spark/connect/commands.proto | 3 +-
.../src/main/protobuf/spark/connect/ml.proto | 107 ++++++
.../main/protobuf/spark/connect/ml_common.proto | 108 ++++++
.../main/protobuf/spark/connect/relations.proto | 53 +++
.../org/apache/spark/sql/connect/ml/MLCache.scala | 70 ++++
.../apache/spark/sql/connect/ml/MLException.scala | 32 ++
.../apache/spark/sql/connect/ml/MLHandler.scala | 268 +++++++++++++
.../org/apache/spark/sql/connect/ml/MLUtils.scala | 353 +++++++++++++++++
.../apache/spark/sql/connect/ml/Serializer.scala | 151 ++++++++
.../sql/connect/planner/SparkConnectPlanner.scala | 21 +
.../spark/sql/connect/service/SessionHolder.scala | 6 +
.../org/apache/spark/sql/connect/ml/MLSuite.scala | 390 +++++++++++++++++++
.../connect/planner/SparkConnectPlannerSuite.scala | 4 +-
47 files changed, 4334 insertions(+), 656 deletions(-)
copy python/pyspark/sql/connect/proto/__init__.py =>
mllib/src/main/resources/META-INF/services/org.apache.spark.ml.Estimator (65%)
copy python/pyspark/sql/connect/proto/__init__.py =>
mllib/src/main/resources/META-INF/services/org.apache.spark.ml.Transformer (65%)
create mode 100644 mllib/src/main/scala/org/apache/spark/ml/util/Summary.scala
copy {examples/src/main/python => python/pyspark/ml/remote}/__init__.py (100%)
create mode 100644 python/pyspark/ml/remote/proto.py
create mode 100644 python/pyspark/ml/remote/readwrite.py
create mode 100644 python/pyspark/ml/remote/serialize.py
create mode 100644 python/pyspark/ml/remote/util.py
create mode 100644
python/pyspark/ml/tests/connect/test_connect_spark_ml_classification.py
create mode 100644 python/pyspark/ml/tests/test_classification.py
create mode 100644 python/pyspark/sql/connect/proto/ml_common_pb2.py
create mode 100644 python/pyspark/sql/connect/proto/ml_common_pb2.pyi
create mode 100644 python/pyspark/sql/connect/proto/ml_pb2.py
create mode 100644 python/pyspark/sql/connect/proto/ml_pb2.pyi
create mode 100644 sql/connect/common/src/main/protobuf/spark/connect/ml.proto
create mode 100644
sql/connect/common/src/main/protobuf/spark/connect/ml_common.proto
create mode 100644
sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/MLCache.scala
create mode 100644
sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/MLException.scala
create mode 100644
sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/MLHandler.scala
create mode 100644
sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/MLUtils.scala
create mode 100644
sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/Serializer.scala
create mode 100644
sql/connect/server/src/test/scala/org/apache/spark/sql/connect/ml/MLSuite.scala
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
