This is an automated email from the ASF dual-hosted git repository.
WeichenXu123 pushed a change to branch SPARK-34591
in repository https://gitbox.apache.org/repos/asf/spark.git
from 3a7945fa8263 since 4.3.0
add b4c1880baf52 [SPARK-56762][PYTHON][TESTS] Speed up
test_crossvalidator_on_pipeline
add c26a127ba331 [SPARK-56766][INFRA] Reduce the frequency of scheduled
build in GitHub Actions
add a2dd75708c38 [SPARK-56772][SQL][TESTS-ONLY] Improve `Window` testing
add 0fc754675ed9 [SPARK-56519][PYTHON] Isolate communication part from
python udf worker
add 5ed9636e5b49 [SPARK-56546][SQL] Block-chunked segment-tree window
frame for non-invertible sliding aggregates
add bb72aef62854 [MINOR][INFRA] Ignore AGENTS.md and CONTRIBUTING.md in
determine_modules_for_files
add a342461677e2 [SPARK-56770][DOCS] Update PROJ version to 9.8.1 in the
documentation for geospatial types
add d905e7378b6d [SPARK-56482][SQL][FOLLOWUP] Simplify UnionExec codegen
and narrow partition-index gate
add 1f4650674a66 [SPARK-34591][ML] Add decision tree pruning as a parameter
add 2c9cd4fb33bd [SPARK-56768][PYTHON][INFRA] Share SBT compile artifact
across pyspark CI jobs
add 4be5711835fe Revert "[SPARK-34591][ML] Add decision tree pruning as a
parameter"
add 484bb5b2cd71 [SPARK-56312][PYTHON] Refactor SQL_COGROUPED_MAP_ARROW_UDF
add d48462a53162 Merge branch 'master' into SPARK-34591
No new revisions were added by this update.
Summary of changes:
.github/workflows/build_and_test.yml | 92 ++-
.github/workflows/build_branch35.yml | 2 +-
.github/workflows/build_branch35_python.yml | 2 +-
.github/workflows/build_branch40.yml | 2 +-
.github/workflows/build_branch40_java21.yml | 2 +-
.github/workflows/build_branch40_maven.yml | 2 +-
.github/workflows/build_branch40_maven_java21.yml | 2 +-
.github/workflows/build_branch40_non_ansi.yml | 2 +-
.github/workflows/build_branch40_python.yml | 2 +-
.../workflows/build_branch40_python_pypy3.10.yml | 2 +-
.github/workflows/build_branch41.yml | 2 +-
.github/workflows/build_branch41_java21.yml | 2 +-
.github/workflows/build_branch41_maven.yml | 2 +-
.github/workflows/build_branch41_maven_java21.yml | 2 +-
.github/workflows/build_branch41_non_ansi.yml | 2 +-
.github/workflows/build_branch41_python.yml | 2 +-
.github/workflows/build_branch41_python_3.14.yml | 2 +-
.../workflows/build_branch41_python_pypy3.10.yml | 2 +-
.github/workflows/build_branch42.yml | 2 +-
.github/workflows/build_branch42_java21.yml | 2 +-
.github/workflows/build_branch42_maven.yml | 2 +-
.github/workflows/build_branch42_maven_java21.yml | 2 +-
.github/workflows/build_branch42_non_ansi.yml | 2 +-
.github/workflows/build_branch42_python.yml | 2 +-
.github/workflows/build_branch42_python_3.14.yml | 2 +-
.../workflows/build_branch42_python_pypy3.10.yml | 2 +-
.github/workflows/build_branch4x.yml | 2 +-
.github/workflows/build_branch4x_java21.yml | 2 +-
.github/workflows/build_branch4x_maven.yml | 2 +-
.github/workflows/build_branch4x_maven_java21.yml | 2 +-
.github/workflows/build_branch4x_non_ansi.yml | 2 +-
.github/workflows/build_branch4x_python.yml | 2 +-
.github/workflows/build_branch4x_python_3.14.yml | 2 +-
.../workflows/build_branch4x_python_pypy3.10.yml | 2 +-
.github/workflows/build_coverage.yml | 2 +-
.github/workflows/build_java21.yml | 2 +-
.github/workflows/build_java25.yml | 2 +-
.github/workflows/build_maven.yml | 2 +-
.github/workflows/build_maven_java21.yml | 2 +-
.github/workflows/build_maven_java21_arm.yml | 2 +-
.github/workflows/build_maven_java21_macos26.yml | 2 +-
.github/workflows/build_maven_java25.yml | 2 +-
.github/workflows/build_non_ansi.yml | 2 +-
.github/workflows/build_python_3.10.yml | 2 +-
.github/workflows/build_python_3.11.yml | 2 +-
.github/workflows/build_python_3.12_arm.yml | 2 +-
.../workflows/build_python_3.12_classic_only.yml | 2 +-
.github/workflows/build_python_3.12_macos26.yml | 2 +-
.github/workflows/build_python_3.12_pandas_3.yml | 2 +-
.github/workflows/build_python_3.13.yml | 2 +-
.github/workflows/build_python_3.14.yml | 2 +-
.github/workflows/build_python_3.14_nogil.yml | 2 +-
.github/workflows/build_python_connect.yml | 2 +-
.github/workflows/build_python_connect40.yml | 2 +-
.github/workflows/build_python_minimum.yml | 2 +-
.github/workflows/build_python_ps_minimum.yml | 2 +-
.github/workflows/build_rockdb_as_ui_backend.yml | 2 +-
.github/workflows/build_sparkr_window.yml | 2 +-
.github/workflows/build_uds.yml | 2 +-
.github/workflows/publish_snapshot.yml | 2 +-
.github/workflows/release.yml | 2 +-
dev/run-tests.py | 6 +-
dev/sparktestsupport/utils.py | 8 +-
docs/sql-ref-geospatial-types.md | 2 +-
.../ml/tests/connect/test_legacy_mode_tuning.py | 3 +-
python/pyspark/sql/conversion.py | 14 +
python/pyspark/taskcontext.py | 6 +-
python/pyspark/worker.py | 254 +++---
python/pyspark/worker_message.py | 272 +++++++
python/pyspark/worker_util.py | 145 ++--
.../spark/sql/errors/QueryExecutionErrors.scala | 10 +
.../org/apache/spark/sql/internal/SQLConf.scala | 53 ++
.../benchmarks/WindowBenchmark-jdk21-results.txt | 172 ++++
.../benchmarks/WindowBenchmark-jdk25-results.txt | 172 ++++
sql/core/benchmarks/WindowBenchmark-results.txt | 172 ++++
.../sql/execution/basicPhysicalOperators.scala | 159 ++--
.../sql/execution/window/AggregateProcessor.scala | 24 +-
.../window/SegmentTreeWindowFunctionFrame.scala | 294 +++++++
.../execution/window/WindowEvaluatorFactory.scala | 8 +-
.../window/WindowEvaluatorFactoryBase.scala | 127 ++-
.../spark/sql/execution/window/WindowExec.scala | 11 +-
.../sql/execution/window/WindowSegmentTree.scala | 614 ++++++++++++++
.../analyzer-results/named-window.sql.out | 619 ++++++++++++++
.../resources/sql-tests/inputs/named-window.sql | 207 +++++
.../sql-tests/results/named-window.sql.out | 822 +++++++++++++++++++
.../spark/sql/execution/UnionCodegenSuite.scala | 51 ++
.../sql/execution/benchmark/WindowBenchmark.scala | 372 +++++++++
.../window/SegmentTreeWindowFunctionSuite.scala | 896 +++++++++++++++++++++
.../window/SegmentTreeWindowMetricsSuite.scala | 213 +++++
.../window/SegmentTreeWindowTestHelper.scala | 176 ++++
.../window/WindowSegmentTreeAllowlistSuite.scala | 187 +++++
.../window/WindowSegmentTreeMemorySuite.scala | 351 ++++++++
.../window/WindowSegmentTreePropertySuite.scala | 475 +++++++++++
.../execution/window/WindowSegmentTreeSuite.scala | 521 ++++++++++++
94 files changed, 7267 insertions(+), 361 deletions(-)
create mode 100644 python/pyspark/worker_message.py
create mode 100644 sql/core/benchmarks/WindowBenchmark-jdk21-results.txt
create mode 100644 sql/core/benchmarks/WindowBenchmark-jdk25-results.txt
create mode 100644 sql/core/benchmarks/WindowBenchmark-results.txt
create mode 100644
sql/core/src/main/scala/org/apache/spark/sql/execution/window/SegmentTreeWindowFunctionFrame.scala
create mode 100644
sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowSegmentTree.scala
create mode 100644
sql/core/src/test/resources/sql-tests/analyzer-results/named-window.sql.out
create mode 100644
sql/core/src/test/resources/sql-tests/inputs/named-window.sql
create mode 100644
sql/core/src/test/resources/sql-tests/results/named-window.sql.out
create mode 100644
sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/WindowBenchmark.scala
create mode 100644
sql/core/src/test/scala/org/apache/spark/sql/execution/window/SegmentTreeWindowFunctionSuite.scala
create mode 100644
sql/core/src/test/scala/org/apache/spark/sql/execution/window/SegmentTreeWindowMetricsSuite.scala
create mode 100644
sql/core/src/test/scala/org/apache/spark/sql/execution/window/SegmentTreeWindowTestHelper.scala
create mode 100644
sql/core/src/test/scala/org/apache/spark/sql/execution/window/WindowSegmentTreeAllowlistSuite.scala
create mode 100644
sql/core/src/test/scala/org/apache/spark/sql/execution/window/WindowSegmentTreeMemorySuite.scala
create mode 100644
sql/core/src/test/scala/org/apache/spark/sql/execution/window/WindowSegmentTreePropertySuite.scala
create mode 100644
sql/core/src/test/scala/org/apache/spark/sql/execution/window/WindowSegmentTreeSuite.scala
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
