This is an automated email from the ASF dual-hosted git repository.
Abacn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 32d54e821a6 [Spark] Run ValidatesRunner tests serially, runner metrics
are JVM wide (#40428)
32d54e821a6 is described below
commit 32d54e821a6b14e78b2d65353c519c9b1a3adfaf
Author: Tobias Kaymak <[email protected]>
AuthorDate: Tue Oct 6 20:57:21 2026 +0200
[Spark] Run ValidatesRunner tests serially, runner metrics are JVM wide
(#40428)
Fixes #40427.
---
.github/trigger_files/beam_PostCommit_Java_PVR_Spark3_Streaming.json | 2 +-
.github/trigger_files/beam_PostCommit_Java_PVR_Spark4_Batch.json | 3 ++-
.github/trigger_files/beam_PostCommit_Java_PVR_Spark4_Streaming.json | 3 ++-
.../beam_PostCommit_Java_PVR_Spark4_StructuredStreaming.json | 2 +-
.github/trigger_files/beam_PostCommit_Java_PVR_Spark_Batch.json | 2 +-
.../trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark.json | 2 +-
.../trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark4.json | 2 +-
...eam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json | 3 ++-
runners/spark/job-server/spark_job_server.gradle | 5 +++++
runners/spark/spark_runner.gradle | 5 +++++
10 files changed, 21 insertions(+), 8 deletions(-)
diff --git
a/.github/trigger_files/beam_PostCommit_Java_PVR_Spark3_Streaming.json
b/.github/trigger_files/beam_PostCommit_Java_PVR_Spark3_Streaming.json
index cc664e5ed96..219fe098b66 100644
--- a/.github/trigger_files/beam_PostCommit_Java_PVR_Spark3_Streaming.json
+++ b/.github/trigger_files/beam_PostCommit_Java_PVR_Spark3_Streaming.json
@@ -1,5 +1,5 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to
run",
- "modification": 9,
+ "modification": 10,
"https://github.com/apache/beam/pull/39990": "removing dead code from
FnApiDoFnRunner"
}
diff --git a/.github/trigger_files/beam_PostCommit_Java_PVR_Spark4_Batch.json
b/.github/trigger_files/beam_PostCommit_Java_PVR_Spark4_Batch.json
index 04197c72efe..441fc7a0f75 100644
--- a/.github/trigger_files/beam_PostCommit_Java_PVR_Spark4_Batch.json
+++ b/.github/trigger_files/beam_PostCommit_Java_PVR_Spark4_Batch.json
@@ -1,4 +1,5 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to
run.",
- "https://github.com/apache/beam/pull/39990": "removing dead code from
FnApiDoFnRunner"
+ "https://github.com/apache/beam/pull/39990": "removing dead code from
FnApiDoFnRunner",
+ "https://github.com/apache/beam/issues/40427": "Spark ValidatesRunner tests
run serially"
}
diff --git
a/.github/trigger_files/beam_PostCommit_Java_PVR_Spark4_Streaming.json
b/.github/trigger_files/beam_PostCommit_Java_PVR_Spark4_Streaming.json
index 04197c72efe..441fc7a0f75 100644
--- a/.github/trigger_files/beam_PostCommit_Java_PVR_Spark4_Streaming.json
+++ b/.github/trigger_files/beam_PostCommit_Java_PVR_Spark4_Streaming.json
@@ -1,4 +1,5 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to
run.",
- "https://github.com/apache/beam/pull/39990": "removing dead code from
FnApiDoFnRunner"
+ "https://github.com/apache/beam/pull/39990": "removing dead code from
FnApiDoFnRunner",
+ "https://github.com/apache/beam/issues/40427": "Spark ValidatesRunner tests
run serially"
}
diff --git
a/.github/trigger_files/beam_PostCommit_Java_PVR_Spark4_StructuredStreaming.json
b/.github/trigger_files/beam_PostCommit_Java_PVR_Spark4_StructuredStreaming.json
index 5abe02fc09c..3a009261f4f 100644
---
a/.github/trigger_files/beam_PostCommit_Java_PVR_Spark4_StructuredStreaming.json
+++
b/.github/trigger_files/beam_PostCommit_Java_PVR_Spark4_StructuredStreaming.json
@@ -1,4 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to
run.",
- "modification": 1
+ "modification": 2
}
diff --git a/.github/trigger_files/beam_PostCommit_Java_PVR_Spark_Batch.json
b/.github/trigger_files/beam_PostCommit_Java_PVR_Spark_Batch.json
index 975b99265d4..c1bb7d6c379 100644
--- a/.github/trigger_files/beam_PostCommit_Java_PVR_Spark_Batch.json
+++ b/.github/trigger_files/beam_PostCommit_Java_PVR_Spark_Batch.json
@@ -1,5 +1,5 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to
run",
- "modification": 7,
+ "modification": 8,
"https://github.com/apache/beam/pull/39990": "removing dead code from
FnApiDoFnRunner"
}
diff --git
a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark.json
b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark.json
index 62ae7886c57..2160d3c6800 100644
--- a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark.json
+++ b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark.json
@@ -1,4 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to
run",
- "modification": 4
+ "modification": 5
}
diff --git
a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark4.json
b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark4.json
index b2683333323..c537844dc84 100644
--- a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark4.json
+++ b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark4.json
@@ -1,4 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to
run",
- "modification": 2
+ "modification": 3
}
diff --git
a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json
b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json
index 373c31ff234..ea9b041f3fc 100644
---
a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json
+++
b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json
@@ -9,5 +9,6 @@
"https://github.com/apache/beam/pull/34155": "noting that PR #34155 should
run this test",
"https://github.com/apache/beam/pull/35159": "moving WindowedValue and
making an interface",
"https://github.com/apache/beam/pull/39793": "noting that PR #39793 should
run this test",
- "https://github.com/apache/beam/pull/40103": "noting that PR #40103 should
run this test"
+ "https://github.com/apache/beam/pull/40103": "noting that PR #40103 should
run this test",
+ "https://github.com/apache/beam/issues/40427": "Spark ValidatesRunner tests
run serially"
}
diff --git a/runners/spark/job-server/spark_job_server.gradle
b/runners/spark/job-server/spark_job_server.gradle
index 637ac98375d..7e5e65aa4c4 100644
--- a/runners/spark/job-server/spark_job_server.gradle
+++ b/runners/spark/job-server/spark_job_server.gradle
@@ -94,6 +94,11 @@ def sickbayTests = [
'org.apache.beam.sdk.transforms.ReshuffleTest.testReshufflePreservesMetadata',
]
+// One pipeline per JVM at a time, Spark runner metrics and the SparkContext
are JVM wide.
+tasks.withType(Test).configureEach {
+ systemProperty "beam.validatesRunner.parallelThreads", "1"
+}
+
def portableValidatesRunnerTask(String name, boolean streaming, boolean
docker, ArrayList<String> sickbayTests, boolean structuredStreaming = false) {
def pipelineOptions = []
if (structuredStreaming) {
diff --git a/runners/spark/spark_runner.gradle
b/runners/spark/spark_runner.gradle
index 2a161db0a82..c40eb788b61 100644
--- a/runners/spark/spark_runner.gradle
+++ b/runners/spark/spark_runner.gradle
@@ -218,6 +218,11 @@ test {
if(project.hasProperty("rerun-tests")) { outputs.upToDateWhen {false} }
}
+// One pipeline per JVM at a time, Spark runner metrics and the SparkContext
are JVM wide.
+tasks.withType(Test).configureEach {
+ systemProperty "beam.validatesRunner.parallelThreads", "1"
+}
+
class SparkComponents {
List<String> components
}
