This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new e59aa47c352 Fix linter violation in SparkSegmentTarPushJobRunner and
also apply fix for spark2 (#16488)
e59aa47c352 is described below
commit e59aa47c3525cdde3e0d8db3bb31fa874ab30cfb
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Fri Aug 1 13:30:53 2025 -0600
Fix linter violation in SparkSegmentTarPushJobRunner and also apply fix for
spark2 (#16488)
---
.../ingestion/batch/spark/SparkSegmentTarPushJobRunner.java | 10 ++++------
.../ingestion/batch/spark3/SparkSegmentTarPushJobRunner.java | 11 ++++++-----
2 files changed, 10 insertions(+), 11 deletions(-)
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark/SparkSegmentTarPushJobRunner.java
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark/SparkSegmentTarPushJobRunner.java
index babe30e7690..499ec2daba6 100644
---
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark/SparkSegmentTarPushJobRunner.java
+++
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark/SparkSegmentTarPushJobRunner.java
@@ -36,9 +36,7 @@ import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.VoidFunction;
-
public class SparkSegmentTarPushJobRunner extends
BaseSparkSegmentTarPushJobRunner {
- private SegmentGenerationJobSpec _spec;
public SparkSegmentTarPushJobRunner() {
super();
@@ -48,8 +46,8 @@ public class SparkSegmentTarPushJobRunner extends
BaseSparkSegmentTarPushJobRunn
super(spec);
}
- public void parallelizeTarPushJob(List<PinotFSSpec> pinotFSSpecs,
- List<String> segmentUris, int pushParallelism, URI outputDirURI) {
+ public void parallelizeTarPushJob(List<PinotFSSpec> pinotFSSpecs,
List<String> segmentUris, int pushParallelism,
+ URI outputDirURI) {
JavaSparkContext sparkContext =
JavaSparkContext.fromSparkContext(SparkContext.getOrCreate());
JavaRDD<String> pathRDD = sparkContext.parallelize(segmentUris,
pushParallelism);
URI finalOutputDirURI = outputDirURI;
@@ -61,8 +59,8 @@ public class SparkSegmentTarPushJobRunner extends
BaseSparkSegmentTarPushJobRunn
throws Exception {
PluginManager.get().init();
for (PinotFSSpec pinotFSSpec : pinotFSSpecs) {
- PinotFSFactory
- .register(pinotFSSpec.getScheme(), pinotFSSpec.getClassName(),
new PinotConfiguration(pinotFSSpec));
+ PinotFSFactory.register(pinotFSSpec.getScheme(),
pinotFSSpec.getClassName(),
+ new PinotConfiguration(pinotFSSpec));
}
try {
SegmentPushUtils.pushSegments(_spec,
PinotFSFactory.create(finalOutputDirURI.getScheme()),
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentTarPushJobRunner.java
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentTarPushJobRunner.java
index 4abfc449f39..ca27dd2b983 100644
---
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentTarPushJobRunner.java
+++
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentTarPushJobRunner.java
@@ -35,8 +35,9 @@ import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.VoidFunction;
+
public class SparkSegmentTarPushJobRunner extends
BaseSparkSegmentTarPushJobRunner {
-
+
public SparkSegmentTarPushJobRunner() {
super();
}
@@ -45,8 +46,8 @@ public class SparkSegmentTarPushJobRunner extends
BaseSparkSegmentTarPushJobRunn
super(spec);
}
- public void parallelizeTarPushJob(List<PinotFSSpec> pinotFSSpecs,
- List<String> segmentUris, int pushParallelism, URI outputDirURI) {
+ public void parallelizeTarPushJob(List<PinotFSSpec> pinotFSSpecs,
List<String> segmentUris, int pushParallelism,
+ URI outputDirURI) {
JavaSparkContext sparkContext =
JavaSparkContext.fromSparkContext(SparkContext.getOrCreate());
JavaRDD<String> pathRDD = sparkContext.parallelize(segmentUris,
pushParallelism);
URI finalOutputDirURI = outputDirURI;
@@ -58,8 +59,8 @@ public class SparkSegmentTarPushJobRunner extends
BaseSparkSegmentTarPushJobRunn
throws Exception {
PluginManager.get().init();
for (PinotFSSpec pinotFSSpec : pinotFSSpecs) {
- PinotFSFactory
- .register(pinotFSSpec.getScheme(), pinotFSSpec.getClassName(),
new PinotConfiguration(pinotFSSpec));
+ PinotFSFactory.register(pinotFSSpec.getScheme(),
pinotFSSpec.getClassName(),
+ new PinotConfiguration(pinotFSSpec));
}
try {
SegmentPushUtils.pushSegments(_spec,
PinotFSFactory.create(finalOutputDirURI.getScheme()),
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
