[pinot] branch master updated: Add cleanup method for HadoopSegmentCreationJob (#7371)

[jlli]

This is an automated email from the ASF dual-hosted git repository.

jlli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 29f27f8  Add cleanup method for HadoopSegmentCreationJob (#7371)
29f27f8 is described below

commit 29f27f88abc7759e90eed08e3a3801f3722321f9
Author: Jialiang Li <j...@linkedin.com>
AuthorDate: Tue Aug 31 10:11:03 2021 -0700

    Add cleanup method for HadoopSegmentCreationJob (#7371)
    
    Co-authored-by: Jack Li(Analytics Engineering) <j...@jlli-mn1.linkedin.biz>
---
 .../apache/pinot/hadoop/job/HadoopSegmentCreationJob.java  | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git 
a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentCreationJob.java
 
b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentCreationJob.java
index 474d9a4..06b02f3 100644
--- 
a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentCreationJob.java
+++ 
b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentCreationJob.java
@@ -134,9 +134,7 @@ public class HadoopSegmentCreationJob extends 
SegmentCreationJob {
 
     moveSegmentsToOutputDir();
 
-    // Delete the staging directory
-    _logger.info("Deleting the staging directory: {}", _stagingDir);
-    _outputDirFileSystem.delete(new Path(_stagingDir), true);
+    cleanup(job);
   }
 
   protected void validateTableConfig(TableConfig tableConfig) {
@@ -177,4 +175,14 @@ public class HadoopSegmentCreationJob extends 
SegmentCreationJob {
     Path segmentTarDir = new Path(new Path(_stagingDir, "output"), 
JobConfigConstants.SEGMENT_TAR_DIR);
     movePath(_outputDirFileSystem, segmentTarDir.toString(), _outputDir, true);
   }
+
+  /**
+   * Cleans up after the job completes.
+   */
+  protected void cleanup(Job job)
+      throws Exception {
+    // Delete the staging directory
+    _logger.info("Deleting the staging directory: {}", _stagingDir);
+    _outputDirFileSystem.delete(new Path(_stagingDir), true);
+  }
 }

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org