[pinot] 01/01: Fix retention for cleaning up segment lineage

[jlli]

This is an automated email from the ASF dual-hosted git repository.

jlli pushed a commit to branch fix-retention-for-segment-lineage
in repository https://gitbox.apache.org/repos/asf/pinot.git

commit 7656e3a50fd6d9e2b3544b892540a54fc24e8d48
Author: Jack Li(Analytics Engineering) <j...@jlli-mn1.linkedin.biz>
AuthorDate: Mon Sep 13 11:17:19 2021 -0700

    Fix retention for cleaning up segment lineage
---
 .../helix/core/retention/RetentionManager.java         | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git 
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java
 
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java
index b9b0d09..79da07f 100644
--- 
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java
+++ 
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java
@@ -90,7 +90,16 @@ public class RetentionManager extends 
ControllerPeriodicTask<Void> {
   }
 
   private void manageRetentionForTable(String tableNameWithType) {
+    // Manage table retention except segment lineage cleanup.
+    // The reason of separating them is that REFRESH only table will be 
skipped in the first part,
+    // whereas the segment lineage cleanup needs to be handled.
+    manageRetentionExceptSegmentLineageCleanup(tableNameWithType);
 
+    // Delete segments based on segment lineage and clean up segment lineage 
metadata.
+    manageSegmentLineageCleanupForTable(tableNameWithType);
+  }
+
+  private void manageRetentionExceptSegmentLineageCleanup(String 
tableNameWithType) {
     // Build retention strategy from table config
     TableConfig tableConfig = 
_pinotHelixResourceManager.getTableConfig(tableNameWithType);
     if (tableConfig == null) {
@@ -123,9 +132,6 @@ public class RetentionManager extends 
ControllerPeriodicTask<Void> {
     } else {
       manageRetentionForRealtimeTable(tableNameWithType, retentionStrategy);
     }
-
-    // Delete segments based on segment lineage and clean up segment lineage 
metadata
-    manageSegmentLineageCleanupForTable(tableNameWithType);
   }
 
   private void manageRetentionForOfflineTable(String offlineTableName, 
RetentionStrategy retentionStrategy) {
@@ -202,6 +208,8 @@ public class RetentionManager extends 
ControllerPeriodicTask<Void> {
         if (segmentLineageZNRecord == null) {
           return true;
         }
+        LOGGER.info("Start cleaning up segment lineage for table: {}", 
tableNameWithType);
+        long cleanupStartTime = System.currentTimeMillis();
         SegmentLineage segmentLineage = 
SegmentLineage.fromZNRecord(segmentLineageZNRecord);
         int expectedVersion = segmentLineageZNRecord.getVersion();
 
@@ -246,8 +254,12 @@ public class RetentionManager extends 
ControllerPeriodicTask<Void> {
             
.writeSegmentLineage(_pinotHelixResourceManager.getPropertyStore(), 
segmentLineage, expectedVersion)) {
           // Delete segments based on the segment lineage
           _pinotHelixResourceManager.deleteSegments(tableNameWithType, 
segmentsToDelete);
+          LOGGER.info("Finished cleaning up segment lineage for table: {} in 
{}ms", tableNameWithType,
+              (System.currentTimeMillis() - cleanupStartTime));
           return true;
         } else {
+          LOGGER.info("Failed to write segment lineage back when cleaning up 
segment lineage for table: {}",
+              tableNameWithType);
           return false;
         }
       });

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org