(pinot) branch master updated: Reduce the overhead of PinotHelixResourceManager.getSegmentsFor() when no time filtering is required (#13928)

[xiangfu]

This is an automated email from the ASF dual-hosted git repository.

xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 3f324a4f0b Reduce the overhead of 
PinotHelixResourceManager.getSegmentsFor() when no time filtering is required 
(#13928)
3f324a4f0b is described below

commit 3f324a4f0bfdecb1b14e459deec647a2ff2f734d
Author: Xiaotian (Jackie) Jiang <17555551+jackie-ji...@users.noreply.github.com>
AuthorDate: Mon Sep 2 21:02:53 2024 -0700

    Reduce the overhead of PinotHelixResourceManager.getSegmentsFor() when no 
time filtering is required (#13928)
---
 .../helix/core/PinotHelixResourceManager.java      | 39 +++++++++++-----------
 1 file changed, 19 insertions(+), 20 deletions(-)

diff --git 
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
 
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
index e8122b5485..3d3a7ded35 100644
--- 
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
+++ 
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
@@ -882,32 +882,31 @@ public class PinotHelixResourceManager {
       long startTimestamp, long endTimestamp, boolean excludeOverlapping) {
     IdealState idealState = getTableIdealState(tableNameWithType);
     Preconditions.checkState(idealState != null, "Failed to find ideal state 
for table: %s", tableNameWithType);
-    List<String> segments = new ArrayList<>(idealState.getPartitionSet());
-    List<SegmentZKMetadata> segmentZKMetadataList = 
getSegmentsZKMetadata(tableNameWithType);
-    List<String> selectedSegments = new ArrayList<>();
-    ArrayList<String> filteredSegments = new ArrayList<>();
-    for (SegmentZKMetadata segmentZKMetadata : segmentZKMetadataList) {
-      String segmentName = segmentZKMetadata.getSegmentName();
-      // Compute the interesction of segmentZK metadata and idealstate for 
valid segmnets
-      if (!segments.contains(segmentName)) {
-        filteredSegments.add(segmentName);
-        continue;
-      }
-      // No need to filter by time if the time range is not specified
-      if (startTimestamp == Long.MIN_VALUE && endTimestamp == Long.MAX_VALUE) {
-        selectedSegments.add(segmentName);
-      } else {
+    Set<String> segmentSet = idealState.getPartitionSet();
+    List<String> selectedSegments;
+    if (startTimestamp == Long.MIN_VALUE && endTimestamp == Long.MAX_VALUE) {
+      selectedSegments = new ArrayList<>(segmentSet);
+    } else {
+      selectedSegments = new ArrayList<>();
+      List<SegmentZKMetadata> segmentZKMetadataList = 
getSegmentsZKMetadata(tableNameWithType);
+      ArrayList<String> filteredSegments = new ArrayList<>();
+      for (SegmentZKMetadata segmentZKMetadata : segmentZKMetadataList) {
+        String segmentName = segmentZKMetadata.getSegmentName();
+        // Compute the intersection of segmentZK metadata and idealstate for 
valid segments
+        if (!segmentSet.contains(segmentName)) {
+          filteredSegments.add(segmentName);
+          continue;
+        }
         // Filter by time if the time range is specified
         if (isSegmentWithinTimeStamps(segmentZKMetadata, startTimestamp, 
endTimestamp, excludeOverlapping)) {
           selectedSegments.add(segmentName);
         }
       }
+      LOGGER.info(
+          "Successfully computed the segments for table : {}. # of filtered 
segments: {}, the filtered segment list: "
+              + "{}. Only showing up to 100 filtered segments.", 
tableNameWithType, filteredSegments.size(),
+          filteredSegments.size() > 100 ? filteredSegments.subList(0, 100) : 
filteredSegments);
     }
-    LOGGER.info(
-        "Successfully computed the segments for table : {}. # of filtered 
segments: {}, the filtered segment list: "
-            + "{}. Only showing up to 100 filtered segments.", 
tableNameWithType, filteredSegments.size(),
-        (filteredSegments.size() > 0) ? filteredSegments.subList(0, 
Math.min(filteredSegments.size(), 100))
-            : filteredSegments);
     return shouldExcludeReplacedSegments ? 
excludeReplacedSegments(tableNameWithType, selectedSegments)
         : selectedSegments;
   }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org