This is an automated email from the ASF dual-hosted git repository.
yashmayya pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new a1455b22319 Rebuild H3 index on segment reload if resolution config is
updated (#16953)
a1455b22319 is described below
commit a1455b22319b928a1a47a18e801d8f4da08bfa38
Author: Yash Mayya <[email protected]>
AuthorDate: Mon Oct 6 17:55:59 2025 -0700
Rebuild H3 index on segment reload if resolution config is updated (#16953)
---
.../index/loader/invertedindex/H3IndexHandler.java | 44 ++++++++++++++++++++++
.../index/loader/SegmentPreProcessorTest.java | 26 +++++++++++++
2 files changed, 70 insertions(+)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/H3IndexHandler.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/H3IndexHandler.java
index 67b9a561289..32d39f6f263 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/H3IndexHandler.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/H3IndexHandler.java
@@ -20,12 +20,14 @@ package
org.apache.pinot.segment.local.segment.index.loader.invertedindex;
import com.google.common.base.Preconditions;
import java.io.File;
+import java.io.IOException;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.segment.index.loader.BaseIndexHandler;
import org.apache.pinot.segment.local.segment.index.loader.LoaderUtils;
+import
org.apache.pinot.segment.local.segment.index.readers.geospatial.ImmutableH3IndexReader;
import org.apache.pinot.segment.local.utils.GeometrySerializer;
import org.apache.pinot.segment.spi.ColumnMetadata;
import org.apache.pinot.segment.spi.V1Constants;
@@ -40,6 +42,7 @@ import
org.apache.pinot.segment.spi.index.creator.H3IndexConfig;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
+import org.apache.pinot.segment.spi.index.reader.H3IndexReader;
import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.FieldSpec.DataType;
@@ -70,6 +73,23 @@ public class H3IndexHandler extends BaseIndexHandler {
if (!columnsToAddIdx.remove(column)) {
LOGGER.info("Need to remove existing H3 index from segment: {},
column: {}", segmentName, column);
return true;
+ } else {
+ // Index already exists, check for change in resolution config
+ short newResolution =
_h3Configs.get(column).getResolution().serialize();
+ short oldResolution;
+ try (H3IndexReader indexReader = new ImmutableH3IndexReader(
+ segmentReader.getIndexFor(column, StandardIndexes.h3()))) {
+ oldResolution = indexReader.getH3IndexResolution().serialize();
+ } catch (IOException e) {
+ LOGGER.warn("Failed to read existing H3 index for segment: {},
column: {}", segmentName, column, e);
+ continue;
+ }
+ if (newResolution != oldResolution) {
+ LOGGER.info(
+ "H3 index resolution changed for segment: {}, column: {}, old
resolution: {}, new resolution: {}."
+ + " Index needs to be rebuilt.", segmentName, column,
oldResolution, newResolution);
+ return true;
+ }
}
}
// Check if any new index need to be added.
@@ -95,6 +115,30 @@ public class H3IndexHandler extends BaseIndexHandler {
LOGGER.info("Removing existing H3 index from segment: {}, column: {}",
segmentName, column);
segmentWriter.removeIndex(column, StandardIndexes.h3());
LOGGER.info("Removed existing H3 index from segment: {}, column: {}",
segmentName, column);
+ } else {
+ // Index already exists, check for change in resolution config
+ short newResolution =
_h3Configs.get(column).getResolution().serialize();
+ short oldResolution;
+
+ try (H3IndexReader indexReader = new ImmutableH3IndexReader(
+ segmentWriter.getIndexFor(column, StandardIndexes.h3()))) {
+ oldResolution = indexReader.getH3IndexResolution().serialize();
+ } catch (IOException e) {
+ LOGGER.warn("Failed to read existing H3 index for segment: {},
column: {}", segmentName, column, e);
+ segmentWriter.removeIndex(column, StandardIndexes.h3());
+ columnsToAddIdx.add(column);
+ continue;
+ }
+
+ if (newResolution != oldResolution) {
+ LOGGER.info(
+ "H3 index resolution changed for segment: {}, column: {}, old
resolution: {}, new resolution: {}. "
+ + "Deleting existing H3 index before rebuilding a new one.",
+ segmentName, column, oldResolution, newResolution);
+ segmentWriter.removeIndex(column, StandardIndexes.h3());
+ LOGGER.info("Removed existing H3 index from segment: {}, column:
{}", segmentName, column);
+ columnsToAddIdx.add(column);
+ }
}
}
for (String column : columnsToAddIdx) {
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
index f53f3ba3569..20e395ad375 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
@@ -1506,6 +1506,32 @@ public class SegmentPreProcessorTest implements
PinotBuffersAfterClassCheckRule
});
}
+ @Test(dataProvider = "bothV1AndV3")
+ public void testH3IndexResolutionUpdate(SegmentVersion segmentVersion)
+ throws Exception {
+ buildSegment(segmentVersion);
+
+ // Create H3 index with resolution 5.
+ _fieldConfigMap.put("newH3Col",
+ new FieldConfig("newH3Col", FieldConfig.EncodingType.DICTIONARY,
List.of(FieldConfig.IndexType.H3), null,
+ Map.of("resolutions", "5")));
+ runPreProcessor(_newColumnsSchemaWithH3Json);
+
+ verifyProcessNotNeeded();
+
+ // Update H3 index resolution to 4
+ _fieldConfigMap.put("newH3Col",
+ new FieldConfig("newH3Col", FieldConfig.EncodingType.DICTIONARY,
List.of(FieldConfig.IndexType.H3), null,
+ Map.of("resolutions", "4")));
+
+ // Verify that preprocessing is needed, update the index, and verify again
that no more processing is needed
+ verifyProcessNeeded();
+
+ // Remove new indexes
+ resetIndexConfigs();
+ runPreProcessor(_newColumnsSchemaWithH3Json);
+ }
+
private void verifyProcessNeeded()
throws Exception {
try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
