(pinot) branch master updated: Adding a new config skipSegmentPreprocess in table IndexingConfig (#14982)

[xiangfu]

This is an automated email from the ASF dual-hosted git repository.

xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 3c02ec48b9 Adding a new config skipSegmentPreprocess in table 
IndexingConfig (#14982)
3c02ec48b9 is described below

commit 3c02ec48b9b4291b07518df519d89f59d6cbaa8b
Author: Xiang Fu <xiangfu.1...@gmail.com>
AuthorDate: Mon Feb 3 23:01:00 2025 -0800

    Adding a new config skipSegmentPreprocess in table IndexingConfig (#14982)
---
 .../local/indexsegment/immutable/ImmutableSegmentLoader.java     | 3 +++
 .../segment/local/segment/index/loader/IndexLoadingConfig.java   | 4 ++++
 .../java/org/apache/pinot/spi/config/table/IndexingConfig.java   | 9 +++++++++
 3 files changed, 16 insertions(+)

diff --git 
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
 
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
index 4fe545eba5..4650c832e3 100644
--- 
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
+++ 
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
@@ -282,6 +282,9 @@ public class ImmutableSegmentLoader {
   public static boolean needPreprocess(SegmentDirectory segmentDirectory, 
IndexLoadingConfig indexLoadingConfig,
       @Nullable Schema schema)
       throws Exception {
+    if (indexLoadingConfig.isSkipSegmentPreprocess()) {
+      return false;
+    }
     if (needConvertSegmentFormat(indexLoadingConfig, 
segmentDirectory.getSegmentMetadata())) {
       return true;
     }
diff --git 
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java
 
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java
index 77512a2a38..cf9fd5e30d 100644
--- 
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java
+++ 
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java
@@ -337,6 +337,10 @@ public class IndexLoadingConfig {
     _errorOnColumnBuildFailure = errorOnColumnBuildFailure;
   }
 
+  public boolean isSkipSegmentPreprocess() {
+    return _tableConfig != null && 
_tableConfig.getIndexingConfig().isSkipSegmentPreprocess();
+  }
+
   @Nullable
   public FieldIndexConfigs getFieldIndexConfig(String columnName) {
     if (_indexConfigsByColName == null || _dirty) {
diff --git 
a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/IndexingConfig.java 
b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/IndexingConfig.java
index 5beb126e0d..16642e2a7f 100644
--- 
a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/IndexingConfig.java
+++ 
b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/IndexingConfig.java
@@ -66,6 +66,7 @@ public class IndexingConfig extends BaseJsonConfig {
   private boolean _aggregateMetrics;
   private boolean _nullHandlingEnabled;
   private boolean _columnMajorSegmentBuilderEnabled = true;
+  private boolean _skipSegmentPreprocess;
 
   /**
    * If `optimizeDictionary` enabled, dictionary is not created for the 
high-cardinality
@@ -361,6 +362,14 @@ public class IndexingConfig extends BaseJsonConfig {
     _columnMajorSegmentBuilderEnabled = columnMajorSegmentBuilderEnabled;
   }
 
+  public boolean isSkipSegmentPreprocess() {
+    return _skipSegmentPreprocess;
+  }
+
+  public void setSkipSegmentPreprocess(boolean skipSegmentPreprocess) {
+    _skipSegmentPreprocess = skipSegmentPreprocess;
+  }
+
   public boolean isOptimizeDictionary() {
     return _optimizeDictionary;
   }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org