[pinot] branch master updated: When extracting root-level field name for complex type handling, use the whole delimiter (#11005)

[jackie]

This is an automated email from the ASF dual-hosted git repository.

jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new fac02cf4df When extracting root-level field name for complex type 
handling, use the whole delimiter (#11005)
fac02cf4df is described below

commit fac02cf4df857fbf5f3d09ab2c3eedba0b265ce4
Author: Rekha Seethamraju <rekha.k...@gmail.com>
AuthorDate: Fri Jun 30 15:41:19 2023 -0700

    When extracting root-level field name for complex type handling, use the 
whole delimiter (#11005)
---
 .../apache/pinot/segment/local/utils/IngestionUtils.java |  2 +-
 .../pinot/segment/local/utils/IngestionUtilsTest.java    | 16 ++++++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git 
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/IngestionUtils.java
 
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/IngestionUtils.java
index 72aadb70cb..1d73c4cfd5 100644
--- 
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/IngestionUtils.java
+++ 
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/IngestionUtils.java
@@ -328,7 +328,7 @@ public final class IngestionUtils {
     String delimiter = complexTypeConfig.getDelimiter() == null ? 
ComplexTypeTransformer.DEFAULT_DELIMITER
         : complexTypeConfig.getDelimiter();
     for (String field : fieldsToRead) {
-      result.add(StringUtils.split(field, delimiter)[0]);
+      result.add(StringUtils.splitByWholeSeparator(field, delimiter)[0]);
     }
     return result;
   }
diff --git 
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/IngestionUtilsTest.java
 
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/IngestionUtilsTest.java
index 9b54600e93..817ce0362a 100644
--- 
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/IngestionUtilsTest.java
+++ 
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/IngestionUtilsTest.java
@@ -233,4 +233,20 @@ public class IngestionUtilsTest {
     Assert.assertEquals(fields.size(), 1);
     Assert.assertTrue(fields.containsAll(Sets.newHashSet("s1")));
   }
+
+  @Test
+  public void testComplexTypeConfig() {
+    IngestionConfig ingestionConfig = new IngestionConfig();
+    ComplexTypeConfig complexTypeConfig = new ComplexTypeConfig(null, "__",
+        ComplexTypeConfig.CollectionNotUnnestedToJson.NON_PRIMITIVE, null);
+    Schema schema = new Schema();
+
+    ingestionConfig.setComplexTypeConfig(complexTypeConfig);
+    schema.addField(new DimensionFieldSpec("a_b__c_d", 
FieldSpec.DataType.STRING, true));
+    schema.addField(new DimensionFieldSpec("f_d", FieldSpec.DataType.STRING, 
false));
+    schema.addField(new DimensionFieldSpec("ab__cd", 
FieldSpec.DataType.STRING, true));
+    Set<String> fields = 
IngestionUtils.getFieldsForRecordExtractor(ingestionConfig, schema);
+    Assert.assertEquals(fields.size(), 3);
+    Assert.assertTrue(fields.containsAll(Sets.newHashSet("a_b", "f_d", "ab")));
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org