This is an automated email from the ASF dual-hosted git repository.
yupeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 086c3f6 Allow overriding delimiter in Complex type config (#6905)
086c3f6 is described below
commit 086c3f6af1e417449bddf6810776faaf1a2367d9
Author: Yupeng Fu <yupe...@users.noreply.github.com>
AuthorDate: Wed May 12 20:18:47 2021 -0700
Allow overriding delimiter in Complex type config (#6905)
* allow overriding delimiter
* comments
---
.../common/utils/config/TableConfigSerDeTest.java | 2 +-
.../recordtransformer/ComplexTypeTransformer.java | 41 ++++++++++++++--------
.../ComplexTypeTransformerTest.java | 26 +++++++++++---
.../config/table/ingestion/ComplexTypeConfig.java | 12 ++++++-
4 files changed, 60 insertions(+), 21 deletions(-)
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java
index 84ebe3d..c065112 100644
---
a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java
+++
b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java
@@ -278,7 +278,7 @@ public class TableConfigSerDeTest {
IngestionConfig ingestionConfig =
new IngestionConfig(new BatchIngestionConfig(batchConfigMaps,
"APPEND", "HOURLY"),
new StreamIngestionConfig(streamConfigMaps), new
FilterConfig("filterFunc(foo)"), transformConfigs,
- new ComplexTypeConfig(unnestFields));
+ new ComplexTypeConfig(unnestFields, "."));
TableConfig tableConfig =
tableConfigBuilder.setIngestionConfig(ingestionConfig).build();
checkIngestionConfig(tableConfig);
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/ComplexTypeTransformer.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/ComplexTypeTransformer.java
index 8760bf7..01b8add 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/ComplexTypeTransformer.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/ComplexTypeTransformer.java
@@ -77,28 +77,41 @@ import org.apache.pinot.spi.data.readers.GenericRow;
*
*/
public class ComplexTypeTransformer implements RecordTransformer {
- // TODO: make configurable
- private static final CharSequence DELIMITER = ".";
+ private static final String DEFAULT_DELIMITER = ".";
private final List<String> _unnestFields;
+ private final String _delimiter;
public ComplexTypeTransformer(TableConfig tableConfig) {
- if (tableConfig.getIngestionConfig() != null &&
tableConfig.getIngestionConfig().getComplexTypeConfig() != null) {
- _unnestFields =
tableConfig.getIngestionConfig().getComplexTypeConfig().getUnnestFields() !=
null ? tableConfig
- .getIngestionConfig().getComplexTypeConfig().getUnnestFields() : new
ArrayList<>();
- // the unnest fields are sorted to achieve the topological sort of the
collections, so that the parent collection
- // (e.g. foo) is unnested before the child collection (e.g. foo.bar)
- Collections.sort(_unnestFields);
- } else {
- _unnestFields = new ArrayList<>();
- }
+ this(parseUnnestFields(tableConfig), parseDelimiter(tableConfig));
}
@VisibleForTesting
- public ComplexTypeTransformer(List<String> unnestFields) {
+ public ComplexTypeTransformer(List<String> unnestFields, String delimiter) {
_unnestFields = new ArrayList<>(unnestFields);
+ _delimiter = delimiter;
+ // the unnest fields are sorted to achieve the topological sort of the
collections, so that the parent collection
+ // (e.g. foo) is unnested before the child collection (e.g. foo.bar)
Collections.sort(_unnestFields);
}
+ private static List<String> parseUnnestFields(TableConfig tableConfig) {
+ if (tableConfig.getIngestionConfig() != null &&
tableConfig.getIngestionConfig().getComplexTypeConfig() != null
+ &&
tableConfig.getIngestionConfig().getComplexTypeConfig().getUnnestFields() !=
null) {
+ return
tableConfig.getIngestionConfig().getComplexTypeConfig().getUnnestFields();
+ } else {
+ return new ArrayList<>();
+ }
+ }
+
+ private static String parseDelimiter(TableConfig tableConfig) {
+ if (tableConfig.getIngestionConfig() != null &&
tableConfig.getIngestionConfig().getComplexTypeConfig() != null
+ &&
tableConfig.getIngestionConfig().getComplexTypeConfig().getDelimiter() != null)
{
+ return
tableConfig.getIngestionConfig().getComplexTypeConfig().getDelimiter();
+ } else {
+ return DEFAULT_DELIMITER;
+ }
+ }
+
public static boolean isComplexTypeHandlingEnabled(TableConfig tableConfig) {
return tableConfig.getIngestionConfig() != null &&
tableConfig.getIngestionConfig().getComplexTypeConfig() != null;
}
@@ -253,7 +266,7 @@ public class ComplexTypeTransformer implements
RecordTransformer {
}
}
- private static String concat(String left, String right) {
- return String.join(DELIMITER, left, right);
+ private String concat(String left, String right) {
+ return String.join(_delimiter, left, right);
}
}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/recordtransformer/ComplexTypeTransformerTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/recordtransformer/ComplexTypeTransformerTest.java
index c64749b..f58fa64 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/recordtransformer/ComplexTypeTransformerTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/recordtransformer/ComplexTypeTransformerTest.java
@@ -33,7 +33,7 @@ import org.testng.annotations.Test;
public class ComplexTypeTransformerTest {
@Test
public void testFlattenMap() {
- ComplexTypeTransformer transformer = new ComplexTypeTransformer(new
ArrayList<>());
+ ComplexTypeTransformer transformer = new ComplexTypeTransformer(new
ArrayList<>(), ".");
// test flatten root-level tuples
GenericRow genericRow = new GenericRow();
@@ -57,7 +57,7 @@ public class ComplexTypeTransformerTest {
Assert.assertEquals(genericRow.getValue("map2.c"), 3);
// test flattening the tuple inside the collection
- transformer = new ComplexTypeTransformer(Arrays.asList("l1"));
+ transformer = new ComplexTypeTransformer(Arrays.asList("l1"), ".");
genericRow = new GenericRow();
List<Map<String, Object>> list1 = new ArrayList<>();
list1.add(map1);
@@ -70,6 +70,22 @@ public class ComplexTypeTransformerTest {
Assert.assertEquals(map.get("b"), "v");
Assert.assertEquals(map.get("im1.aa"), 2);
Assert.assertEquals(map.get("im1.bb"), "u");
+
+ // test overriding delimiter
+ transformer = new ComplexTypeTransformer(Arrays.asList("l1"), "_");
+ genericRow = new GenericRow();
+ innerMap1 = new HashMap<>();
+ innerMap1.put("aa", 2);
+ innerMap1.put("bb", "u");
+ map1 = new HashMap<>();
+ map1.put("im1", innerMap1);
+ list1 = new ArrayList<>();
+ list1.add(map1);
+ genericRow.putValue("l1", list1);
+ transformer.flattenMap(genericRow, new
ArrayList<>(genericRow.getFieldToValueMap().keySet()));
+ map = (Map<String, Object>) ((Collection)
genericRow.getValue("l1")).iterator().next();
+ Assert.assertEquals(map.get("im1_aa"), 2);
+ Assert.assertEquals(map.get("im1_bb"), "u");
}
@Test
@@ -91,7 +107,7 @@ public class ComplexTypeTransformerTest {
// {
// "array.a":"v2"
// }]
- ComplexTypeTransformer transformer = new
ComplexTypeTransformer(Arrays.asList("array"));
+ ComplexTypeTransformer transformer = new
ComplexTypeTransformer(Arrays.asList("array"), ".");
GenericRow genericRow = new GenericRow();
Object[] array = new Object[2];
Map<String, Object> map1 = new HashMap<>();
@@ -141,7 +157,7 @@ public class ComplexTypeTransformerTest {
// "array.a":"v2","array2.b":"v4"
// }]
//
- transformer = new ComplexTypeTransformer(Arrays.asList("array", "array2"));
+ transformer = new ComplexTypeTransformer(Arrays.asList("array", "array2"),
".");
genericRow = new GenericRow();
Object[] array2 = new Object[2];
Map<String, Object> map3 = new HashMap<>();
@@ -202,7 +218,7 @@ public class ComplexTypeTransformerTest {
// {
// "array.a":"v2"
// }]
- transformer = new ComplexTypeTransformer(Arrays.asList("array",
"array.array2"));
+ transformer = new ComplexTypeTransformer(Arrays.asList("array",
"array.array2"), ".");
genericRow = new GenericRow();
genericRow.putValue("array", array);
map1.put("array2", array2);
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/ingestion/ComplexTypeConfig.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/ingestion/ComplexTypeConfig.java
index 75b7c24..770c6cb 100644
---
a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/ingestion/ComplexTypeConfig.java
+++
b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/ingestion/ComplexTypeConfig.java
@@ -34,13 +34,23 @@ public class ComplexTypeConfig extends BaseJsonConfig {
@JsonPropertyDescription("The fields to unnest")
private final List<String> _unnestFields;
+ @JsonPropertyDescription("The delimiter used to separate components in a
path")
+ private final String _delimiter;
+
@JsonCreator
- public ComplexTypeConfig(@JsonProperty("unnestFields") @Nullable
List<String> unnestFields) {
+ public ComplexTypeConfig(@JsonProperty("unnestFields") @Nullable
List<String> unnestFields,
+ @JsonProperty("delimiter") @Nullable String delimiter) {
_unnestFields = unnestFields;
+ _delimiter = delimiter;
}
@Nullable
public List<String> getUnnestFields() {
return _unnestFields;
}
+
+ @Nullable
+ public String getDelimiter() {
+ return _delimiter;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
