This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 624b90b6d15 Fix ExpressionTransformer collection handling (#17363)
624b90b6d15 is described below
commit 624b90b6d15073257221426ec74cd014145d994e
Author: Xiang Fu <[email protected]>
AuthorDate: Fri Dec 12 16:42:56 2025 -0800
Fix ExpressionTransformer collection handling (#17363)
---
.../recordtransformer/ExpressionTransformer.java | 6 +--
.../ExpressionTransformerTest.java | 56 ++++++++++++++++++++++
2 files changed, 59 insertions(+), 3 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/ExpressionTransformer.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/ExpressionTransformer.java
index 7c62372f409..3e7649f663f 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/ExpressionTransformer.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/ExpressionTransformer.java
@@ -20,7 +20,7 @@ package org.apache.pinot.segment.local.recordtransformer;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import java.util.Collections;
+import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
@@ -154,7 +154,7 @@ public class ExpressionTransformer implements
RecordTransformer {
_throttledLogger.warn("Caught exception while evaluation transform
function for column: " + column, e);
record.markIncomplete();
}
- } else if (existingValue.getClass().isArray() || existingValue
instanceof Collections
+ } else if (existingValue.getClass().isArray() || existingValue
instanceof Collection
|| existingValue instanceof Map) {
try {
Object transformedValue =
transformFunctionEvaluator.evaluate(record);
@@ -174,7 +174,7 @@ public class ExpressionTransformer implements
RecordTransformer {
if (transformedValue.getClass() == existingValue.getClass()) {
return true;
}
- if (transformedValue instanceof Collections && existingValue instanceof
Collections) {
+ if (transformedValue instanceof Collection && existingValue instanceof
Collection) {
return true;
}
if (transformedValue instanceof Map && existingValue instanceof Map) {
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/recordtransformer/ExpressionTransformerTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/recordtransformer/ExpressionTransformerTest.java
index 70935130b92..c8601a71a6e 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/recordtransformer/ExpressionTransformerTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/recordtransformer/ExpressionTransformerTest.java
@@ -241,6 +241,34 @@ public class ExpressionTransformerTest {
Assert.assertEquals(genericRow.getValue("outgoing"), "123");
}
+ @Test
+ public void testExistingCollectionIsTransformedWhenIncompatibleType() {
+ Schema schema = new Schema.SchemaBuilder()
+ .addMultiValueDimension("rawBids", FieldSpec.DataType.INT)
+ .addMultiValueDimension("bids", FieldSpec.DataType.INT)
+ .build();
+
+ IngestionConfig ingestionConfig = new IngestionConfig();
+ ingestionConfig.setTransformConfigs(Collections.singletonList(
+ new TransformConfig("bids", "Groovy({rawBids.toArray()}, rawBids)")));
+ TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE)
+ .setTableName("testExistingCollectionIsTransformed")
+ .setIngestionConfig(ingestionConfig)
+ .build();
+
+ ExpressionTransformer expressionTransformer = new
ExpressionTransformer(tableConfig, schema);
+ GenericRow genericRow = new GenericRow();
+ genericRow.putValue("rawBids", Arrays.asList(1, 2, 3));
+ // Simulate pre-existing collection value that should be overwritten by
transform
+ genericRow.putValue("bids", Arrays.asList(10, 20));
+
+ expressionTransformer.transform(genericRow);
+
+ Object transformedValue = genericRow.getValue("bids");
+ Assert.assertTrue(transformedValue.getClass().isArray());
+ Assert.assertEquals(Arrays.asList((Object[]) transformedValue),
Arrays.asList(1, 2, 3));
+ }
+
@Test
public void testTransformFunctionSortOrder() {
Schema schema = new Schema.SchemaBuilder().addSingleValueDimension("a",
FieldSpec.DataType.STRING)
@@ -430,4 +458,32 @@ public class ExpressionTransformerTest {
Assert.assertEquals(map.get("a"), 1);
Assert.assertEquals(map.get("b"), "x");
}
+
+ @Test
+ public void testJsonToArrayIngestionTransform() {
+ Schema schema = new Schema.SchemaBuilder()
+ .addSingleValueDimension("columnJson", FieldSpec.DataType.STRING)
+ .addMultiValueDimension("columnArray", FieldSpec.DataType.STRING)
+ .build();
+
+ IngestionConfig ingestionConfig = new IngestionConfig();
+ ingestionConfig.setTransformConfigs(Collections.singletonList(
+ new TransformConfig("columnArray", "jsonPathArray(columnJson, '$')")));
+ TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE)
+ .setTableName("testJsonToArrayIngestionTransform")
+ .setIngestionConfig(ingestionConfig)
+ .build();
+
+ ExpressionTransformer expressionTransformer = new
ExpressionTransformer(tableConfig, schema);
+
+ GenericRow row = new GenericRow();
+ row.putValue("columnJson", "[\"a\",\"b\",\"c\"]");
+ // Pre-existing collection should be overwritten because transform returns
an array (incompatible type)
+ row.putValue("columnArray", Arrays.asList("preExisting"));
+
+ expressionTransformer.transform(row);
+ Object transformedValue = row.getValue("columnArray");
+ Assert.assertTrue(transformedValue.getClass().isArray());
+ Assert.assertEquals(Arrays.asList((Object[]) transformedValue),
Arrays.asList("a", "b", "c"));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
