This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 467b01f2ce4 Include record when filter transform throws exception
(#17218)
467b01f2ce4 is described below
commit 467b01f2ce42853dfe6b4361b2de60f5abebdede
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Tue Nov 18 10:34:56 2025 -0800
Include record when filter transform throws exception (#17218)
---
.../segment/local/recordtransformer/FilterTransformer.java | 1 +
.../local/recordtransformer/RecordTransformerTest.java | 14 ++++++++++++++
2 files changed, 15 insertions(+)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/FilterTransformer.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/FilterTransformer.java
index 3b215ed7e21..79915c2b894 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/FilterTransformer.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/FilterTransformer.java
@@ -85,6 +85,7 @@ public class FilterTransformer implements RecordTransformer {
LOGGER.debug("Caught exception while executing filter function: {}
for record: {}", _filterFunction,
record.toString(), e);
record.markIncomplete();
+ filteredRecords.add(record);
}
}
}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/recordtransformer/RecordTransformerTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/recordtransformer/RecordTransformerTest.java
index 2247727cf62..a790883bce0 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/recordtransformer/RecordTransformerTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/recordtransformer/RecordTransformerTest.java
@@ -164,6 +164,20 @@ public class RecordTransformerTest {
// expected
}
+ // invalid function at runtime
+ ingestionConfig.setFilterConfig(new FilterConfig("svInt = 'abc'"));
+ transformer = new FilterTransformer(tableConfig);
+ try {
+ transformer.transform(List.of(genericRow));
+ fail("Should have failed executing function");
+ } catch (Exception e) {
+ // expected
+ }
+ ingestionConfig.setContinueOnError(true);
+ transformer = new FilterTransformer(tableConfig);
+ assertFalse(transformer.transform(List.of(genericRow)).isEmpty());
+ assertEquals(transformer.getNumRecordsFiltered(), 0);
+
// multi value column
ingestionConfig.setFilterConfig(new FilterConfig("Groovy({svFloat.max() <
500}, svFloat)"));
transformer = new FilterTransformer(tableConfig);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
