This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new a68a44b6c11 Trim of redundant IS_TRUE function when creation filter on
leaf stage (#16966)
a68a44b6c11 is described below
commit a68a44b6c1183ef75bb05640a9425dae8da9ea90
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Wed Oct 8 14:47:47 2025 -0700
Trim of redundant IS_TRUE function when creation filter on leaf stage
(#16966)
---
.../request/context/RequestContextUtils.java | 40 +++++++++++++++------
.../tests/NullHandlingIntegrationTest.java | 41 ++++++++++++++++++++++
2 files changed, 71 insertions(+), 10 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/request/context/RequestContextUtils.java
b/pinot-common/src/main/java/org/apache/pinot/common/request/context/RequestContextUtils.java
index 4517b1f43d0..fabaa72b100 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/request/context/RequestContextUtils.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/request/context/RequestContextUtils.java
@@ -106,11 +106,21 @@ public class RequestContextUtils {
* missing an EQUALS filter operator.
*/
public static FilterContext getFilter(Expression thriftExpression) {
+ Function function = thriftExpression.getFunctionCall();
+ // Trim off outer IS_TRUE function as it is redundant
+ if (function != null && function.getOperator().equals("istrue")) {
+ return getFilter(function.getOperands().get(0));
+ } else {
+ return getFilterInner(thriftExpression);
+ }
+ }
+
+ private static FilterContext getFilterInner(Expression thriftExpression) {
ExpressionType type = thriftExpression.getType();
switch (type) {
case FUNCTION:
Function thriftFunction = thriftExpression.getFunctionCall();
- return getFilter(thriftFunction);
+ return getFilterInner(thriftFunction);
case IDENTIFIER:
// Convert "WHERE a" to "WHERE a = true"
return FilterContext.forPredicate(new
EqPredicate(getExpression(thriftExpression), "true"));
@@ -121,7 +131,7 @@ public class RequestContextUtils {
}
}
- public static FilterContext getFilter(Function thriftFunction) {
+ private static FilterContext getFilterInner(Function thriftFunction) {
String functionOperator = thriftFunction.getOperator();
// convert "WHERE startsWith(col, 'str')" to "WHERE startsWith(col, 'str')
= true"
@@ -137,7 +147,7 @@ public class RequestContextUtils {
case AND: {
List<FilterContext> children = new ArrayList<>(numOperands);
for (Expression operand : operands) {
- FilterContext filter = getFilter(operand);
+ FilterContext filter = getFilterInner(operand);
if (!filter.isConstant()) {
children.add(filter);
} else {
@@ -158,7 +168,7 @@ public class RequestContextUtils {
case OR: {
List<FilterContext> children = new ArrayList<>(numOperands);
for (Expression operand : operands) {
- FilterContext filter = getFilter(operand);
+ FilterContext filter = getFilterInner(operand);
if (!filter.isConstant()) {
children.add(filter);
} else {
@@ -178,7 +188,7 @@ public class RequestContextUtils {
}
case NOT: {
assert numOperands == 1;
- FilterContext filter = getFilter(operands.get(0));
+ FilterContext filter = getFilterInner(operands.get(0));
if (!filter.isConstant()) {
return FilterContext.forNot(filter);
} else {
@@ -287,11 +297,21 @@ public class RequestContextUtils {
* missing an EQUALS filter operator.
*/
public static FilterContext getFilter(ExpressionContext filterExpression) {
+ FunctionContext function = filterExpression.getFunction();
+ // Trim off outer IS_TRUE function as it is redundant
+ if (function != null && function.getFunctionName().equals("istrue")) {
+ return getFilter(function.getArguments().get(0));
+ } else {
+ return getFilterInner(filterExpression);
+ }
+ }
+
+ private static FilterContext getFilterInner(ExpressionContext
filterExpression) {
ExpressionContext.Type type = filterExpression.getType();
switch (type) {
case FUNCTION:
FunctionContext filterFunction = filterExpression.getFunction();
- return getFilter(filterFunction);
+ return getFilterInner(filterFunction);
case IDENTIFIER:
return FilterContext.forPredicate(
new EqPredicate(filterExpression,
getStringValue(RequestUtils.getLiteralExpression(true))));
@@ -302,7 +322,7 @@ public class RequestContextUtils {
}
}
- public static FilterContext getFilter(FunctionContext filterFunction) {
+ private static FilterContext getFilterInner(FunctionContext filterFunction) {
String functionOperator = filterFunction.getFunctionName().toUpperCase();
// convert "WHERE startsWith(col, 'str')" to "WHERE startsWith(col, 'str')
= true"
@@ -317,7 +337,7 @@ public class RequestContextUtils {
case AND: {
List<FilterContext> children = new ArrayList<>(numOperands);
for (ExpressionContext operand : operands) {
- FilterContext filter = getFilter(operand);
+ FilterContext filter = getFilterInner(operand);
if (!filter.isConstant()) {
children.add(filter);
} else {
@@ -338,7 +358,7 @@ public class RequestContextUtils {
case OR: {
List<FilterContext> children = new ArrayList<>(numOperands);
for (ExpressionContext operand : operands) {
- FilterContext filter = getFilter(operand);
+ FilterContext filter = getFilterInner(operand);
if (!filter.isConstant()) {
children.add(filter);
} else {
@@ -358,7 +378,7 @@ public class RequestContextUtils {
}
case NOT: {
assert numOperands == 1;
- FilterContext filter = getFilter(operands.get(0));
+ FilterContext filter = getFilterInner(operands.get(0));
if (!filter.isConstant()) {
return FilterContext.forNot(filter);
} else {
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/NullHandlingIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/NullHandlingIntegrationTest.java
index 9e31c21292b..c25af339add 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/NullHandlingIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/NullHandlingIntegrationTest.java
@@ -459,4 +459,45 @@ public class NullHandlingIntegrationTest extends
BaseClusterIntegrationTestSet
{String.format("SELECT tan(null) FROM %s", getTableName()), "null"}
};
}
+
+ /// This test ensures IS_TRUE can be trimmed off on leaf stage
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testFilteredAggregationNoScanInFilter(boolean
useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+
+ String query = "SELECT city, COUNT(*), COUNT(*) FILTER(WHERE description =
'unknown') FROM mytable GROUP BY city";
+
+ if (useMultiStageQueryEngine) {
+ // MSE will insert IS_TRUE to the aggregate filter
+ explainLogical(query,
+ "Execution Plan\n"
+ + "PinotLogicalAggregate(group=[{0}], agg#0=[COUNT($1)],
agg#1=[COUNT($2)], aggType=[FINAL])\n"
+ + " PinotLogicalExchange(distribution=[hash[0]])\n"
+ + " PinotLogicalAggregate(group=[{0}], agg#0=[COUNT()],
agg#1=[COUNT() FILTER $1], aggType=[LEAF])\n"
+ + " LogicalProject(city=[$5], $f1=[IS TRUE(=($7,
_UTF-8'unknown'))])\n"
+ + " PinotLogicalTableScan(table=[[default, mytable]])\n");
+ // IS_TRUE should be trimmed off, then the filter becomes always false
in the server execution plan
+ explainAskingServers(query,
+ "Execution Plan\n"
+ + "PinotLogicalAggregate(group=[{0}], agg#0=[COUNT($1)],
agg#1=[COUNT($2)], aggType=[FINAL])\n"
+ + " PinotLogicalExchange(distribution=[hash[0]])\n"
+ + " LeafStageCombineOperator(table=[mytable])\n"
+ + " StreamingInstanceResponse\n"
+ + " CombineGroupBy\n"
+ + " GroupByFiltered(groupKeys=[[city]],
aggregations=[[count(*), count(*)]])\n"
+ + " Project(columns=[[city]])\n"
+ + " DocIdSet(maxDocs=[20000])\n"
+ + " FilterEmpty\n"
+ + " Project(columns=[[city]])\n"
+ + " DocIdSet(maxDocs=[20000])\n"
+ + " FilterMatchEntireSegment(numDocs=[100])\n"
+ );
+ }
+
+ // There should be no scan in the filter since description = 'unknown'
matches no value
+ JsonNode response = postQuery(query);
+ assertNoError(response);
+ assertEquals(response.get("numEntriesScannedInFilter").asLong(), 0L);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
