This is an automated email from the ASF dual-hosted git repository.
yashmayya pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new e2ff6cc7ded Clean up SSE query and statement optimizer handling
(#16991)
e2ff6cc7ded is described below
commit e2ff6cc7dedd2aa4b63f758898bbe089cd0ca877
Author: Yash Mayya <[email protected]>
AuthorDate: Thu Oct 9 16:51:57 2025 -0700
Clean up SSE query and statement optimizer handling (#16991)
---
.../BaseSingleStageBrokerRequestHandler.java | 5 +-
.../pinot/core/query/optimizer/QueryOptimizer.java | 10 +-
.../statement/JsonStatementOptimizer.java | 610 ---------------------
.../optimizer/statement/StatementOptimizer.java | 3 +-
.../statement/StringPredicateFilterOptimizer.java | 3 +-
.../statement/JsonStatementOptimizerTest.java | 241 --------
.../StringPredicateFilterOptimizerTest.java | 16 +-
.../core/query/optimizer/statement/TestHelper.java | 7 +-
.../apache/pinot/queries/BaseJsonQueryTest.java | 2 +-
.../org/apache/pinot/queries/BaseQueriesTest.java | 5 +-
.../pinot/queries/JsonMalformedIndexTest.java | 2 +-
.../pinot/queries/TextSearchQueriesTest.java | 6 +-
...BenchmarkAggregateGroupByOrderByQueriesSSE.java | 6 +-
.../plan/server/ServerPlanRequestUtils.java | 2 +-
14 files changed, 25 insertions(+), 893 deletions(-)
diff --git
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseSingleStageBrokerRequestHandler.java
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseSingleStageBrokerRequestHandler.java
index 4cf79bd233d..e1c4d387768 100644
---
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseSingleStageBrokerRequestHandler.java
+++
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseSingleStageBrokerRequestHandler.java
@@ -563,7 +563,6 @@ public abstract class BaseSingleStageBrokerRequestHandler
extends BaseBrokerRequ
attachTimeBoundary(offlinePinotQuery, timeBoundaryInfo, true);
handleExpressionOverride(offlinePinotQuery,
_tableCache.getExpressionOverrideMap(offlineTableName));
handleTimestampIndexOverride(offlinePinotQuery, offlineTableConfig);
- _queryOptimizer.optimize(offlinePinotQuery, offlineTableConfig, schema);
offlineBrokerRequest =
CalciteSqlCompiler.convertToBrokerRequest(offlinePinotQuery);
PinotQuery realtimePinotQuery = serverPinotQuery.deepCopy();
@@ -571,7 +570,6 @@ public abstract class BaseSingleStageBrokerRequestHandler
extends BaseBrokerRequ
attachTimeBoundary(realtimePinotQuery, timeBoundaryInfo, false);
handleExpressionOverride(realtimePinotQuery,
_tableCache.getExpressionOverrideMap(realtimeTableName));
handleTimestampIndexOverride(realtimePinotQuery, realtimeTableConfig);
- _queryOptimizer.optimize(realtimePinotQuery, realtimeTableConfig,
schema);
realtimeBrokerRequest =
CalciteSqlCompiler.convertToBrokerRequest(realtimePinotQuery);
requestContext.setFanoutType(RequestContext.FanoutType.HYBRID);
@@ -582,7 +580,6 @@ public abstract class BaseSingleStageBrokerRequestHandler
extends BaseBrokerRequ
setTableName(serverBrokerRequest, offlineTableName);
handleExpressionOverride(serverPinotQuery,
_tableCache.getExpressionOverrideMap(offlineTableName));
handleTimestampIndexOverride(serverPinotQuery, offlineTableConfig);
- _queryOptimizer.optimize(serverPinotQuery, offlineTableConfig, schema);
offlineBrokerRequest = serverBrokerRequest;
requestContext.setFanoutType(RequestContext.FanoutType.OFFLINE);
@@ -592,7 +589,6 @@ public abstract class BaseSingleStageBrokerRequestHandler
extends BaseBrokerRequ
setTableName(serverBrokerRequest, realtimeTableName);
handleExpressionOverride(serverPinotQuery,
_tableCache.getExpressionOverrideMap(realtimeTableName));
handleTimestampIndexOverride(serverPinotQuery, realtimeTableConfig);
- _queryOptimizer.optimize(serverPinotQuery, realtimeTableConfig, schema);
realtimeBrokerRequest = serverBrokerRequest;
requestContext.setFanoutType(RequestContext.FanoutType.REALTIME);
@@ -1009,6 +1005,7 @@ public abstract class BaseSingleStageBrokerRequestHandler
extends BaseBrokerRequ
if (schema != null) {
handleDistinctMultiValuedOverride(serverPinotQuery, schema);
}
+ _queryOptimizer.optimize(serverPinotQuery, schema);
return new CompileResult(pinotQuery, serverPinotQuery, schema, tableName,
rawTableName);
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/QueryOptimizer.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/QueryOptimizer.java
index dbd9b5c70c6..a465249ec56 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/QueryOptimizer.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/QueryOptimizer.java
@@ -34,7 +34,6 @@ import
org.apache.pinot.core.query.optimizer.filter.TextMatchFilterOptimizer;
import
org.apache.pinot.core.query.optimizer.filter.TimePredicateFilterOptimizer;
import org.apache.pinot.core.query.optimizer.statement.StatementOptimizer;
import
org.apache.pinot.core.query.optimizer.statement.StringPredicateFilterOptimizer;
-import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.Schema;
@@ -57,13 +56,6 @@ public class QueryOptimizer {
* Optimizes the given query.
*/
public void optimize(PinotQuery pinotQuery, @Nullable Schema schema) {
- optimize(pinotQuery, null, schema);
- }
-
- /**
- * Optimizes the given query.
- */
- public void optimize(PinotQuery pinotQuery, @Nullable TableConfig
tableConfig, @Nullable Schema schema) {
Expression filterExpression = pinotQuery.getFilterExpression();
if (filterExpression != null) {
for (FilterOptimizer filterOptimizer : FILTER_OPTIMIZERS) {
@@ -74,7 +66,7 @@ public class QueryOptimizer {
// Run statement optimizer after filter has already been optimized.
for (StatementOptimizer statementOptimizer : STATEMENT_OPTIMIZERS) {
- statementOptimizer.optimize(pinotQuery, tableConfig, schema);
+ statementOptimizer.optimize(pinotQuery, schema);
}
}
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/statement/JsonStatementOptimizer.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/statement/JsonStatementOptimizer.java
deleted file mode 100644
index 138e55b3b32..00000000000
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/statement/JsonStatementOptimizer.java
+++ /dev/null
@@ -1,610 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.core.query.optimizer.statement;
-
-import java.lang.reflect.Method;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import javax.annotation.Nullable;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.pinot.common.function.scalar.ArithmeticFunctions;
-import org.apache.pinot.common.function.scalar.DateTimeFunctions;
-import org.apache.pinot.common.request.Expression;
-import org.apache.pinot.common.request.ExpressionType;
-import org.apache.pinot.common.request.Function;
-import org.apache.pinot.common.request.Identifier;
-import org.apache.pinot.common.request.Literal;
-import org.apache.pinot.common.request.PinotQuery;
-import org.apache.pinot.common.utils.DataSchema;
-import org.apache.pinot.common.utils.request.RequestUtils;
-import org.apache.pinot.segment.spi.AggregationFunctionType;
-import org.apache.pinot.spi.config.table.IndexingConfig;
-import org.apache.pinot.spi.config.table.JsonIndexConfig;
-import org.apache.pinot.spi.config.table.TableConfig;
-import org.apache.pinot.spi.data.FieldSpec;
-import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.sql.FilterKind;
-
-
-/**
- * NOTE: This class is not in an active code path, but exists to allow us to
experiment and gain better understanding
- * of identifier dot notation usage in SQL.
- *
- * This class will rewrite a query that has json path expressions into a query
that uses JSON_EXTRACT_SCALAR and
- * JSON_MATCH functions.
- *
- * Example 1:
- * From : SELECT jsonColumn.name.first
- * FROM testTable
- * WHERE jsonColumn.name.first IS NOT NULL
- * TO : SELECT JSON_EXTRACT_SCALAR(jsonColumn, '$.name.first', 'STRING',
'null') AS jsonColum.name.first
- * FROM testTable
- * WHERE JSON_MATCH('"$.name.first" IS NOT NULL')
- *
- * Output datatype of any json path expression is 'STRING'. However, if json
path expression appears as an argument to
- * a numerical function, then output of json path expression is set to
'DOUBLE' as shown in the example below.
- *
- * Example 2:
- * From: SELECT MIN(jsonColumn.id - 5)
- * FROM testTable
- * WHERE jsonColumn.id IS NOT NULL
- * To: SELECT MIN(MINUS(JSON_EXTRACT_SCALAR(jsonColumn, '$.id',
'DOUBLE', Double.NEGATIVE_INFINITY),5)) AS min
- * (minus(jsonColum.id, '5'))
- * FROM testTable
- * WHERE JSON_MATCH('"$.id" IS NOT NULL')
- *
- * Example 3:
- * From: SELECT jsonColumn.id, count(*)
- * FROM testTable
- * WHERE jsonColumn.name.first = 'Daffy' OR jsonColumn.id = 101
- * GROUP BY jsonColumn.id
- * To: SELECT JSON_EXTRACT_SCALAR(jsonColumn, '$.id', 'STRING', 'null')
AS jsonColumn.id, count(*)
- * FROM testTable
- * WHERE JSON_MATCH('"$.name.first" = ''Daffy''') OR
JSON_MATCH('"$.id" = 101')
- * GROUP BY JSON_EXTRACT_SCALAR(jsonColumn, '$.id', 'STRING', 'null');
- *
- * Example 4:
- * From: SELECT jsonColumn.name.last, count(*)
- * FROM testTable
- * GROUP BY jsonColumn.name.last
- * HAVING jsonColumn.name.last = 'mouse'
- * To: SELECT JSON_EXTRACT_SCALAR(jsonColumn, '$.name.last', 'STRING',
'null') AS jsonColumn.name.last, count(*)
- * FROM testTable
- * GROUP BY JSON_EXTRACT_SCALAR(jsonColumn, '$.name.last', 'STRING',
'null')
- * HAVING JSON_EXTRACT_SCALAR(jsonColumn, '$.name.last', 'STRING',
'null') = 'mouse'
- *
- * Notes:
- * 1) In a filter expression, if json path appears on the left-hand side, the
right-hand side must be a literal. In
- * future this can be changed to have any expression on the right-hand side
by implementing a function that would
- * convert any {@link Expression} into SQL fragment that can be used in
JSON_MATCH. Currently only literals are
- * converted into SQL fragments {see @link #getLiteralSQL} function.
- * 2) In WHERE clause each json path expression will be replaced with a
JSON_MATCH function. If there are multiple
- * json path expressions, they will be replaced by multiple JSON_MATCH
functions. We currently don't fold multiple
- * JSON_MATCH functions into a single JSON_MATCH_FUNCTION.
- */
-public class JsonStatementOptimizer implements StatementOptimizer {
-
- /**
- * Maintain a list of numerical functions that requiring json path
expression to output numerical values. This allows
- * us to implicitly convert the output of json path expression to DOUBLE.
TODO: There are better ways of doing this
- * if we were to move to a new storage (currently STRING) for JSON or
functions were to pre-declare their input
- * data types.
- */
- private static final Set<String> NUMERICAL_FUNCTIONS =
getNumericalFunctionList();
-
- /**
- * A list of functions that require json path expression to output LONG
value. This allows us to implicitly convert
- * the output of json path expression to LONG.
- */
- private static final Set<String> DATETIME_FUNCTIONS =
getDateTimeFunctionList();
-
- @Override
- public void optimize(PinotQuery query, @Nullable TableConfig tableConfig,
@Nullable Schema schema) {
- // If schema doesn't have any JSON columns, there is no need to run this
optimizer.
- if (schema == null || !schema.hasJSONColumn()) {
- return;
- }
-
- // In SELECT clause, replace JSON path expressions with
JSON_EXTRACT_SCALAR function with an alias.
- List<Expression> expressions = query.getSelectList();
- for (Expression expression : expressions) {
- Pair<String, Boolean> result = optimizeJsonIdentifier(expression,
schema, DataSchema.ColumnDataType.STRING);
- if (expression.getType() == ExpressionType.FUNCTION &&
!expression.getFunctionCall().getOperator().equals("AS")
- && result.getRight()) {
- // Since this is not an AS function (user-specified alias) and the
function or its arguments contain json path
- // expression, set an alias for the expression after replacing json
path expression with JSON_EXTRACT_SCALAR
- // function.
- Function aliasFunction = getAliasFunction(result.getLeft(),
expression.getFunctionCall());
- expression.setFunctionCall(aliasFunction);
- }
- }
-
- // In WHERE clause, replace JSON path expressions with JSON_MATCH function.
- Expression filter = query.getFilterExpression();
- if (filter != null) {
- optimizeJsonPredicate(filter, tableConfig, schema);
- }
-
- // In GROUP BY clause, replace JSON path expressions with
JSON_EXTRACT_SCALAR function without an alias.
- expressions = query.getGroupByList();
- if (expressions != null) {
- for (Expression expression : expressions) {
- optimizeJsonIdentifier(expression, schema,
DataSchema.ColumnDataType.STRING);
- }
- }
-
- // In ORDER BY clause, replace JSON path expression with
JSON_EXTRACT_SCALAR. This expression must match the
- // corresponding SELECT list expression except for the alias.
- expressions = query.getOrderByList();
- if (expressions != null) {
- for (Expression expression : expressions) {
- optimizeJsonIdentifier(expression, schema,
DataSchema.ColumnDataType.STRING);
- }
- }
-
- // In HAVING clause, replace JSON path expressions with
JSON_EXTRACT_SCALAR. This expression must match the
- // corresponding SELECT list expression except for the alias.
- Expression expression = query.getHavingExpression();
- if (expression != null) {
- optimizeJsonIdentifier(expression, schema,
DataSchema.ColumnDataType.STRING);
- }
- }
-
- /**
- * Replace an json path expression with an aliased JSON_EXTRACT_SCALAR
function.
- * @param expression input expression to rewrite into JSON_EXTRACT_SCALAR
function if the expression is json path.
- * @param outputDataType to keep track of output datatype of
JSON_EXTRACT_SCALAR function which depends upon the outer
- * function that json path expression appears in.
- * @return A {@link Pair} of values where the first value is alias for the
input expression and second
- * value indicates whether json path expression was found (true) or not
(false) in the expression.
- */
- private static Pair<String, Boolean> optimizeJsonIdentifier(Expression
expression, @Nullable Schema schema,
- DataSchema.ColumnDataType outputDataType) {
- switch (expression.getType()) {
- case LITERAL:
- return Pair.of(getLiteralSQL(expression.getLiteral(), true), false);
- case IDENTIFIER: {
- boolean hasJsonPathExpression = false;
- String columnName = expression.getIdentifier().getName();
- if (!schema.hasColumn(columnName)) {
- String[] parts = getIdentifierParts(expression.getIdentifier());
- if (parts.length > 1 && isValidJSONColumn(parts[0], schema)) {
- // replace <column-name>.<json-path> with
json_extract_scalar(<column-name>, '<json-path>', 'STRING',
- // <JSON-null-value>)
- Function jsonExtractScalarFunction = getJsonExtractFunction(parts,
outputDataType);
- expression.setIdentifier(null);
- expression.setType(ExpressionType.FUNCTION);
- expression.setFunctionCall(jsonExtractScalarFunction);
- hasJsonPathExpression = true;
- }
- }
- return Pair.of(columnName, hasJsonPathExpression);
- }
- case FUNCTION: {
- Function function = expression.getFunctionCall();
- List<Expression> operands = function.getOperands();
-
- boolean hasJsonPathExpression = false;
- StringBuffer alias = new StringBuffer();
- if (function.getOperator().toUpperCase().equals("AS")) {
- // We don't need to compute an alias for AS function since AS
function defines its own alias.
- hasJsonPathExpression = optimizeJsonIdentifier(operands.get(0),
schema, outputDataType).getRight();
-
alias.append(function.getOperands().get(1).getIdentifier().getName());
- } else {
- // For all functions besides AS function, process the operands and
compute the alias.
- alias.append(function.getOperator().toLowerCase()).append("(");
-
- // Output datatype of JSON_EXTRACT_SCALAR will depend upon the
function within which json path expression
- // appears.
- outputDataType = getJsonExtractOutputDataType(function);
-
- for (int i = 0; i < operands.size(); i++) {
- // recursively check to see if there is a
<json-column>.<json-path> identifier in this expression.
- Pair<String, Boolean> operandResult =
optimizeJsonIdentifier(operands.get(i), schema, outputDataType);
- hasJsonPathExpression |= operandResult.getRight();
- if (i > 0) {
- alias.append(",");
- }
- alias.append(operandResult.getLeft());
- }
- alias.append(")");
- }
-
- return Pair.of(alias.toString(), hasJsonPathExpression);
- }
- default:
- break;
- }
-
- return Pair.of("", false);
- }
-
- /**
- * Example:
- * Input:
- * alias : "jsoncolumn.x.y.z",
- * function: JSON_EXTRACT_SCALAR('jsoncolumn', 'x.y.z', 'STRING', 'null')
- * Output: AS(JSON_EXTRACT_SCALAR('jsoncolumn', 'x.y.z', 'STRING',
'null'), 'jsoncolumn.x.y.z')
- *
- * @return a Function with "AS" operator that wraps another function.
- */
- private static Function getAliasFunction(String alias, Function function) {
- Function aliasFunction = new Function("as");
-
- List<Expression> operands = new ArrayList<>();
- Expression expression = new Expression(ExpressionType.FUNCTION);
- expression.setFunctionCall(function);
- operands.add(expression);
- operands.add(RequestUtils.getIdentifierExpression(alias));
- aliasFunction.setOperands(operands);
-
- return aliasFunction;
- }
-
- /**
- * Example:
- * Input : ["jsoncolumn", "x","y","z[2]"]
- * Output: JSON_EXTRACT_SCALAR('jsoncolumn','$.x.y.z[2]','STRING','null')
- *
- * @param parts All the subparts of a fully qualified identifier (json path
expression).
- * @param dataType Output datatype of JSON_EXTRACT_SCALAR function.
- * @return a Function with JSON_EXTRACT_SCALAR operator created using parts
of fully qualified identifier name.
- */
- private static Function getJsonExtractFunction(String[] parts,
DataSchema.ColumnDataType dataType) {
- Function jsonExtractScalarFunction = new Function("jsonextractscalar");
- List<Expression> operands = new ArrayList<>();
- operands.add(RequestUtils.getIdentifierExpression(parts[0]));
- operands.add(RequestUtils.getLiteralExpression(getJsonPath(parts, false)));
- operands.add(RequestUtils.getLiteralExpression(dataType.toString()));
-
operands.add(RequestUtils.getLiteralExpression(getDefaultNullValueForType(dataType)));
- jsonExtractScalarFunction.setOperands(operands);
- return jsonExtractScalarFunction;
- }
-
- /**
- * Example 1:
- * Input : "jsonColumn.name.first = 'daffy'"
- * Output: "JSON_MATCH(jsonColumn, '\"$.name.first\" = ''daffy''').
- *
- * Example 2:
- * Input : "jsonColumn.id = 4"
- * Output: "JSON_MATCH(jsonColumn, '\"$.id\" = 4')
- */
- private static void optimizeJsonPredicate(Expression expression, @Nullable
TableConfig tableConfig,
- @Nullable Schema schema) {
- if (expression.getType() == ExpressionType.FUNCTION) {
- Function function = expression.getFunctionCall();
- String operator = function.getOperator();
- FilterKind kind = FilterKind.valueOf(operator);
- List<Expression> operands = function.getOperands();
- switch (kind) {
- case AND:
- case OR:
- case NOT:
- operands.forEach(operand -> optimizeJsonPredicate(operand,
tableConfig, schema));
- break;
- case EQUALS:
- case NOT_EQUALS:
- case GREATER_THAN:
- case GREATER_THAN_OR_EQUAL:
- case LESS_THAN:
- case LESS_THAN_OR_EQUAL: {
- Expression left = operands.get(0);
- Expression right = operands.get(1);
- if (left.getType() == ExpressionType.IDENTIFIER && right.getType()
== ExpressionType.LITERAL) {
- if (!schema.hasColumn(left.getIdentifier().getName())) {
- String[] parts = getIdentifierParts(left.getIdentifier());
- if (parts.length > 1 && isValidJSONColumn(parts[0], schema)) {
- if (isIndexedJSONColumn(parts[0], tableConfig)) {
- Function jsonMatchFunction = new
Function(FilterKind.JSON_MATCH.name());
-
- List<Expression> jsonMatchFunctionOperands = new
ArrayList<>();
-
jsonMatchFunctionOperands.add(RequestUtils.getIdentifierExpression(parts[0]));
-
jsonMatchFunctionOperands.add(RequestUtils.getLiteralExpression(
- getJsonPath(parts, true) + getOperatorSQL(kind) +
getLiteralSQL(right.getLiteral(), false)));
- jsonMatchFunction.setOperands(jsonMatchFunctionOperands);
-
- expression.setFunctionCall(jsonMatchFunction);
- } else {
- left.clear();
- left.setType(ExpressionType.FUNCTION);
- left.setFunctionCall(getJsonExtractFunction(parts,
getColumnTypeForLiteral(right.getLiteral())));
- }
- }
- }
- }
- break;
- }
- case IS_NULL:
- case IS_NOT_NULL: {
- Expression operand = operands.get(0);
- if (operand.getType() == ExpressionType.IDENTIFIER) {
- if (!schema.hasColumn(operand.getIdentifier().getName())) {
- String[] parts = getIdentifierParts(operand.getIdentifier());
- if (parts.length > 1 && isValidJSONColumn(parts[0], schema)) {
- if (isIndexedJSONColumn(parts[0], tableConfig)) {
- Function jsonMatchFunction = new
Function(FilterKind.JSON_MATCH.name());
-
- List<Expression> jsonMatchFunctionOperands = new
ArrayList<>();
-
jsonMatchFunctionOperands.add(RequestUtils.getIdentifierExpression(parts[0]));
- jsonMatchFunctionOperands.add(
- RequestUtils.getLiteralExpression(getJsonPath(parts,
true) + getOperatorSQL(kind)));
- jsonMatchFunction.setOperands(jsonMatchFunctionOperands);
-
- expression.setFunctionCall(jsonMatchFunction);
- } else {
- operand.clear();
- operand.setType(ExpressionType.FUNCTION);
- operand.setFunctionCall(getJsonExtractFunction(parts,
DataSchema.ColumnDataType.JSON));
- }
- }
- }
- }
- break;
- }
- default:
- break;
- }
- }
- }
-
- /**
- * @return A two element String array where the first element is the column
name and second element is the JSON
- * path expression. If column name is not suffixed by JSON path expression,
then array will contain only a single
- * element representing the column name. For example:
- * 1) Identifier "jsonColumn.name.first" -> {"jsonColumn", ".name.first"}
- * 2) Identifier "jsonColumn[0]" -> {"jsonColumn", "[0]"}
- * 3) Identifier "jsonColumn" -> {"jsonColumn"}
- */
- private static String[] getIdentifierParts(Identifier identifier) {
- String name = identifier.getName();
- int dotIndex = name.indexOf('.');
- int openBracketIndex = name.indexOf('[');
-
- // column name followed by top-level array expression.
- if (openBracketIndex != -1) {
- // name has an '[', check if this path expression refers to a top-level
JSON array.
- if (dotIndex == -1 || openBracketIndex < dotIndex) {
- // This path expression refers to a top-level JSON array.
- return new String[]{name.substring(0, openBracketIndex),
name.substring(openBracketIndex)};
- }
- }
-
- // column name followed by all other JSON path expression
- if (dotIndex != -1) {
- return new String[]{name.substring(0, dotIndex),
name.substring(dotIndex)};
- }
-
- // column name without any JSON path expression
- return new String[]{name};
- }
-
- /**
- * Builds a json path expression when given identifier parts. For
example,given [jsonColumn, name, first], this
- * function will return "$.name.first" as json path expression.
- * @param parts identifier parts
- * @param applyDoubleQuote delimit json path with double quotes if true;
otherwise, don't delimit json path.
- * @return JSON path expression associated with the given identifier parts.
- */
- private static String getJsonPath(String[] parts, boolean applyDoubleQuote) {
- StringBuilder builder = new StringBuilder();
- if (applyDoubleQuote) {
- builder.append("\"");
- }
-
- builder.append("$");
- builder.append(parts[1]);
-
- if (applyDoubleQuote) {
- builder.append("\"");
- }
-
- return builder.toString();
- }
-
- /** @return true if specified column has column datatype of JSON; otherwise,
return false */
- private static boolean isValidJSONColumn(String columnName, @Nullable Schema
schema) {
- return schema != null && schema.hasColumn(columnName) &&
schema.getFieldSpecFor(columnName).getDataType()
- .equals(FieldSpec.DataType.JSON);
- }
-
- /** @return true if specified column has a JSON Index. */
- private static boolean isIndexedJSONColumn(String columnName, @Nullable
TableConfig tableConfig) {
- if (tableConfig == null) {
- return false;
- }
-
- IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
- if (indexingConfig == null) {
- return false;
- }
-
- // Ignore jsonIndexColumns when jsonIndexConfigs is configured
- Map<String, JsonIndexConfig> jsonIndexConfigs =
indexingConfig.getJsonIndexConfigs();
- if (jsonIndexConfigs != null) {
- return jsonIndexConfigs.containsKey(columnName);
- }
-
- List<String> jsonIndexColumns = indexingConfig.getJsonIndexColumns();
- if (jsonIndexColumns != null) {
- return jsonIndexColumns.contains(columnName);
- }
-
- return false;
- }
-
- /** @return symbolic representation of function operator delimited by
spaces. */
- private static String getOperatorSQL(FilterKind kind) {
- switch (kind) {
- case EQUALS:
- return " = ";
- case NOT_EQUALS:
- return " != ";
- case GREATER_THAN:
- return " > ";
- case GREATER_THAN_OR_EQUAL:
- return " >= ";
- case LESS_THAN:
- return " < ";
- case LESS_THAN_OR_EQUAL:
- return " <= ";
- case IN:
- return " IN ";
- case NOT_IN:
- return " NOT IN ";
- case IS_NULL:
- return " IS NULL";
- case IS_NOT_NULL:
- return " IS NOT NULL";
- default:
- return " ";
- }
- }
-
- /**
- * @param literal {@link Literal} to convert to a {@link String}.
- * @param aliasing When true, generate string for use in an alias;
otherwise, generate SQL string representation.
- * @return Literal value converted into either an alias name or an SQL
string. BYTE, STRING, and BINARY values are
- * delimited by quotes in SQL and everything is delimited by quotes for use
in alias.
- * */
- private static String getLiteralSQL(Literal literal, boolean aliasing) {
- StringBuffer result = new StringBuffer();
- result.append(aliasing ? "'" : "");
- switch (literal.getSetField()) {
- case BOOL_VALUE:
- result.append(String.valueOf(literal.getBinaryValue()));
- break;
- case BYTE_VALUE:
- result.append(
- aliasing ? String.valueOf(literal.getByteValue()) : "'" +
String.valueOf(literal.getByteValue()) + "'");
- break;
- case SHORT_VALUE:
- result.append(
- aliasing ? String.valueOf(literal.getShortValue()) : "'" +
String.valueOf(literal.getShortValue()) + "'");
- break;
- case INT_VALUE:
- result.append(String.valueOf(literal.getIntValue()));
- break;
- case LONG_VALUE:
- result.append(String.valueOf(literal.getLongValue()));
- break;
- case DOUBLE_VALUE:
- result.append(String.valueOf(literal.getDoubleValue()));
- break;
- case STRING_VALUE:
- result.append("'" + literal.getStringValue() + "'");
- break;
- case BINARY_VALUE:
- result.append(
- aliasing ? String.valueOf(literal.getBinaryValue()) : "'" +
String.valueOf(literal.getBinaryValue()) + "'");
- break;
- default:
- break;
- }
-
- result.append(aliasing ? "'" : "");
- return result.toString();
- }
-
- private static DataSchema.ColumnDataType getColumnTypeForLiteral(Literal
literal) {
- switch (literal.getSetField()) {
- case BOOL_VALUE:
- return DataSchema.ColumnDataType.BOOLEAN;
- case SHORT_VALUE:
- case INT_VALUE:
- case LONG_VALUE:
- return DataSchema.ColumnDataType.LONG;
- case DOUBLE_VALUE:
- return DataSchema.ColumnDataType.DOUBLE;
- case STRING_VALUE:
- return DataSchema.ColumnDataType.STRING;
- case BYTE_VALUE:
- case BINARY_VALUE:
- return DataSchema.ColumnDataType.BYTES;
- default:
- return DataSchema.ColumnDataType.STRING;
- }
- }
-
- /**
- * Returns the default null value for the given data type.
- */
- private static Object getDefaultNullValueForType(DataSchema.ColumnDataType
dataType) {
- switch (dataType) {
- case INT:
- return FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_INT;
- case LONG:
- return FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_LONG;
- case FLOAT:
- return FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_FLOAT;
- case DOUBLE:
- return FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_DOUBLE;
- case STRING:
- default:
- return FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_STRING;
- }
- }
-
- /** Output datatype of JSON_EXTRACT_SCALAR depends upon the function within
which json path expression appears. */
- private static DataSchema.ColumnDataType
getJsonExtractOutputDataType(Function function) {
- DataSchema.ColumnDataType dataType = DataSchema.ColumnDataType.STRING;
- if (NUMERICAL_FUNCTIONS.contains(function.getOperator().toUpperCase())) {
- // If json path expression appears as argument of a numeric function,
then it will be rewritten into a
- // JSON_EXTRACT_SCALAR function that returns 'DOUBLE'
- dataType = DataSchema.ColumnDataType.DOUBLE;
- } else if
(DATETIME_FUNCTIONS.contains(function.getOperator().toUpperCase())) {
- // If json path expression appears as argument of a datetime function,
then it will be rewritten into a
- // JSON_EXTRACT_SCALAR function that returns 'LONG'
- dataType = DataSchema.ColumnDataType.LONG;
- }
- return dataType;
- }
-
- /** List of function that require input to be in a number. */
- public static Set<String> getNumericalFunctionList() {
- Set<String> set = new HashSet<>();
- // Include all ArithmeticFunctions functions
- Method[] methods = ArithmeticFunctions.class.getDeclaredMethods();
- for (Method method : methods) {
- set.add(method.getName().toUpperCase());
- }
-
- // Include all aggregation functions
- AggregationFunctionType[] aggs = AggregationFunctionType.values();
- for (AggregationFunctionType agg : aggs) {
- set.add(agg.getName().toUpperCase());
- }
-
- return set;
- }
-
- /** List of DateTime functions which require input to be of long type. */
- public static Set<String> getDateTimeFunctionList() {
- Set<String> set = new HashSet<>();
- Method[] methods = DateTimeFunctions.class.getDeclaredMethods();
- for (Method method : methods) {
- set.add(method.getName().toUpperCase());
- }
-
- return set;
- }
-}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/statement/StatementOptimizer.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/statement/StatementOptimizer.java
index 361703be670..1d27b06707f 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/statement/StatementOptimizer.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/statement/StatementOptimizer.java
@@ -20,7 +20,6 @@ package org.apache.pinot.core.query.optimizer.statement;
import javax.annotation.Nullable;
import org.apache.pinot.common.request.PinotQuery;
-import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.Schema;
@@ -31,5 +30,5 @@ import org.apache.pinot.spi.data.Schema;
public interface StatementOptimizer {
/** Optimize the given SQL statement. */
- void optimize(PinotQuery query, @Nullable TableConfig tableConfig, @Nullable
Schema schema);
+ void optimize(PinotQuery query, @Nullable Schema schema);
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/statement/StringPredicateFilterOptimizer.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/statement/StringPredicateFilterOptimizer.java
index d8494684acf..7e1eb6c540f 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/statement/StringPredicateFilterOptimizer.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/statement/StringPredicateFilterOptimizer.java
@@ -26,7 +26,6 @@ import org.apache.pinot.common.request.Expression;
import org.apache.pinot.common.request.ExpressionType;
import org.apache.pinot.common.request.Function;
import org.apache.pinot.common.request.PinotQuery;
-import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.sql.FilterKind;
@@ -48,7 +47,7 @@ public class StringPredicateFilterOptimizer implements
StatementOptimizer {
private static final String STRCMP_OPERATOR_NAME = "strcmp";
@Override
- public void optimize(PinotQuery query, @Nullable TableConfig tableConfig,
@Nullable Schema schema) {
+ public void optimize(PinotQuery query, @Nullable Schema schema) {
if (schema == null) {
return;
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/statement/JsonStatementOptimizerTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/statement/JsonStatementOptimizerTest.java
deleted file mode 100644
index 250016d0717..00000000000
---
a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/statement/JsonStatementOptimizerTest.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.core.query.optimizer.statement;
-
-import java.util.Arrays;
-import org.apache.pinot.spi.config.table.TableConfig;
-import org.apache.pinot.spi.config.table.TableType;
-import org.apache.pinot.spi.data.FieldSpec;
-import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
-import org.testng.annotations.Test;
-
-
-/**
- * NOTE: These test cases are inactive since {@link JsonStatementOptimizer} is
currently disabled.
- *
- * Test cases to verify that {@link JsonStatementOptimizer} is properly
rewriting queries that use JSON path expressions
- * into equivalent queries that use JSON_MATCH and JSON_EXTRACT_SCALAR
functions.
- */
-@Test(enabled = false)
-public class JsonStatementOptimizerTest {
- private static final Schema SCHEMA =
- new
Schema.SchemaBuilder().setSchemaName("testTable").addSingleValueDimension("intColumn",
FieldSpec.DataType.INT)
- .addSingleValueDimension("longColumn", FieldSpec.DataType.LONG)
- .addSingleValueDimension("stringColumn", FieldSpec.DataType.STRING)
- .addSingleValueDimension("jsonColumn",
FieldSpec.DataType.JSON).build();
- private static final TableConfig TABLE_CONFIG_WITH_INDEX =
- new TableConfigBuilder(TableType.OFFLINE).setTableName("testTable")
- .setJsonIndexColumns(Arrays.asList("jsonColumn")).build();
- private static final TableConfig TABLE_CONFIG_WITHOUT_INDEX = null;
-
- /** Test that a json path expression in SELECT list is properly converted to
a JSON_EXTRACT_SCALAR function within
- * an AS function. */
- @Test(enabled = false)
- public void testJsonSelect() {
- // SELECT using json column.
- TestHelper.assertEqualsQuery("SELECT jsonColumn FROM testTable", "SELECT
jsonColumn FROM testTable",
- TABLE_CONFIG_WITH_INDEX, SCHEMA);
-
- // SELECT using a simple json path expression.
- TestHelper.assertEqualsQuery("SELECT jsonColumn.x FROM testTable",
- "SELECT JSON_EXTRACT_SCALAR(jsonColumn, '$.x', 'STRING', 'null') AS
\"jsonColumn.x\" FROM testTable",
- TABLE_CONFIG_WITH_INDEX, SCHEMA);
-
- // SELECT using json path expressions with array addressing.
- TestHelper.assertEqualsQuery("SELECT jsonColumn.data[0][1].a.b[0] FROM
testTable",
- "SELECT JSON_EXTRACT_SCALAR(jsonColumn, '$.data[0][1].a.b[0]',
'STRING', 'null') AS \"jsonColumn.data[0][1].a"
- + ".b[0]\" FROM testTable", TABLE_CONFIG_WITH_INDEX, SCHEMA);
-
- // SELECT using json path expressions within double quotes.
- TestHelper.assertEqualsQuery("SELECT
\"jsonColumn.a.b.c[0][1][2][3].d.e.f[0].g\" FROM testTable",
- "SELECT JSON_EXTRACT_SCALAR(jsonColumn,
'$.a.b.c[0][1][2][3].d.e.f[0].g', 'STRING', 'null') AS \"jsonColumn.a"
- + ".b.c[0][1][2][3].d.e.f[0].g\" FROM testTable",
TABLE_CONFIG_WITH_INDEX, SCHEMA);
- }
-
- /** Test that a predicate comparing a json path expression with literal is
properly converted into a JSON_MATCH
- * function. */
- @Test(enabled = false)
- public void testJsonFilter() {
- // Comparing json path expression with a string value.
- TestHelper.assertEqualsQuery("SELECT * FROM testTable WHERE
jsonColumn.name.first = 'daffy'",
- "SELECT * FROM testTable WHERE JSON_MATCH(jsonColumn,
'\"$.name.first\" = ''daffy''')", TABLE_CONFIG_WITH_INDEX,
- SCHEMA);
-
- TestHelper.assertEqualsQuery("SELECT * FROM testTable WHERE
jsonColumn.name.first = 'daffy'",
- "SELECT * FROM testTable WHERE JSON_EXTRACT_SCALAR(jsonColumn,
'$.name.first', 'STRING', 'null') = 'daffy'",
- TABLE_CONFIG_WITHOUT_INDEX, SCHEMA);
-
- // Comparing json path expression with a numerical value.
- TestHelper.assertEqualsQuery("SELECT * FROM testTable WHERE jsonColumn.id
= 101",
- "SELECT * FROM testTable WHERE JSON_MATCH(jsonColumn, '\"$.id\" =
101')", TABLE_CONFIG_WITH_INDEX, SCHEMA);
-
- TestHelper.assertEqualsQuery("SELECT * FROM testTable WHERE jsonColumn.id
= 101",
- "SELECT * FROM testTable WHERE JSON_EXTRACT_SCALAR(jsonColumn, '$.id',
'LONG', -9223372036854775808) = 101",
- TABLE_CONFIG_WITHOUT_INDEX, SCHEMA);
-
- // Comparing json path expression with a numerical value and checking for
null value.
- TestHelper.assertEqualsQuery("SELECT * FROM testTable WHERE jsonColumn.id
IS NOT NULL AND jsonColumn.id = 101",
- "SELECT * FROM testTable WHERE JSON_MATCH(jsonColumn, '\"$.id\" IS NOT
NULL') AND JSON_MATCH(jsonColumn, '\"$"
- + ".id\" = 101')", TABLE_CONFIG_WITH_INDEX, SCHEMA);
-
- TestHelper.assertEqualsQuery("SELECT * FROM testTable WHERE jsonColumn.id
IS NOT NULL AND jsonColumn.id = 101",
- "SELECT * FROM testTable WHERE JSON_EXTRACT_SCALAR(jsonColumn, '$.id',
'JSON', 'null') IS NOT NULL AND "
- + "JSON_EXTRACT_SCALAR(jsonColumn, '$.id', 'LONG',
-9223372036854775808) = 101", TABLE_CONFIG_WITHOUT_INDEX,
- SCHEMA);
- }
-
- /** Test that a json path expression in GROUP BY clause is properly
converted into a JSON_EXTRACT_SCALAR function. */
- @Test(enabled = false)
- public void testJsonGroupBy() {
- TestHelper.assertEqualsQuery("SELECT jsonColumn.id, count(*) FROM
testTable GROUP BY jsonColumn.id",
- "SELECT JSON_EXTRACT_SCALAR(jsonColumn, '$.id', 'STRING', 'null') AS
\"jsonColumn.id\", count(*) FROM "
- + "testTable GROUP BY JSON_EXTRACT_SCALAR(jsonColumn, '$.id',
'STRING', 'null')", TABLE_CONFIG_WITH_INDEX,
- SCHEMA);
-
- TestHelper.assertEqualsQuery("SELECT jsonColumn.id, count(*) FROM
testTable GROUP BY jsonColumn.id",
- "SELECT JSON_EXTRACT_SCALAR(jsonColumn, '$.id', 'STRING', 'null') AS
\"jsonColumn.id\", count(*) FROM "
- + "testTable GROUP BY JSON_EXTRACT_SCALAR(jsonColumn, '$.id',
'STRING', 'null')",
- TABLE_CONFIG_WITHOUT_INDEX, SCHEMA);
- }
-
- /** Test that a json path expression in HAVING clause is properly converted
into a JSON_EXTRACT_SCALAR function. */
- @Test(enabled = false)
- public void testJsonGroupByHaving() {
- TestHelper.assertEqualsQuery(
- "SELECT jsonColumn.name.last, count(*) FROM testTable GROUP BY
jsonColumn.name.last HAVING jsonColumn.name"
- + ".last = 'mouse'",
- "SELECT JSON_EXTRACT_SCALAR(jsonColumn, '$.name.last', 'STRING',
'null') AS \"jsonColumn.name.last\", count"
- + "(*) FROM testTable GROUP BY JSON_EXTRACT_SCALAR(jsonColumn,
'$.name.last', 'STRING', 'null') HAVING "
- + "JSON_EXTRACT_SCALAR(jsonColumn, '$.name.last', 'STRING',
'null') = 'mouse'", TABLE_CONFIG_WITH_INDEX,
- SCHEMA);
-
- TestHelper.assertEqualsQuery(
- "SELECT jsonColumn.name.last, count(*) FROM testTable GROUP BY
jsonColumn.name.last HAVING jsonColumn.name"
- + ".last = 'mouse'",
- "SELECT JSON_EXTRACT_SCALAR(jsonColumn, '$.name.last', 'STRING',
'null') AS \"jsonColumn.name.last\", count"
- + "(*) FROM testTable GROUP BY JSON_EXTRACT_SCALAR(jsonColumn,
'$.name.last', 'STRING', 'null') HAVING "
- + "JSON_EXTRACT_SCALAR(jsonColumn, '$.name.last', 'STRING',
'null') = 'mouse'", TABLE_CONFIG_WITHOUT_INDEX,
- SCHEMA);
- }
-
- /** Test a complex SQL statement with json path expression in SELECT, WHERE,
and GROUP BY clauses. */
- @Test(enabled = false)
- public void testJsonSelectFilterGroupBy() {
- TestHelper.assertEqualsQuery(
- "SELECT jsonColumn.name.last, count(*) FROM testTable WHERE
jsonColumn.id = 101 GROUP BY jsonColumn.name.last",
- "SELECT JSON_EXTRACT_SCALAR(jsonColumn, '$.name.last', 'STRING',
'null') AS \"jsonColumn.name.last\", count"
- + "(*) FROM testTable WHERE JSON_MATCH(jsonColumn, '\"$.id\" =
101') GROUP BY JSON_EXTRACT_SCALAR"
- + "(jsonColumn, '$.name.last', 'STRING', 'null')",
TABLE_CONFIG_WITH_INDEX, SCHEMA);
-
- TestHelper.assertEqualsQuery(
- "SELECT jsonColumn.name.last, count(*) FROM testTable WHERE
jsonColumn.id = 101 GROUP BY jsonColumn.name.last",
- "SELECT JSON_EXTRACT_SCALAR(jsonColumn, '$.name.last', 'STRING',
'null') AS \"jsonColumn.name.last\", count"
- + "(*) FROM testTable WHERE JSON_EXTRACT_SCALAR(jsonColumn,
'$.id', 'LONG', -9223372036854775808) = 101 "
- + "GROUP BY JSON_EXTRACT_SCALAR(jsonColumn, '$.name.last',
'STRING', 'null')", TABLE_CONFIG_WITHOUT_INDEX,
- SCHEMA);
- }
-
- /** Test an aggregation function over json path expression in SELECT clause.
*/
- @Test(enabled = false)
- public void testTransformFunctionOverJsonPathSelectExpression() {
- // Apply string transform function on json path expression.
- TestHelper.assertEqualsQuery("SELECT UPPER(jsonColumn.name.first) FROM
testTable",
- "SELECT UPPER(JSON_EXTRACT_SCALAR(jsonColumn, '$.name.first',
'STRING', 'null')) AS \"upper(jsonColumn.name"
- + ".first)\" FROM testTable", TABLE_CONFIG_WITH_INDEX, SCHEMA);
-
- // Apply date transform function on json path expression and check for IS
NULL
- TestHelper.assertEqualsQuery("SELECT FROMEPOCHDAYS(jsonColumn.days) FROM
testTable WHERE jsonColumn.days IS NULL",
- "SELECT FROMEPOCHDAYS(JSON_EXTRACT_SCALAR(jsonColumn, '$.days',
'LONG', -9223372036854775808)) AS "
- + "\"fromepochdays(jsonColumn.days)\" FROM testTable WHERE
JSON_MATCH(jsonColumn, '\"$.days\" IS NULL')",
- TABLE_CONFIG_WITH_INDEX, SCHEMA);
-
- // Apply date transform function on json path expression and check for IS
NOT NULL
- TestHelper
- .assertEqualsQuery("SELECT FROMEPOCHDAYS(jsonColumn.days) FROM
testTable WHERE jsonColumn.days IS NOT NULL",
- "SELECT FROMEPOCHDAYS(JSON_EXTRACT_SCALAR(jsonColumn, '$.days',
'LONG', -9223372036854775808)) AS "
- + "\"fromepochdays(jsonColumn.days)\" FROM testTable WHERE
JSON_MATCH(jsonColumn, '\"$.days\" IS NOT "
- + "NULL')", TABLE_CONFIG_WITH_INDEX, SCHEMA);
-
- TestHelper
- .assertEqualsQuery("SELECT FROMEPOCHDAYS(jsonColumn.days) FROM
testTable WHERE jsonColumn.days IS NOT NULL",
- "SELECT FROMEPOCHDAYS(JSON_EXTRACT_SCALAR(jsonColumn, '$.days',
'LONG', -9223372036854775808)) AS "
- + "\"fromepochdays(jsonColumn.days)\" FROM testTable WHERE
JSON_EXTRACT_SCALAR(jsonColumn, '$.days', "
- + "'JSON', 'null') IS NOT NULL", TABLE_CONFIG_WITHOUT_INDEX,
SCHEMA);
- }
-
- /** Test a numerical function over json path expression in SELECT clause. */
- @Test(enabled = false)
- public void testNumericalFunctionOverJsonPathSelectExpression() {
-
- // Test without user-specified alias.
- TestHelper.assertEqualsQuery("SELECT MAX(jsonColumn.id) FROM testTable",
- "SELECT MAX(JSON_EXTRACT_SCALAR(jsonColumn, '$.id', 'DOUBLE', '" +
Double.NEGATIVE_INFINITY
- + "')) AS \"max(jsonColumn.id)\" FROM testTable",
TABLE_CONFIG_WITH_INDEX, SCHEMA);
-
- // Test with user-specified alias.
- TestHelper.assertEqualsQuery("SELECT MAX(jsonColumn.id) AS x FROM
testTable",
- "SELECT MAX(JSON_EXTRACT_SCALAR(jsonColumn, '$.id', 'DOUBLE', '" +
Double.NEGATIVE_INFINITY
- + "')) AS x FROM testTable", TABLE_CONFIG_WITH_INDEX, SCHEMA);
-
- // Test with nested function calls (minus function being used within max
function).
- TestHelper.assertEqualsQuery("SELECT MAX(jsonColumn.id - 5) FROM
testTable",
- "SELECT MAX(JSON_EXTRACT_SCALAR(jsonColumn, '$.id', 'DOUBLE', '" +
Double.NEGATIVE_INFINITY
- + "') - 5) AS \"max(minus(jsonColumn.id,'5'))\" FROM testTable",
TABLE_CONFIG_WITH_INDEX, SCHEMA);
- }
-
- @Test(enabled = false)
- public void testTopLevelArrayPathExpressions() {
- // SELECT using json path expression with top-level array addressing.
- TestHelper.assertEqualsQuery("SELECT jsonColumn[0] FROM testTable",
- "SELECT JSON_EXTRACT_SCALAR(jsonColumn, '$[0]', 'STRING', 'null') AS
\"jsonColumn[0]\" FROM testTable",
- TABLE_CONFIG_WITH_INDEX, SCHEMA);
-
- TestHelper.assertEqualsQuery("SELECT jsonColumn[0].a FROM testTable",
- "SELECT JSON_EXTRACT_SCALAR(jsonColumn, '$[0].a', 'STRING', 'null') AS
\"jsonColumn[0].a\" FROM testTable",
- TABLE_CONFIG_WITH_INDEX, SCHEMA);
-
- TestHelper.assertEqualsQuery("SELECT jsonColumn.a[0] FROM testTable",
- "SELECT JSON_EXTRACT_SCALAR(jsonColumn, '$.a[0]', 'STRING', 'null') AS
\"jsonColumn.a[0]\" FROM testTable",
- TABLE_CONFIG_WITH_INDEX, SCHEMA);
-
- TestHelper.assertEqualsQuery("SELECT jsonColumn[1].i2 FROM testTable WHERE
jsonColumn[1].i2 IS NOT NULL",
- "SELECT JSON_EXTRACT_SCALAR(jsonColumn, '$[1].i2', 'STRING', 'null')
AS \"jsonColumn[1].i2\" FROM testTable "
- + "WHERE JSON_MATCH(jsonColumn, '\"$[1].i2\" IS NOT NULL')",
- TABLE_CONFIG_WITH_INDEX, SCHEMA);
-
- // Use top-level array addressing in json path expression in
JSON_EXTRACT_SCALAR filter.
- TestHelper.assertEqualsQuery("SELECT * FROM testTable WHERE jsonColumn[2]
IS NOT NULL and jsonColumn[2] = 'test'",
- "SELECT * FROM testTable WHERE JSON_EXTRACT_SCALAR(jsonColumn, '$[2]',
'JSON', 'null') IS NOT NULL AND "
- + "JSON_EXTRACT_SCALAR(jsonColumn, '$[2]', 'STRING', 'null') =
'test'",
- TABLE_CONFIG_WITHOUT_INDEX, SCHEMA);
-
- // Use top-level array addressing in json path expression in JSON_MATCH
filter
- TestHelper.assertEqualsQuery("SELECT * FROM testTable WHERE jsonColumn[2]
IS NOT NULL and jsonColumn[2] = 'test'",
- "SELECT * FROM testTable WHERE JSON_MATCH(jsonColumn, '\"$[2]\" IS NOT
NULL') AND JSON_MATCH(jsonColumn, "
- + "'\"$[2]\" = ''test''')",
- TABLE_CONFIG_WITH_INDEX, SCHEMA);
-
- // Use top-level array addressing in json path expression in GROUP BY
clause.
- TestHelper.assertEqualsQuery("SELECT jsonColumn[0], count(*) FROM
testTable GROUP BY jsonColumn[0]",
- "SELECT JSON_EXTRACT_SCALAR(jsonColumn, '$[0]', 'STRING', 'null') AS
\"jsonColumn[0]\", count(*) FROM "
- + "testTable GROUP BY JSON_EXTRACT_SCALAR(jsonColumn, '$[0]',
'STRING', 'null')",
- TABLE_CONFIG_WITH_INDEX, SCHEMA);
- }
-}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/statement/StringPredicateFilterOptimizerTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/statement/StringPredicateFilterOptimizerTest.java
index bade3a81bc2..9f9ad2ac881 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/statement/StringPredicateFilterOptimizerTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/statement/StringPredicateFilterOptimizerTest.java
@@ -37,34 +37,32 @@ public class StringPredicateFilterOptimizerTest {
public void testReplaceMinusWithCompare() {
// 'WHERE strColumn1=strColumn2' gets replaced with 'strcmp(strColumn1,
strColumn2) = 0'
TestHelper.assertEqualsQuery("SELECT * FROM testTable WHERE
strColumn1=strColumn2",
- "SELECT * FROM testTable WHERE strcmp(strColumn1,strColumn2) = 0",
TABLE_CONFIG_WITHOUT_INDEX, SCHEMA);
+ "SELECT * FROM testTable WHERE strcmp(strColumn1,strColumn2) = 0",
SCHEMA);
// 'WHERE trim(strColumn1)=strColumn2' gets replaced with
'strcmp(trim(strColumn1), strColumn2) = 0'
TestHelper.assertEqualsQuery("SELECT * FROM testTable WHERE
trim(strColumn1)=strColumn2",
- "SELECT * FROM testTable WHERE strcmp(trim(strColumn1),strColumn2) =
0", TABLE_CONFIG_WITHOUT_INDEX, SCHEMA);
+ "SELECT * FROM testTable WHERE strcmp(trim(strColumn1),strColumn2) =
0", SCHEMA);
// 'WHERE strColumn1=trim(strColumn2)' gets replaced with
'strcmp(strColumn1, trim(strColumn2)) = 0'
TestHelper.assertEqualsQuery("SELECT * FROM testTable WHERE
strColumn1=trim(strColumn2)",
- "SELECT * FROM testTable WHERE strcmp(strColumn1,trim(strColumn2)) =
0", TABLE_CONFIG_WITHOUT_INDEX, SCHEMA);
+ "SELECT * FROM testTable WHERE strcmp(strColumn1,trim(strColumn2)) =
0", SCHEMA);
// 'WHERE strColumn1>strColumn2' gets replaced with 'strcmp(strColumn1,
strColumn2) > 0'
TestHelper.assertEqualsQuery("SELECT * FROM testTable WHERE
strColumn1>strColumn2",
- "SELECT * FROM testTable WHERE strcmp(strColumn1,strColumn2) > 0",
TABLE_CONFIG_WITHOUT_INDEX, SCHEMA);
+ "SELECT * FROM testTable WHERE strcmp(strColumn1,strColumn2) > 0",
SCHEMA);
// 'HAVING strColumn1=strColumn2' gets replaced with 'strcmp(strColumn1,
strColumn2) = 0'
TestHelper.assertEqualsQuery("SELECT strColumn1, strColumn2 FROM testTable
HAVING strColumn1=strColumn2",
- "SELECT strColumn1, strColumn2 FROM testTable HAVING
strcmp(strColumn1,strColumn2)=0",
- TABLE_CONFIG_WITHOUT_INDEX, SCHEMA);
+ "SELECT strColumn1, strColumn2 FROM testTable HAVING
strcmp(strColumn1,strColumn2)=0", SCHEMA);
// 'HAVING strColumn1=strColumn2' gets replaced with 'strcmp(strColumn1,
strColumn2) < 0'
TestHelper.assertEqualsQuery("SELECT strColumn1, strColumn2 FROM testTable
HAVING strColumn1<strColumn2",
- "SELECT strColumn1, strColumn2 FROM testTable HAVING
strcmp(strColumn1,strColumn2)<0",
- TABLE_CONFIG_WITHOUT_INDEX, SCHEMA);
+ "SELECT strColumn1, strColumn2 FROM testTable HAVING
strcmp(strColumn1,strColumn2)<0", SCHEMA);
// 'WHERE strColumn1=strColumn2 AND strColumn1=strColumn3' gets replaced
with 'strcmp(strColumn1, strColumn2) = 0
// AND strcmp(strColumn1, strColumn3) = 0'
TestHelper.assertEqualsQuery("SELECT * FROM testTable WHERE
strColumn1=strColumn2 OR strColumn1=strColumn3",
"SELECT * FROM testTable WHERE strcmp(strColumn1,strColumn2) = 0 OR
strcmp(strColumn1,strColumn3) = 0",
- TABLE_CONFIG_WITHOUT_INDEX, SCHEMA);
+ SCHEMA);
}
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/statement/TestHelper.java
b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/statement/TestHelper.java
index cd895837c8f..dce37106336 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/statement/TestHelper.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/statement/TestHelper.java
@@ -20,7 +20,6 @@ package org.apache.pinot.core.query.optimizer.statement;
import org.apache.pinot.common.request.PinotQuery;
import org.apache.pinot.core.query.optimizer.QueryOptimizer;
-import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.sql.parsers.CalciteSqlParser;
@@ -38,11 +37,11 @@ class TestHelper {
* Given two queries, this function will validate that the query obtained
after rewriting the first query is the
* same as the second query.
*/
- static void assertEqualsQuery(String queryOriginal, String
queryAfterRewrite, TableConfig config, Schema schema) {
+ static void assertEqualsQuery(String queryOriginal, String
queryAfterRewrite, Schema schema) {
PinotQuery userQuery = CalciteSqlParser.compileToPinotQuery(queryOriginal);
- OPTIMIZER.optimize(userQuery, config, schema);
+ OPTIMIZER.optimize(userQuery, schema);
PinotQuery rewrittenQuery =
CalciteSqlParser.compileToPinotQuery(queryAfterRewrite);
- OPTIMIZER.optimize(rewrittenQuery, config, schema);
+ OPTIMIZER.optimize(rewrittenQuery, schema);
// Currently there is no way to specify Double.NEGATIVE_INFINITY in SQL,
so in the test cases we specify string
// '-Infinity' as
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/BaseJsonQueryTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/BaseJsonQueryTest.java
index 6ef6e89d665..901e66c8b68 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/BaseJsonQueryTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/BaseJsonQueryTest.java
@@ -179,7 +179,7 @@ public abstract class BaseJsonQueryTest extends
BaseQueriesTest {
}
protected void checkResult(String query, Object[][] expectedResults) {
- BrokerResponseNative brokerResponse =
getBrokerResponseForOptimizedQuery(query, tableConfig(), schema());
+ BrokerResponseNative brokerResponse =
getBrokerResponseForOptimizedQuery(query, schema());
QueriesTestUtils.testInterSegmentsResult(brokerResponse,
Arrays.asList(expectedResults));
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/BaseQueriesTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/BaseQueriesTest.java
index e87990c1dc8..3d652d1910d 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/BaseQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/BaseQueriesTest.java
@@ -47,7 +47,6 @@ import org.apache.pinot.core.transport.ServerRoutingInstance;
import org.apache.pinot.core.util.GapfillUtils;
import org.apache.pinot.segment.spi.IndexSegment;
import org.apache.pinot.segment.spi.SegmentContext;
-import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.env.PinotConfiguration;
@@ -270,9 +269,9 @@ public abstract class BaseQueriesTest {
* @see StatisticalQueriesTest for an example use case.
*/
protected BrokerResponseNative
getBrokerResponseForOptimizedQuery(@Language("sql") String query,
- @Nullable TableConfig config, @Nullable Schema schema) {
+ @Nullable Schema schema) {
PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
- OPTIMIZER.optimize(pinotQuery, config, schema);
+ OPTIMIZER.optimize(pinotQuery, schema);
return getBrokerResponse(pinotQuery, PLAN_MAKER);
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/JsonMalformedIndexTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/JsonMalformedIndexTest.java
index c6513710a07..bc3b498a481 100644
---
a/pinot-core/src/test/java/org/apache/pinot/queries/JsonMalformedIndexTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/queries/JsonMalformedIndexTest.java
@@ -69,7 +69,7 @@ public class JsonMalformedIndexTest extends BaseQueriesTest {
}
protected void checkResult(String query, Object[][] expectedResults) {
- BrokerResponseNative brokerResponse =
getBrokerResponseForOptimizedQuery(query, TABLE_CONFIG, SCHEMA);
+ BrokerResponseNative brokerResponse =
getBrokerResponseForOptimizedQuery(query, SCHEMA);
QueriesTestUtils.testInterSegmentsResult(brokerResponse,
Arrays.asList(expectedResults));
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/TextSearchQueriesTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/TextSearchQueriesTest.java
index 5861b3fec10..2bfd1c107d7 100644
---
a/pinot-core/src/test/java/org/apache/pinot/queries/TextSearchQueriesTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/queries/TextSearchQueriesTest.java
@@ -2310,7 +2310,7 @@ public class TextSearchQueriesTest extends
BaseQueriesTest {
+ ", '*CUDA*', 'parser=CLASSIC,allowLeadingWildcard=true') AND " +
"TEXT_MATCH(" + SKILLS_TEXT_COL_NAME
+ ", '*Python*', 'parser=CLASSIC,allowLeadingWildcard=true') LIMIT
50000";
- BrokerResponseNative brokerResponse =
getBrokerResponseForOptimizedQuery(query, getTableConfig(), SCHEMA);
+ BrokerResponseNative brokerResponse =
getBrokerResponseForOptimizedQuery(query, SCHEMA);
assertTrue(brokerResponse.getNumDocsScanned() > 0, "Query should scan some
documents");
}
@@ -2322,7 +2322,7 @@ public class TextSearchQueriesTest extends
BaseQueriesTest {
"SELECT INT_COL, SKILLS_TEXT_COL FROM " + TABLE_NAME + " WHERE " +
"TEXT_MATCH(" + SKILLS_TEXT_COL_NAME
+ ", '*CUDA*', 'parser=CLASSIC,allowLeadingWildcard=true') AND " +
"TEXT_MATCH(" + SKILLS_TEXT_COL_NAME
+ ", 'Python*', 'parser=STANDARD') LIMIT 50000";
- BrokerResponseNative responseTrailing =
getBrokerResponseForOptimizedQuery(queryTrailing, getTableConfig(), SCHEMA);
+ BrokerResponseNative responseTrailing =
getBrokerResponseForOptimizedQuery(queryTrailing, SCHEMA);
assertTrue(responseTrailing.getNumDocsScanned() > 0, "Trailing wildcard
should scan some documents");
// This should fail: leading wildcard is NOT allowed with parser=STANDARD
@@ -2332,7 +2332,7 @@ public class TextSearchQueriesTest extends
BaseQueriesTest {
+ ", '*CUDA*', 'parser=CLASSIC,allowLeadingWildcard=true') AND " +
"TEXT_MATCH(" + SKILLS_TEXT_COL_NAME
+ ", '*Python*', 'parser=STANDARD') LIMIT 50000";
- BrokerResponseNative responseLeading =
getBrokerResponseForOptimizedQuery(queryLeading, getTableConfig(), SCHEMA);
+ BrokerResponseNative responseLeading =
getBrokerResponseForOptimizedQuery(queryLeading, SCHEMA);
List<QueryProcessingException> exceptions =
responseLeading.getExceptions();
assertFalse(exceptions.isEmpty(), "Expected error for leading wildcard
with parser=STANDARD");
String errorMsg = exceptions.toString();
diff --git
a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkAggregateGroupByOrderByQueriesSSE.java
b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkAggregateGroupByOrderByQueriesSSE.java
index 81d2868b260..16fd7c9f90f 100644
---
a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkAggregateGroupByOrderByQueriesSSE.java
+++
b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkAggregateGroupByOrderByQueriesSSE.java
@@ -487,10 +487,10 @@ public class BenchmarkAggregateGroupByOrderByQueriesSSE {
* This can be particularly useful to test statistical aggregation functions.
* @see StatisticalQueriesTest for an example use case.
*/
- protected BrokerResponseNative getBrokerResponseForOptimizedQuery(
- @Language("sql") String query, @Nullable TableConfig config, @Nullable
Schema schema) {
+ protected BrokerResponseNative
getBrokerResponseForOptimizedQuery(@Language("sql") String query,
+ @Nullable Schema schema) {
PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
- OPTIMIZER.optimize(pinotQuery, config, schema);
+ OPTIMIZER.optimize(pinotQuery, schema);
return getBrokerResponse(pinotQuery, PLAN_MAKER);
}
diff --git
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/server/ServerPlanRequestUtils.java
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/server/ServerPlanRequestUtils.java
index 47f719be022..639b1a134ab 100644
---
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/server/ServerPlanRequestUtils.java
+++
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/server/ServerPlanRequestUtils.java
@@ -234,7 +234,7 @@ public class ServerPlanRequestUtils {
for (QueryRewriter queryRewriter : QUERY_REWRITERS) {
pinotQuery = queryRewriter.rewrite(pinotQuery);
}
- QUERY_OPTIMIZER.optimize(pinotQuery, tableConfig, schema);
+ QUERY_OPTIMIZER.optimize(pinotQuery, schema);
// 2. Update query options according to requestMetadataMap
updateQueryOptions(pinotQuery, executionContext);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
