This is an automated email from the ASF dual-hosted git repository. rongr pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push: new 8fbb2d5451 support case-insensitive query options in SET syntax (#9912) 8fbb2d5451 is described below commit 8fbb2d545185c4def350fcb962719aae48dc4e36 Author: Almog Gavra <almog.ga...@gmail.com> AuthorDate: Thu Dec 15 09:59:04 2022 -0800 support case-insensitive query options in SET syntax (#9912) * support case-insensitive query options in SET syntax * support for all ways of setting options * move QueryOptionsUtils * lint --- .../requesthandler/BaseBrokerRequestHandler.java | 2 +- .../MultiStageBrokerRequestHandler.java | 2 +- .../ReplicaGroupInstanceSelector.java | 2 +- .../segmentselector/RealtimeSegmentSelector.java | 2 +- .../common/utils/config}/QueryOptionsUtils.java | 59 +++++++++++++++++++++- .../apache/pinot/sql/parsers/CalciteSqlParser.java | 10 +--- .../pinot/sql/parsers/SqlNodeAndOptions.java | 3 +- .../common/utils/config/QueryOptionsUtilsTest.java | 46 +++++++++++++++++ .../pinot/core/operator/docidsets/AndDocIdSet.java | 2 +- .../apache/pinot/core/plan/SelectionPlanNode.java | 2 +- .../core/plan/maker/InstancePlanMakerImplV2.java | 2 +- .../query/executor/ServerQueryExecutorV1Impl.java | 2 +- .../query/reduce/ExplainPlanDataTableReducer.java | 2 +- .../core/query/request/context/QueryContext.java | 2 +- 14 files changed, 118 insertions(+), 20 deletions(-) diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java index 7a704bcffa..c9277395a4 100644 --- a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java +++ b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java @@ -68,13 +68,13 @@ import org.apache.pinot.common.response.ProcessingException; import org.apache.pinot.common.response.broker.BrokerResponseNative; import org.apache.pinot.common.response.broker.ResultTable; import org.apache.pinot.common.utils.DataSchema; +import org.apache.pinot.common.utils.config.QueryOptionsUtils; import org.apache.pinot.common.utils.request.RequestUtils; import org.apache.pinot.core.query.optimizer.QueryOptimizer; import org.apache.pinot.core.routing.RoutingTable; import org.apache.pinot.core.routing.TimeBoundaryInfo; import org.apache.pinot.core.transport.ServerInstance; import org.apache.pinot.core.util.GapfillUtils; -import org.apache.pinot.core.util.QueryOptionsUtils; import org.apache.pinot.spi.config.table.FieldConfig; import org.apache.pinot.spi.config.table.QueryConfig; import org.apache.pinot.spi.config.table.TableConfig; diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/MultiStageBrokerRequestHandler.java b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/MultiStageBrokerRequestHandler.java index 34acd15b48..e480e8f5d6 100644 --- a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/MultiStageBrokerRequestHandler.java +++ b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/MultiStageBrokerRequestHandler.java @@ -39,9 +39,9 @@ import org.apache.pinot.common.response.BrokerResponse; import org.apache.pinot.common.response.broker.BrokerResponseNative; import org.apache.pinot.common.response.broker.ResultTable; import org.apache.pinot.common.utils.DataSchema; +import org.apache.pinot.common.utils.config.QueryOptionsUtils; import org.apache.pinot.common.utils.request.RequestUtils; import org.apache.pinot.core.transport.ServerInstance; -import org.apache.pinot.core.util.QueryOptionsUtils; import org.apache.pinot.query.QueryEnvironment; import org.apache.pinot.query.catalog.PinotCatalog; import org.apache.pinot.query.mailbox.MailboxService; diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/routing/instanceselector/ReplicaGroupInstanceSelector.java b/pinot-broker/src/main/java/org/apache/pinot/broker/routing/instanceselector/ReplicaGroupInstanceSelector.java index 4389fcccbe..957e140c8e 100644 --- a/pinot-broker/src/main/java/org/apache/pinot/broker/routing/instanceselector/ReplicaGroupInstanceSelector.java +++ b/pinot-broker/src/main/java/org/apache/pinot/broker/routing/instanceselector/ReplicaGroupInstanceSelector.java @@ -29,7 +29,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.pinot.broker.routing.adaptiveserverselector.AdaptiveServerSelector; import org.apache.pinot.common.metrics.BrokerMetrics; import org.apache.pinot.common.utils.HashUtil; -import org.apache.pinot.core.util.QueryOptionsUtils; +import org.apache.pinot.common.utils.config.QueryOptionsUtils; /** diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/routing/segmentselector/RealtimeSegmentSelector.java b/pinot-broker/src/main/java/org/apache/pinot/broker/routing/segmentselector/RealtimeSegmentSelector.java index 804267eef6..72d44ffd81 100644 --- a/pinot-broker/src/main/java/org/apache/pinot/broker/routing/segmentselector/RealtimeSegmentSelector.java +++ b/pinot-broker/src/main/java/org/apache/pinot/broker/routing/segmentselector/RealtimeSegmentSelector.java @@ -33,7 +33,7 @@ import org.apache.pinot.common.request.BrokerRequest; import org.apache.pinot.common.utils.HLCSegmentName; import org.apache.pinot.common.utils.LLCSegmentName; import org.apache.pinot.common.utils.SegmentName; -import org.apache.pinot.core.util.QueryOptionsUtils; +import org.apache.pinot.common.utils.config.QueryOptionsUtils; import org.apache.pinot.spi.utils.CommonConstants.Helix.StateModel.SegmentStateModel; diff --git a/pinot-core/src/main/java/org/apache/pinot/core/util/QueryOptionsUtils.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/QueryOptionsUtils.java similarity index 65% rename from pinot-core/src/main/java/org/apache/pinot/core/util/QueryOptionsUtils.java rename to pinot-common/src/main/java/org/apache/pinot/common/utils/config/QueryOptionsUtils.java index 4a0d6e5d8c..dd6f5d5101 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/util/QueryOptionsUtils.java +++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/QueryOptionsUtils.java @@ -16,9 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pinot.core.util; +package org.apache.pinot.common.utils.config; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.HashMap; import java.util.Map; import javax.annotation.Nullable; import org.apache.pinot.spi.utils.CommonConstants.Broker.Request.QueryOptionKey; @@ -32,6 +36,59 @@ public class QueryOptionsUtils { private QueryOptionsUtils() { } + + private static final Map<String, String> CONFIG_RESOLVER; + private static final RuntimeException CLASS_LOAD_ERROR; + + static { + // this is a bit hacky, but lots of the code depends directly on usage of + // Map<String, String> (JSON serialization/GRPC code) so we cannot just + // refactor all code to use a case-insensitive abstraction like PinotConfiguration + // without a lot of work - additionally, the config constants are string constants + // instead of enums so there's no good way to iterate over them, but they are + // public API so we cannot just change them to be an enum + Map<String, String> configResolver = new HashMap<>(); + Throwable classLoadError = null; + + try { + for (Field declaredField : QueryOptionKey.class.getDeclaredFields()) { + if (declaredField.getType().equals(String.class)) { + int mods = declaredField.getModifiers(); + if (Modifier.isStatic(mods) && Modifier.isFinal(mods)) { + String config = (String) declaredField.get(null); + configResolver.put(config.toLowerCase(), config); + } + } + } + } catch (IllegalAccessException e) { + // prefer rethrowing this during runtime instead of a ClassNotFoundException + configResolver = null; + classLoadError = e; + } + + CONFIG_RESOLVER = configResolver == null ? null : ImmutableMap.copyOf(configResolver); + CLASS_LOAD_ERROR = classLoadError == null ? null + : new RuntimeException("Failure to build case insensitive mapping.", classLoadError); + } + + public static Map<String, String> resolveCaseInsensitiveOptions(Map<String, String> queryOptions) { + if (CLASS_LOAD_ERROR != null) { + throw CLASS_LOAD_ERROR; + } + + Map<String, String> resolved = new HashMap<>(); + for (Map.Entry<String, String> configEntry : queryOptions.entrySet()) { + String config = CONFIG_RESOLVER.get(configEntry.getKey().toLowerCase()); + if (config != null) { + resolved.put(config, configEntry.getValue()); + } else { + resolved.put(configEntry.getKey(), configEntry.getValue()); + } + } + + return resolved; + } + @Nullable public static Long getTimeoutMs(Map<String, String> queryOptions) { String timeoutMsString = queryOptions.get(QueryOptionKey.TIMEOUT_MS); diff --git a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java index d77461e978..c5755a4c87 100644 --- a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java +++ b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java @@ -58,6 +58,7 @@ import org.apache.pinot.common.request.ExpressionType; import org.apache.pinot.common.request.Function; import org.apache.pinot.common.request.Identifier; import org.apache.pinot.common.request.PinotQuery; +import org.apache.pinot.common.utils.config.QueryOptionsUtils; import org.apache.pinot.common.utils.request.RequestUtils; import org.apache.pinot.segment.spi.AggregationFunctionType; import org.apache.pinot.spi.utils.Pairs; @@ -171,7 +172,7 @@ public class CalciteSqlParser { if (sqlType == null) { throw new SqlCompilationException("SqlNode with executable statement not found!"); } - return new SqlNodeAndOptions(statementNode, sqlType, options); + return new SqlNodeAndOptions(statementNode, sqlType, QueryOptionsUtils.resolveCaseInsensitiveOptions(options)); } public static PinotQuery compileToPinotQuery(String sql) @@ -493,13 +494,6 @@ public class CalciteSqlParser { return options; } - private static void setOptions(PinotQuery pinotQuery, List<String> optionsStatements) { - if (optionsStatements.isEmpty()) { - return; - } - pinotQuery.setQueryOptions(extractOptionsMap(optionsStatements)); - } - /** * Removes comments from the query. * NOTE: Comment indicator within single quotes (literal) and double quotes (identifier) are ignored. diff --git a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/SqlNodeAndOptions.java b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/SqlNodeAndOptions.java index 666a8bda54..63fd23fc8e 100644 --- a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/SqlNodeAndOptions.java +++ b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/SqlNodeAndOptions.java @@ -20,6 +20,7 @@ package org.apache.pinot.sql.parsers; import java.util.Map; import org.apache.calcite.sql.SqlNode; +import org.apache.pinot.common.utils.config.QueryOptionsUtils; public class SqlNodeAndOptions { @@ -57,7 +58,7 @@ public class SqlNodeAndOptions { } public void setExtraOptions(Map<String, String> extractOptionsMap) { - for (Map.Entry<String, String> e : extractOptionsMap.entrySet()) { + for (Map.Entry<String, String> e : QueryOptionsUtils.resolveCaseInsensitiveOptions(extractOptionsMap).entrySet()) { _options.putIfAbsent(e.getKey(), e.getValue()); } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/QueryOptionsUtilsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/QueryOptionsUtilsTest.java new file mode 100644 index 0000000000..1564fcc15e --- /dev/null +++ b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/QueryOptionsUtilsTest.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pinot.common.utils.config; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.pinot.spi.utils.CommonConstants; +import org.testng.Assert; +import org.testng.annotations.Test; + + +public class QueryOptionsUtilsTest { + + @Test + public void shouldConvertCaseInsensitiveMapToUseCorrectValues() { + // Given: + Map<String, String> configs = ImmutableMap.of( + "ENABLENullHandling", "true", + "useMULTISTAGEEngine", "false" + ); + + // When: + Map<String, String> resolved = QueryOptionsUtils.resolveCaseInsensitiveOptions(configs); + + // Then: + Assert.assertEquals(resolved.get(CommonConstants.Broker.Request.QueryOptionKey.ENABLE_NULL_HANDLING), "true"); + Assert.assertEquals(resolved.get(CommonConstants.Broker.Request.QueryOptionKey.USE_MULTISTAGE_ENGINE), "false"); + } +} diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/docidsets/AndDocIdSet.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/docidsets/AndDocIdSet.java index b93d86c5e4..8b43b7d340 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/docidsets/AndDocIdSet.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/docidsets/AndDocIdSet.java @@ -23,13 +23,13 @@ import java.util.Comparator; import java.util.List; import java.util.Map; import org.apache.commons.collections.MapUtils; +import org.apache.pinot.common.utils.config.QueryOptionsUtils; import org.apache.pinot.core.common.BlockDocIdIterator; import org.apache.pinot.core.operator.dociditerators.AndDocIdIterator; import org.apache.pinot.core.operator.dociditerators.BitmapBasedDocIdIterator; import org.apache.pinot.core.operator.dociditerators.RangelessBitmapDocIdIterator; import org.apache.pinot.core.operator.dociditerators.ScanBasedDocIdIterator; import org.apache.pinot.core.operator.dociditerators.SortedDocIdIterator; -import org.apache.pinot.core.util.QueryOptionsUtils; import org.apache.pinot.core.util.SortedRangeIntersection; import org.apache.pinot.spi.utils.Pairs.IntPair; import org.roaringbitmap.buffer.ImmutableRoaringBitmap; diff --git a/pinot-core/src/main/java/org/apache/pinot/core/plan/SelectionPlanNode.java b/pinot-core/src/main/java/org/apache/pinot/core/plan/SelectionPlanNode.java index 9ed48c5983..cd598a7309 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/plan/SelectionPlanNode.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/plan/SelectionPlanNode.java @@ -23,6 +23,7 @@ import java.util.List; import javax.annotation.Nullable; import org.apache.pinot.common.request.context.ExpressionContext; import org.apache.pinot.common.request.context.OrderByExpressionContext; +import org.apache.pinot.common.utils.config.QueryOptionsUtils; import org.apache.pinot.core.common.Operator; import org.apache.pinot.core.operator.blocks.results.SelectionResultsBlock; import org.apache.pinot.core.operator.query.EmptySelectionOperator; @@ -33,7 +34,6 @@ import org.apache.pinot.core.operator.query.SelectionPartiallyOrderedByDescOpera import org.apache.pinot.core.operator.transform.TransformOperator; import org.apache.pinot.core.query.request.context.QueryContext; import org.apache.pinot.core.query.selection.SelectionOperatorUtils; -import org.apache.pinot.core.util.QueryOptionsUtils; import org.apache.pinot.segment.spi.IndexSegment; import org.apache.pinot.segment.spi.datasource.DataSource; import org.apache.pinot.segment.spi.index.reader.NullValueVectorReader; diff --git a/pinot-core/src/main/java/org/apache/pinot/core/plan/maker/InstancePlanMakerImplV2.java b/pinot-core/src/main/java/org/apache/pinot/core/plan/maker/InstancePlanMakerImplV2.java index b4e862518d..391a8797f0 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/plan/maker/InstancePlanMakerImplV2.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/plan/maker/InstancePlanMakerImplV2.java @@ -34,6 +34,7 @@ import org.apache.pinot.common.request.context.ExpressionContext; import org.apache.pinot.common.request.context.FilterContext; import org.apache.pinot.common.request.context.OrderByExpressionContext; import org.apache.pinot.common.request.context.predicate.Predicate; +import org.apache.pinot.common.utils.config.QueryOptionsUtils; import org.apache.pinot.core.plan.AcquireReleaseColumnsSegmentPlanNode; import org.apache.pinot.core.plan.AggregationPlanNode; import org.apache.pinot.core.plan.CombinePlanNode; @@ -52,7 +53,6 @@ import org.apache.pinot.core.query.prefetch.FetchPlannerRegistry; import org.apache.pinot.core.query.request.context.QueryContext; import org.apache.pinot.core.query.request.context.utils.QueryContextUtils; import org.apache.pinot.core.util.GroupByUtils; -import org.apache.pinot.core.util.QueryOptionsUtils; import org.apache.pinot.segment.spi.FetchContext; import org.apache.pinot.segment.spi.IndexSegment; import org.apache.pinot.spi.env.PinotConfiguration; diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/executor/ServerQueryExecutorV1Impl.java b/pinot-core/src/main/java/org/apache/pinot/core/query/executor/ServerQueryExecutorV1Impl.java index 803621d52a..92467ba316 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/executor/ServerQueryExecutorV1Impl.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/executor/ServerQueryExecutorV1Impl.java @@ -41,6 +41,7 @@ import org.apache.pinot.common.proto.Server; import org.apache.pinot.common.request.context.ExpressionContext; import org.apache.pinot.common.request.context.FilterContext; import org.apache.pinot.common.request.context.FunctionContext; +import org.apache.pinot.common.utils.config.QueryOptionsUtils; import org.apache.pinot.core.common.ExplainPlanRowData; import org.apache.pinot.core.common.ExplainPlanRows; import org.apache.pinot.core.common.Operator; @@ -65,7 +66,6 @@ import org.apache.pinot.core.query.request.context.QueryContext; import org.apache.pinot.core.query.request.context.TimerContext; import org.apache.pinot.core.query.request.context.utils.QueryContextConverterUtils; import org.apache.pinot.core.query.utils.idset.IdSet; -import org.apache.pinot.core.util.QueryOptionsUtils; import org.apache.pinot.core.util.trace.TraceContext; import org.apache.pinot.segment.local.data.manager.SegmentDataManager; import org.apache.pinot.segment.local.data.manager.TableDataManager; diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/ExplainPlanDataTableReducer.java b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/ExplainPlanDataTableReducer.java index b1ebc5bfce..97b2fbae82 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/ExplainPlanDataTableReducer.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/ExplainPlanDataTableReducer.java @@ -30,6 +30,7 @@ import org.apache.pinot.common.metrics.BrokerMetrics; import org.apache.pinot.common.response.broker.BrokerResponseNative; import org.apache.pinot.common.response.broker.ResultTable; import org.apache.pinot.common.utils.DataSchema; +import org.apache.pinot.common.utils.config.QueryOptionsUtils; import org.apache.pinot.core.common.ExplainPlanRowData; import org.apache.pinot.core.common.ExplainPlanRows; import org.apache.pinot.core.operator.filter.EmptyFilterOperator; @@ -38,7 +39,6 @@ import org.apache.pinot.core.query.request.context.QueryContext; import org.apache.pinot.core.query.request.context.utils.QueryContextUtils; import org.apache.pinot.core.query.selection.SelectionOperatorUtils; import org.apache.pinot.core.transport.ServerRoutingInstance; -import org.apache.pinot.core.util.QueryOptionsUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/request/context/QueryContext.java b/pinot-core/src/main/java/org/apache/pinot/core/query/request/context/QueryContext.java index dbf8d64fbb..5c1bd2fe84 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/request/context/QueryContext.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/request/context/QueryContext.java @@ -35,11 +35,11 @@ import org.apache.pinot.common.request.context.FilterContext; import org.apache.pinot.common.request.context.FunctionContext; import org.apache.pinot.common.request.context.OrderByExpressionContext; import org.apache.pinot.common.request.context.RequestContextUtils; +import org.apache.pinot.common.utils.config.QueryOptionsUtils; import org.apache.pinot.core.plan.maker.InstancePlanMakerImplV2; import org.apache.pinot.core.query.aggregation.function.AggregationFunction; import org.apache.pinot.core.query.aggregation.function.AggregationFunctionFactory; import org.apache.pinot.core.util.MemoizedClassAssociation; -import org.apache.pinot.core.util.QueryOptionsUtils; /** --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org