This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a change to branch
Add-queue-based-throttling-to-ThrottleOnCriticalHeapUsageExecutor
in repository https://gitbox.apache.org/repos/asf/pinot.git
discard 62123e364f7 Implement comprehensive queue-based throttling for
ThrottleOnCriticalHeapUsageExecutor
add f4683f184c3 Annotate RebalanceConfig with ignoreUnknown = true (#16973)
add fb7d220d3ee Create Cluster conf for No dict stats collector (#16967)
add d49bd3c96e0 Remove SendingMailbox.complete method and add that logic
when EOS is sent (#16915)
add cc52edcc144 Bump io.grpc:grpc-bom from 1.75.0 to 1.76.0 (#16974)
add 738cdd27b00 Bump immutables.version from 2.11.4 to 2.11.5 (#16975)
add e257e761c32 Bump software.amazon.awssdk:bom from 2.35.1 to 2.35.2
(#16976)
add cdaa6883680 Task queue UI: surface debug tables (#16968)
add 9ddce8d6055 Only include kafka properties when creating kafka clients
(#16965)
add 04df7fedfe1 Enable table based filtering for minion subtask details
(#16977)
add a68a44b6c11 Trim of redundant IS_TRUE function when creation filter on
leaf stage (#16966)
add c811167b9ae Fix MSE registration for MIN_STRING, MAX_STRING, SUM_INT
(#16972)
add db45f36ee77 Override ZooKeeper Docker image repo for Bitnami Helm
chart (#16984)
add 6674ee36e5d Add support to configure the maximum minion tasks that can
be run per instance (#16981)
add 4b8ec201e93 Add table level distributed locking for ensuring minion
task generation is atomic (#16857)
add 1b25effac68 Bump immutables.version from 2.11.5 to 2.11.6 (#16988)
add c4fdf2ef943 Bump software.amazon.awssdk:bom from 2.35.2 to 2.35.3
(#16989)
add a7357d836bc Fix key map cleanup in group key generator (#16982)
add 10d3a1937a7 Publish Helm chart 0.3.5 with Bitnami ZooKeeper Docker
image repo fix (#16985)
add 2d4e416367f add AuthProvider while fetching segments to replace
(#16978)
add ce33d0d9184 Add support to process segment refresh messages
asynchronously if enabled (#16931)
add e2ff6cc7ded Clean up SSE query and statement optimizer handling
(#16991)
add a7fde6d78aa Fix function polymorphism for
ScalarTransformFunctionWrapper (#16992)
add 56684e980f2 Handle NaN and +/- Infinity in ROUND_DECIMAL (#16993)
add 57bfd12ed7e Bump software.amazon.awssdk:bom from 2.35.3 to 2.35.4
(#16998)
add 6cebaa97f09 Bump com.google.cloud:libraries-bom from 26.69.0 to
26.70.0 (#16997)
add 6b91eacd0f0 Bump com.squareup.okio:okio-bom from 3.16.0 to 3.16.1
(#16995)
add 47efe3eaa3b Fix error message for ScalarTransformFunctionWrapper when
no matching function signature is found (#16994)
add 89034e065a4 Add dictionary / metadata based optimization for MINSTRING
and MAXSTRING aggregations (#16983)
add 30233a1c6dd Automatically rewrite MIN / MAX on string col to MINSTRING
/ MAXSTRING (#16980)
add a1aab649f8d Refactor set operators to allow multiple children for
UNION / UNION ALL (#16990)
add a6a94c57225 Add LONG specific aggregation functions for MIN / MAX /
SUM (#17001)
add a31a2c1b452 fix shading for io.grpc for module pinot-spark-3-connector
(#17000)
add 8ffded01890 Implement comprehensive queue-based throttling for
ThrottleOnCriticalHeapUsageExecutor
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (62123e364f7)
\
N -- N -- N
refs/heads/Add-queue-based-throttling-to-ThrottleOnCriticalHeapUsageExecutor
(8ffded01890)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
helm/index.yaml | 28 +
helm/pinot-0.3.5.tgz | Bin 0 -> 76335 bytes
helm/pinot/Chart.yaml | 2 +-
helm/pinot/charts/zookeeper-13.2.0.tgz | Bin 49265 -> 0 bytes
helm/pinot/charts/zookeeper-13.8.7.tgz | Bin 0 -> 55335 bytes
helm/pinot/requirements.lock | 6 +-
helm/pinot/values.yaml | 9 +-
.../BaseSingleStageBrokerRequestHandler.java | 5 +-
.../function/scalar/ArithmeticFunctions.java | 10 +
.../pinot/common/metadata/ZKMetadataProvider.java | 6 +
.../pinot/common/metrics/ControllerTimer.java | 1 +
.../apache/pinot/common/metrics/ServerGauge.java | 10 +-
.../apache/pinot/common/metrics/ServerMeter.java | 10 +
.../request/context/RequestContextUtils.java | 40 +-
.../pinot/common/utils/helix/HelixHelper.java | 9 +
pinot-connectors/pinot-spark-3-connector/pom.xml | 6 +
.../apache/pinot/controller/ControllerConf.java | 9 +
.../api/resources/PinotTaskRestletResource.java | 31 +-
.../core/minion/DistributedTaskLockManager.java | 353 ++++++
.../core/minion/PinotHelixTaskResourceManager.java | 37 +-
.../helix/core/minion/PinotTaskManager.java | 220 ++--
.../helix/core/minion/TaskSchedulingInfo.java | 6 +
.../helix/core/rebalance/RebalanceConfig.java | 2 +
.../src/main/resources/app/pages/SubTaskDetail.tsx | 4 +-
.../src/main/resources/app/pages/TaskDetail.tsx | 2 +-
.../src/main/resources/app/pages/TaskQueue.tsx | 28 +-
.../src/main/resources/app/requests/index.ts | 4 +-
.../main/resources/app/utils/PinotMethodUtils.ts | 6 +-
.../minion/DistributedTaskLockManagerTest.java | 365 +++++++
.../minion/PinotHelixTaskResourceManagerTest.java | 23 +-
.../PinotTaskManagerDistributedLockingTest.java | 1118 ++++++++++++++++++++
.../java/org/apache/pinot/core/auth/Actions.java | 1 +
.../core/data/manager/BaseTableDataManager.java | 42 +-
.../core/data/manager/InstanceDataManager.java | 5 +
.../provider/DefaultTableDataManagerProvider.java | 7 +-
.../manager/provider/TableDataManagerProvider.java | 6 +-
.../ThrottleOnCriticalHeapUsageExecutor.java | 139 +--
.../query/NonScanBasedAggregationOperator.java | 57 +-
.../function/RoundDecimalTransformFunction.java | 15 +-
.../function/TransformFunctionFactory.java | 31 +-
.../pinot/core/plan/AggregationPlanNode.java | 14 +-
.../function/AggregationFunctionFactory.java | 6 +
...nction.java => MaxLongAggregationFunction.java} | 129 +--
...nction.java => MinLongAggregationFunction.java} | 119 +--
...nction.java => SumLongAggregationFunction.java} | 30 +-
.../groupby/DictionaryBasedGroupKeyGenerator.java | 134 +--
.../pinot/core/query/optimizer/QueryOptimizer.java | 17 +-
.../AggregateFunctionRewriteOptimizer.java | 98 ++
.../statement/JsonStatementOptimizer.java | 610 -----------
.../optimizer/statement/StatementOptimizer.java | 3 +-
.../statement/StringPredicateFilterOptimizer.java | 3 +-
.../query/scheduler/resources/ResourceManager.java | 8 +-
.../data/function/ArithmeticFunctionsTest.java | 24 +
.../BaseTableDataManagerAcquireSegmentTest.java | 3 +-
...ableDataManagerEnqueueSegmentToReplaceTest.java | 336 ++++++
.../data/manager/BaseTableDataManagerTest.java | 177 +++-
.../offline/DimensionTableDataManagerTest.java | 3 +-
.../RoundDecimalTransformFunctionTest.java | 20 +
...st.java => MaxLongAggregationFunctionTest.java} | 72 +-
...st.java => MinLongAggregationFunctionTest.java} | 72 +-
...st.java => SumLongAggregationFunctionTest.java} | 24 +-
.../statement/JsonStatementOptimizerTest.java | 241 -----
.../StringPredicateFilterOptimizerTest.java | 16 +-
.../core/query/optimizer/statement/TestHelper.java | 7 +-
.../apache/pinot/queries/BaseJsonQueryTest.java | 2 +-
.../org/apache/pinot/queries/BaseQueriesTest.java | 5 +-
.../pinot/queries/JsonMalformedIndexTest.java | 2 +-
.../pinot/queries/TextSearchQueriesTest.java | 6 +-
.../tests/BaseClusterIntegrationTestSet.java | 20 +
.../tests/ErrorCodesIntegrationTest.java | 8 -
.../tests/NullHandlingIntegrationTest.java | 41 +
.../tests/OfflineClusterIntegrationTest.java | 52 +-
.../FailureInjectingTableDataManagerProvider.java | 7 +-
.../org/apache/pinot/minion/BaseMinionStarter.java | 16 +
.../java/org/apache/pinot/minion/MinionConf.java | 5 +
...BenchmarkAggregateGroupByOrderByQueriesSSE.java | 6 +-
.../perf/BenchmarkDimensionTableOverhead.java | 3 +-
.../KafkaPartitionLevelConnectionHandler.java | 26 +-
.../KafkaPartitionLevelConnectionHandler.java | 26 +-
.../rules/PinotAggregateFunctionRewriteRule.java | 114 ++
.../calcite/rel/rules/PinotQueryRuleSets.java | 3 +
.../src/test/resources/queries/AggregatePlans.json | 48 +
.../pinot/query/mailbox/GrpcSendingMailbox.java | 12 +-
.../query/mailbox/InMemorySendingMailbox.java | 6 +-
.../apache/pinot/query/mailbox/SendingMailbox.java | 26 +-
.../apache/pinot/query/runtime/QueryRunner.java | 5 +-
.../runtime/operator/exchange/BlockExchange.java | 58 +-
...asedSetOperator.java => BinarySetOperator.java} | 49 +-
.../runtime/operator/set/IntersectAllOperator.java | 2 +-
.../runtime/operator/set/IntersectOperator.java | 2 +-
.../runtime/operator/set/MinusAllOperator.java | 2 +-
.../query/runtime/operator/set/MinusOperator.java | 2 +-
.../query/runtime/operator/set/SetOperator.java | 76 +-
.../runtime/operator/set/UnionAllOperator.java | 36 +-
.../query/runtime/operator/set/UnionOperator.java | 73 +-
.../plan/server/ServerPlanRequestUtils.java | 2 +-
.../pinot/query/mailbox/MailboxServiceTest.java | 5 -
.../operator/exchange/BlockExchangeTest.java | 7 +-
.../runtime/operator/set/UnionAllOperatorTest.java | 7 +-
.../runtime/operator/set/UnionOperatorTest.java | 4 +-
.../query/runtime/queries/QueryRunnerTestBase.java | 3 +
.../src/test/resources/queries/Aggregates.json | 31 +-
.../src/test/resources/queries/SetOpsNonH2.json | 9 +
.../local/data/manager/TableDataManager.java | 4 +-
.../stats/SegmentPreIndexStatsCollectorImpl.java | 3 +-
.../segment/index/loader/ForwardIndexHandler.java | 3 +-
.../defaultcolumn/BaseDefaultColumnHandler.java | 3 +-
.../segment/local/utils/ClusterConfigForTable.java | 87 ++
.../local/utils/ConsistentDataPushUtils.java | 3 +-
.../local/utils/SegmentReloadSemaphore.java | 4 +-
.../pinot/segment/spi/AggregationFunctionType.java | 9 +-
.../server/starter/helix/BaseServerStarter.java | 5 +
.../starter/helix/HelixInstanceDataManager.java | 22 +-
.../helix/HelixInstanceDataManagerConfig.java | 10 +
.../apache/pinot/server/api/BaseResourceTest.java | 2 +-
.../config/instance/InstanceDataManagerConfig.java | 2 +
.../apache/pinot/spi/utils/CommonConstants.java | 5 +
pom.xml | 10 +-
118 files changed, 4084 insertions(+), 1714 deletions(-)
create mode 100644 helm/pinot-0.3.5.tgz
delete mode 100644 helm/pinot/charts/zookeeper-13.2.0.tgz
create mode 100644 helm/pinot/charts/zookeeper-13.8.7.tgz
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/DistributedTaskLockManager.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/minion/DistributedTaskLockManagerTest.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/minion/PinotTaskManagerDistributedLockingTest.java
rename {pinot-spi/src/main/java/org/apache/pinot/spi =>
pinot-core/src/main/java/org/apache/pinot/core}/executor/ThrottleOnCriticalHeapUsageExecutor.java
(77%)
copy
pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/{SumIntAggregationFunction.java
=> MaxLongAggregationFunction.java} (53%)
copy
pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/{SumIntAggregationFunction.java
=> MinLongAggregationFunction.java} (56%)
copy
pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/{SumIntAggregationFunction.java
=> SumLongAggregationFunction.java} (88%)
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/statement/AggregateFunctionRewriteOptimizer.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/statement/JsonStatementOptimizer.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerEnqueueSegmentToReplaceTest.java
copy
pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/{MaxAggregationFunctionTest.java
=> MaxLongAggregationFunctionTest.java} (69%)
copy
pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/{MinAggregationFunctionTest.java
=> MinLongAggregationFunctionTest.java} (71%)
copy
pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/{SumIntAggregationFunctionTest.java
=> SumLongAggregationFunctionTest.java} (85%)
delete mode 100644
pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/statement/JsonStatementOptimizerTest.java
create mode 100644
pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotAggregateFunctionRewriteRule.java
rename
pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/set/{RightRowSetBasedSetOperator.java
=> BinarySetOperator.java} (70%)
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/ClusterConfigForTable.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
