This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a change to branch
Add-queue-based-throttling-to-ThrottleOnCriticalHeapUsageExecutor
in repository https://gitbox.apache.org/repos/asf/pinot.git
discard 1f92dd6d3c3 Implement comprehensive queue-based throttling for
ThrottleOnCriticalHeapUsageExecutor
add 48cd24b9701 Refactor DefaultClusterConfigChangeHandler to common
module to enable sharing same infra across pinot components (#16662)
add 1593c937de1 Bump net.openhft:chronicle-bom from 2.27ea74 to 2.27ea75
(#16695)
add 388497cabfc [audit] Implement support for comma-separated header
filtering (#16694)
add 6d1b8d69888 Bump software.amazon.awssdk:bom from 2.32.29 to 2.32.30
(#16697)
add ed5b6997ee9 Bump software.amazon.awssdk:bom from 2.32.29 to 2.32.31
(#16701)
add 4b457b36685 reduce logging (#16698)
add 44a7b8f82c4 Removes ingestion metrics for consuming -> offline
transition (#16657)
add 71112fa2d9c docs: enhance copilot-instructions.md with comprehensive
guidelines (#16693)
add eb1e77aebeb Fix rebalance for RealtimeSegmentAssignment for ONLINE
segments with COMMITTING status (#16706)
add 4c00889a38d Bump com.gradle:common-custom-user-data-maven-extension
(#16702)
add ceb3ef2b2b8 Fix early termination on MSE operator (#16696)
add 7b1d881e909 [Query Resource Isolation]Interface for Workload Stats
Collection (#16340)
add fb06abc2c2f [Query Resource Isolation] Fix Refresh message (#16636)
add 78b76641f06 Add extra JSON fields (schemas and tableConfigs) in the
validateMultiStageQuery API - MSE (#16626)
add c3ce1cff782 [timeseries] Returning TimeSeriesBlock to API handlers to
eliminate extraneous transformation (#16641)
add 689c2f32d7a Avoid npe in the getdocIds code for Upserts (#16710)
add 0258b9217bb Handle null values gracefully in SSE's post aggregation
functions (#16711)
add 8f91ba79d76 Bump hadoop.version from 3.4.1 to 3.4.2 (#16712)
add 9d06452ca15 Bump com.fasterxml.jackson:jackson-bom from 2.19.2 to
2.20.0 (#16714)
add e9a778d9221 Bump org.apache.zookeeper:zookeeper from 3.9.3 to 3.9.4
(#16715)
add 095fc67dd6a Bump com.microsoft.azure:msal4j from 1.22.0 to 1.23.0
(#16716)
add 5d3e1884722 Bump com.google.cloud:libraries-bom from 26.66.0 to
26.67.0 (#16717)
add 1845ee02ce0 [audit] Enhance user identity resolution with custom
header and jwt. +DI refactor (#16709)
add 828b72427de Bump io.dropwizard.metrics:metrics-bom from 4.2.34 to
4.2.35 (#16713)
add 0e7393a1ac0 Cleanup some unused code in pinot-query-runtime (#16718)
add c44fd796a5b [Auto reset 2/3]Introduce topic 'inactive' status (#16692)
add ed5e306c0fd Add TenantRebalanceChecker for improved tenant rebalance
monitoring (#16455)
add 3388f7ead17 force commit API does not fail for non-existing segment
names (#16725)
add ee7b28c967c Adds Byte based throttling (#16572)
add 2bc41e415bc Bump software.amazon.awssdk:bom from 2.32.31 to 2.33.0
(#16732)
add 4d5b0b803d0 Bump org.webjars:swagger-ui from 5.27.1 to 5.28.0 (#16734)
add 0dd6aee058a Bump com.gradle:common-custom-user-data-maven-extension
(#16735)
add 375e0dfb628 Bump io.dropwizard.metrics:metrics-bom from 4.2.35 to
4.2.36 (#16736)
add 96b118a52f0 Bump org.apache.httpcomponents.core5:httpcore5 from 5.3.4
to 5.3.5 (#16738)
add f5256e1f2de Bump org.jetbrains:annotations from 26.0.2 to 26.0.2-1
(#16739)
add b9014763577 Bump org.yaml:snakeyaml from 2.4 to 2.5 (#16730)
add f7a7f2d9254 Bump org.checkerframework:checker-qual from 3.49.5 to
3.50.0 (#16731)
add 51838e17757 Upgrade avro lib to 1.12.0 (#16729)
add 22c4e446c69 Allow skip CRC check during segment load when server
starting. (#16432)
add e1a0b73dc49 Commit time compaction for upsert tables (#16344)
add 321111595df Add support for SNAPSHOT_WITH_DELETE validDocIdsType to
server APIs. (#16726)
add d6eb3c7dc6c Bump net.openhft:chronicle-bom from 2.27ea75 to 2.27ea76
(#16742)
add 278e5d6d6f9 Add support for minimum should match in lucene text search
(#16650)
add a58ec4c7522 [audit] Replace URL exclusion with PathMatcher glob
patterns (#16723)
add 37fbf1c166c Extract segment metadata from slim tar ball (#16635)
add 4b4e03b1252 Minor cleanups and refactors in instance selector module
(#16744)
add e8f5c93d7ce Bump software.amazon.awssdk:bom from 2.33.0 to 2.33.1
(#16748)
add 90067c5a763 Bump parquet.version from 1.15.2 to 1.16.0 (#16749)
add f314c253ff0 Bump net.openhft:chronicle-bom from 2.27ea76 to 2.27ea77
(#16750)
add 48de81a29fd Support to create combined Lucene Text Index Files and
merge into columns.psf (#16688)
add 881b6296fac [audit] Implement request body handling with payload size
limits and truncation support (#16747)
add 71ff8e43080 Do not remove soft-delete record for commit time
compaction (#16743)
add 9ead8a9dc88 Add Pinot Proxy, unified secureMode, and comprehensive
gRPC support to Spark-3 connector (#16666)
add 8f75d584b33 Do not return UNHEALTHY status when Temporary
ConcurrentModificationException from Kafka (#16741)
add 2adf54b60a2 [audit] Make audit config prefix component-specific
(controller, broker, server) (#16766)
add d4d1f8f9063 Add Multiple Query Requests field into
validateMultiStageQuery API (#16746)
add 896406d226e Add $partitionId as a new virtual column (#16721)
add 9b633ad4ef9 Adding ITs and UTs for UpsertCompactMergeTask (#16592)
add b1632d87197 Bump software.amazon.awssdk:bom from 2.33.1 to 2.33.3
(#16763)
add a60438aa9cc Bump org.codehaus.mojo:versions-maven-plugin from 2.18.0
to 2.19.0 (#16760)
add 1ed67bd1d6f Bump io.github.hakky54:sslcontext-kickstart-bom from 9.1.0
to 9.2.1 (#16761)
add b97e94af64c Bump com.azure:azure-sdk-bom from 1.2.37 to 1.2.38 (#16754)
add dd981fced7b Bump com.nimbusds:nimbus-jose-jwt from 10.4.2 to 10.5
(#16762)
add 5110a25d35b Support more kafka client metrics in jmx exporter
templates (#16745)
add f8d2c697e37 Improve BenchmarkOrderByQueries (#16764)
add af3dc6a3b1e Modify segment metadata call (#14250)
add 5ea97acdb21 Bump software.amazon.awssdk:bom from 2.33.3 to 2.33.5
(#16780)
add 71fad9eba56 Bump org.webjars:swagger-ui from 5.28.0 to 5.28.1 (#16774)
add 541db1a2233 Bump org.asynchttpclient:async-http-client from 3.0.2 to
3.0.3 (#16771)
add ceb4e9ef626 [maven-release-plugin] prepare for next development
iteration (#16759)
add a892361499a Removes recentlyOffloadedConsumingSegments cache (#16740)
add 97cb8a82936 Fix query generator REGEXP_LIKE case sensitivity mismatch
between Pinot and H2 (#16768)
add 364e26f05ff Fix table config validation for upsert / dedup tables with
COMPLETED instance partitions (#16787)
add 6ff9e013892 Make DefaultClusterConfigChangeHandler thread safe (#16778)
add 53b7ebb5285 Add memory and GC time tracking to execution statistics
(#16576)
add 1cee0a39c6c disable spammy test that leaks query threads (#16790)
add 0d16e021528 Rewrite isDirectExchangeCompatible to support joins
without partition functions (#16603)
add 9dea5537c27 Add support for providing custom retention time for
untracked segments (#16719)
add 862c9bdf42f [Ngram UDF] Add Ngrams MV UDFs and minor refactor (#16671)
add d45f508a7b3 Optimize group ID generation by reducing memory allocation
in GroupIdGenerator implementations (#16798)
add db80c9e2906 Add more template to capture the kafka client-id with
double-quoted and use | for prefixes (#16799)
add 41f36160b08 Use /table/livebrokers/ api in pinot spark connector
(#14802)
add 93de8f0dc7c Add support for commit time compaction with
columnMajorBuild (#16769)
add 9286da251fa Enable Json Index for MAP data type (#16808)
add 19ae6b9e64a Add URL filter include patterns for audit logging with
priority semantics (#16807)
add c897df7c894 updated putNodeData and zookeeperPutData to pass data in
the payload/body instead of a query param (#16809)
add fc27fbfe592 Bump software.amazon.awssdk:bom from 2.33.5 to 2.33.8
(#16804)
add 09de32bf7f3 Bump net.openhft:chronicle-bom from 2.27ea77 to 2.27ea78
(#16805)
add c9f11992225 Bump org.jetbrains.kotlin:kotlin-bom from 2.2.10 to 2.2.20
(#16788)
add 32aa1905bea Bump org.jline:jline from 3.30.5 to 3.30.6 (#16795)
add 69b0818421c Bump org.checkerframework:checker-qual from 3.50.0 to
3.51.0 (#16796)
add 6edc8643ac4 Bump jakarta.activation:jakarta.activation-api from 1.2.2
to 2.1.4 (#16806)
add ef804c770ef Bump net.alchim31.maven:scala-maven-plugin from 4.9.5 to
4.9.6 (#16815)
add 5c875c78dde Bump com.google.cloud:libraries-bom from 26.67.0 to
26.68.0 (#16816)
add ccfd78aa268 Bump software.amazon.awssdk:bom from 2.33.8 to 2.33.9
(#16817)
add 9f43495d1ee Bump com.microsoft.azure:msal4j from 1.23.0 to 1.23.1
(#16818)
add c9e2bf26e82 [Auto reset 3/3]Add backfill manager and handler
interfaces (#16724)
add 05c685b1649 Revert Jakarta activation-api from 2.1.4 back to 1.2.2
(#16820)
add 71a3b097ea9 Add easier to read stats (#16794)
add 700cf053285 Optimize RetentionManager performance by replacing List
with Set for segment lookups (#16824)
add 8d59812628e Bump io.dropwizard.metrics:metrics-bom from 4.2.36 to
4.2.37 (#16828)
add 34af03331da Bump com.gradle:common-custom-user-data-maven-extension
(#16829)
add 462ef49329d [minion] configure numConcurrentTasksPerInstance for
minion tenant (#16777)
add c4eaa9f1955 Bump software.amazon.awssdk:bom from 2.33.9 to 2.33.10
(#16827)
add 10d72b1644f [multistage] Add Support for Lite Mode Fan-Out Adjusted
Limit (#16822)
add 7e4bc52158a UCT: Add getValidDocIdsType method for UpsertConfig
handling and validation (#16812)
add 9c3a16be74d Fix retry of partial table deletion (#16835)
add fd39662b083 Fix auto reset and optimize multi-topic ingestion (#16833)
add 5cbc73335df Bump com.gradle:develocity-maven-extension from 2.1 to 2.2
(#16837)
add 87e4e6305bc Bump software.amazon.awssdk:bom from 2.33.10 to 2.33.11
(#16838)
add b1c3fda4b34 [routing] Handle New Segments in
MultiStageReplicaGroupSelector (#16834)
add 2385ca24811 fix wrong timeunit in ExecutorServiceUtils close (#16842)
add ac6bdef51fa Add QUERIES_THROTTLED metric for
ThrottleOnCriticalHeapUsageExecutor (#16676)
add 9bac2500278 issue-16707: Add cursor based pagination support to java
client (#16782)
add ec2eb3cc005 [audit] Extend audit logging support to broker REST APIs
(#16823)
add 6182260c927 Add support for MAPa data type in GenericRowSerializer and
GenericRowDeserializer for File Ingestion Tasks for MAP data type. (#16843)
add 17a95645426 Bump software.amazon.awssdk:bom from 2.33.11 to 2.33.12
(#16846)
add f95ada1ab35 Fix the error handling in
SequentialSortedGroupByCombineOperator (#16840)
add 8d1fb726799 Renamed and updated dependency (#16751)
add 73456856946 [audit] Add response auditing capability to Pinot's audit
logging system (#16851)
add 9ab4a6efbd7 Bump software.amazon.awssdk:bom from 2.33.12 to 2.33.13
(#16855)
add af7ca7c638f Bump org.assertj:assertj-bom from 3.27.4 to 3.27.5 (#16854)
add d1422a970aa Fix server metric initialization (#16850)
add b4a7c88135e Fix NPE for MergeRollupTaskGenerator delay metrics (#16844)
add 7f60f131603 Early short circuit with AND & OR optimizations (#16583)
add cf5aa562df8 Bump org.apache.httpcomponents.core5:httpcore5 from 5.3.5
to 5.3.6 (#16860)
add 684920a3d55 Bump org.codehaus.mojo:versions-maven-plugin from 2.19.0
to 2.19.1 (#16861)
add 3148745a5b0 Bump log4j.version from 2.25.1 to 2.25.2 (#16862)
add b23d5a581b4 Bump software.amazon.awssdk:bom from 2.33.13 to 2.34.0
(#16863)
add d6c3c848d41 Bump com.google.errorprone:error_prone_annotations from
2.41.0 to 2.42.0 (#16865)
add aed9d83008a Fix LagValueWindowFunction type used for the default value
(#16853)
add 9dda1211fa2 Fix fsm creation when segment is in committing state
(#16847)
add cf544cdb629 Fix broker metrics for GrpcMailboxServer (#16870)
add 91084f455fe Fix a typo in MailboxContentObserver where mailbox should
be volatile (#16871)
add 96a5895f2d8 Change BrokerRoutingManager synchronization mechanism to
allow more parallelism (#16791)
add 4abe0ac0f9d Modify AdaptiveSizeBasedWriter based on available capacity
on disk (#16803)
add 0811d3e3609 Create a DescDocIdSetOperator and improve order by desc
when columns are sorted (#16789)
add 4dedc5c3db6 Bump org.assertj:assertj-bom from 3.27.5 to 3.27.6 (#16873)
add 4ba63e2b8d1 Bump jna.version from 5.17.0 to 5.18.0 (#16875)
add d89dc7c1d31 [audit] Add metrics for monitoring audit log performance
(#16869)
add 42b7e275082 [Cleaup] Remove IntIteratorDocIdSetOperator (#16877)
add fa9ab8bea40 Reduce memory footprint for MailboxContentObserver and
ReceivingMailbox (#16872)
add ad47b6e718a Bump net.openhft:chronicle-bom from 2.27ea78 to 2.27ea80
(#16883)
add aec07cc82f0 Bump software.amazon.awssdk:bom from 2.34.0 to 2.34.2
(#16881)
add 6bb5bdc69de Bump spark3.version from 3.5.6 to 3.5.7 (#16882)
add 1b6866d5aea Add SumIntAggregationFunction for optimized INT column
aggregation (#16704)
add 7288f73b885 Configure switch for DICTIONARY scan vs RAW scan in REGEX
LIKE evaluation (#16879)
add d39f93ddd3e Fix bug in default handling for map type stats collector
(#16888)
add 6e28d37f577 Bump org.apache.commons:commons-lang3 from 3.18.0 to
3.19.0 (#16892)
add b09ab1ba0ad Bump com.h2database:h2 from 2.3.232 to 2.4.240 (#16893)
add 0ba95e3d918 Bump immutables.version from 2.11.3 to 2.11.4 (#16894)
add 0ea5bceafaa Bump flink.version from 1.20.2 to 1.20.3 (#16896)
add 9a0f596e627 Bump software.amazon.awssdk:bom from 2.34.2 to 2.34.3
(#16897)
add c8c8f567dad Fix null handling for SumIntAggregationFunction and add
performance enhancements (#16887)
add 03b9d0edc0e Fix null handling issue for string to timestamp casts
(#16885)
add 19b464863c9 [Query Workload Isolation] Cost-split support (#16672)
add 7f123000ca3 Batch delete deleted directory segments (#16848)
add b2760be562e Add benchmark for equi-join queries with various
configurations (#16825)
add 5bb03acf96e Perform table exists check early on for table creation
(#16898)
add 65006362231 Bump software.amazon.awssdk:bom from 2.34.3 to 2.34.4
(#16905)
add 1566f2d0327 Bump org.apache.maven.scm:maven-scm-provider-gitexe from
2.1.0 to 2.2.1 (#16906)
add 10a2a802282 Bump com.github.eirslett:frontend-maven-plugin from 1.15.1
to 1.15.4 (#16907)
add b53bb0dd8e1 Reset and unpause offline tables (#16856)
add 90eb0c17119 Minor cleanups in PinotHelixResourceManager (#16911)
add 9baf132185c Join improvements (#16830)
add 0647e367ec6 Bump org.apache.pulsar:pulsar-bom from 4.0.6 to 4.0.7
(#16918)
add 6667f73cf41 Bump software.amazon.awssdk:bom from 2.34.4 to 2.34.5
(#16916)
add 79b815dd6d2 Bump org.apache.httpcomponents.client5:httpclient5 from
5.5 to 5.5.1 (#16917)
add d8e4a13b382 Revert threshold for RegexpLikePredicateEvaluator based on
numDocs (#16902)
add 5d111363e68 Add support for cluster config change listeners for
controller periodic tasks. (#16908)
add d1a3bb8f793 Update doap for lastest release-1.4.0 (#16930)
add 8570f8db1a5 Bump com.google.cloud:libraries-bom from 26.68.0 to
26.69.0 (#16928)
add f63ab741bba Bump axios from 0.27.2 to 0.30.2 in
/pinot-controller/src/main/resources (#16924)
add 7432f0152a1 Bump software.amazon.awssdk:bom from 2.34.5 to 2.34.6
(#16927)
add 2b552ceb7ae Bump org.codehaus.mojo:exec-maven-plugin from 3.5.1 to
3.6.0 (#16929)
add 88cc4353ef7 Fix some properties in ColumnMetadata (#16921)
add a7cad307adf Change the globalLock of BrokerRoutingManager to be fair
to prevent writer starvation (#16932)
add 9c27d7ead81 Fix the wrong import of Preconditions (#16935)
add 8a416da0466 Introduce QueryExecutionContext to manage query life cycle
(#16728)
add 9fd7fa5ddd7 Add UserAgent in GCS requests (#16884)
add 10644e30de4 Shade grpc library for pinot-spark-3-connector (#16923)
add f3bee79e4e0 Bump jna.version from 5.18.0 to 5.18.1 (#16936)
add c80a6fadb90 Bump it.unimi.dsi:fastutil from 8.5.16 to 8.5.17 (#16937)
add 251014731bb Bump software.amazon.awssdk:bom from 2.34.6 to 2.34.7
(#16938)
add e7551cb8d05 Implement comprehensive queue-based throttling for
ThrottleOnCriticalHeapUsageExecutor
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (1f92dd6d3c3)
\
N -- N -- N
refs/heads/Add-queue-based-throttling-to-ThrottleOnCriticalHeapUsageExecutor
(e7551cb8d05)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/copilot-instructions.md | 135 +-
.mvn/extensions.xml | 4 +-
LICENSE-binary | 4 +-
doap_Pinot.rdf | 6 +-
.../etc/jmx_prometheus_javaagent/configs/pinot.yml | 84 +-
.../jmx_prometheus_javaagent/configs/server.yml | 58 +
pinot-broker/pom.xml | 2 +-
.../broker/api/resources/PinotBrokerDebug.java | 12 +-
.../broker/api/resources/PinotClientRequest.java | 28 +-
.../broker/broker/BrokerAdminApiApplication.java | 7 +-
.../broker/broker/helix/BaseBrokerStarter.java | 61 +-
.../BrokerUserDefinedMessageHandlerFactory.java | 29 +-
.../requesthandler/BaseBrokerRequestHandler.java | 162 +-
.../BaseSingleStageBrokerRequestHandler.java | 59 +-
.../requesthandler/BrokerRequestHandler.java | 7 +-
.../BrokerRequestHandlerDelegate.java | 9 +-
.../requesthandler/GrpcBrokerRequestHandler.java | 8 +-
.../MultiStageBrokerRequestHandler.java | 73 +-
.../SingleConnectionBrokerRequestHandler.java | 10 +-
.../requesthandler/TimeSeriesRequestHandler.java | 41 +-
.../pinot/broker/routing/BrokerRoutingManager.java | 720 ++++---
.../ServerSelectionContext.java | 12 +-
.../instanceselector/BalancedInstanceSelector.java | 103 +-
.../instanceselector/BaseInstanceSelector.java | 94 +-
.../instanceselector/InstanceSelectorFactory.java | 4 +-
.../MultiStageReplicaGroupSelector.java | 25 +-
.../ReplicaGroupInstanceSelector.java | 92 +-
.../instanceselector/SegmentInstanceCandidate.java | 6 +
.../routing/instanceselector/SegmentStates.java | 19 +-
.../BaseSingleStageBrokerRequestHandlerTest.java | 12 +-
.../LiteralOnlyBrokerRequestTest.java | 8 +-
.../BrokerRoutingManagerConcurrencyTest.java | 1994 ++++++++++++++++++++
.../MultiStageReplicaGroupSelectorTest.java | 39 +-
pinot-clients/pinot-java-client/pom.xml | 2 +-
.../apache/pinot/client/BaseResultSetGroup.java | 73 +
.../org/apache/pinot/client/BrokerResponse.java | 31 +-
.../java/org/apache/pinot/client/Connection.java | 71 +-
.../org/apache/pinot/client/ConnectionFactory.java | 5 +
.../pinot/client/CursorAwareBrokerResponse.java | 131 ++
.../org/apache/pinot/client/CursorCapable.java | 188 ++
.../apache/pinot/client/CursorResultSetGroup.java | 121 ++
.../client/JsonAsyncHttpPinotClientTransport.java | 291 ++-
.../java/org/apache/pinot/client/ResultCursor.java | 165 ++
.../org/apache/pinot/client/ResultCursorImpl.java | 366 ++++
.../org/apache/pinot/client/ResultSetGroup.java | 2 +-
.../apache/pinot/client/ConnectionFactoryTest.java | 144 ++
.../client/CursorAwareBrokerResponseTest.java | 163 ++
.../pinot/client/CursorResultSetGroupTest.java | 90 +
.../JsonAsyncHttpPinotClientTransportTest.java | 179 ++
.../org/apache/pinot/client/ResultCursorTest.java | 353 ++++
pinot-clients/pinot-jdbc-client/pom.xml | 2 +-
pinot-clients/pom.xml | 2 +-
pinot-common/pom.xml | 8 +-
.../org/apache/pinot/common/audit/AuditConfig.java | 83 +-
.../pinot/common/audit/AuditConfigManager.java | 134 +-
.../org/apache/pinot/common/audit/AuditEvent.java | 66 +-
.../pinot/common/audit/AuditIdentityResolver.java | 122 ++
.../apache/pinot/common/audit/AuditLogFilter.java | 106 +-
.../apache/pinot/common/audit/AuditMetrics.java | 159 ++
.../pinot/common/audit/AuditRequestProcessor.java | 156 +-
.../pinot/common/audit/AuditResponseContext.java | 39 +-
.../pinot/common/audit/AuditServiceBinder.java | 60 +
.../pinot/common/audit/AuditUrlPathFilter.java | 139 ++
.../config}/DefaultClusterConfigChangeHandler.java | 26 +-
.../common/config/provider/StaticTableCache.java | 223 +++
.../pinot/common/config/provider/TableCache.java | 94 +-
.../pinot/common/config/provider/ZkTableCache.java | 97 +-
.../pinot/common/datablock/DataBlockUtils.java | 16 +-
.../apache/pinot/common/datatable/DataTable.java | 7 +-
.../pinot/common/datatable/DataTableImplV4.java | 6 +-
.../org/apache/pinot/common/datatable/StatMap.java | 2 +-
.../common/function/scalar/InternalFunctions.java | 55 +-
.../function/scalar/string/NgramFunctions.java | 68 +-
.../messages/QueryWorkloadRefreshMessage.java | 7 +-
.../apache/pinot/common/metrics/BrokerMeter.java | 17 +-
.../apache/pinot/common/metrics/BrokerTimer.java | 5 +-
.../pinot/common/metrics/ControllerMeter.java | 8 +-
.../pinot/common/metrics/ControllerTimer.java | 5 +-
.../apache/pinot/common/metrics/ServerGauge.java | 3 +
.../apache/pinot/common/metrics/ServerMeter.java | 13 +-
.../response/PinotBrokerTimeSeriesResponse.java | 64 +-
.../response/broker/QueryProcessingException.java | 5 +
.../response/mapper/TimeSeriesResponseMapper.java | 126 ++
.../common/restlet/resources/ValidDocIdsType.java | 5 +
.../pinot/common/utils/RoaringBitmapUtils.java | 34 +
.../pinot/common/utils/ServiceStartableUtils.java | 3 -
.../common/utils/config/QueryOptionsUtils.java | 36 +-
.../utils/config/QueryWorkloadConfigUtils.java | 171 +-
pinot-common/src/main/proto/worker.proto | 2 +-
.../pinot/common/audit/AuditConfigManagerTest.java | 242 ++-
.../common/audit/AuditIdentityResolverTest.java | 259 +++
.../pinot/common/audit/AuditLogFilterTest.java | 388 ++++
.../pinot/common/audit/AuditMetricsTest.java | 113 ++
.../common/audit/AuditRequestProcessorTest.java | 420 ++++-
.../pinot/common/audit/AuditUrlPathFilterTest.java | 269 +++
.../function/AggregationFunctionTypeTest.java | 1 +
.../common/function/scalar/NgramFunctionTest.java | 88 +
.../function/scalar/StringFunctionsTest.java | 25 -
.../mapper/TimeSeriesResponseMapperTest.java | 157 ++
.../utils/config/QueryWorkloadConfigUtilsTest.java | 313 ++-
pinot-compatibility-verifier/pom.xml | 2 +-
pinot-connectors/pinot-flink-connector/pom.xml | 2 +-
pinot-connectors/pinot-spark-2-connector/pom.xml | 2 +-
pinot-connectors/pinot-spark-3-connector/README.md | 238 +++
.../documentation/read_model.md | 7 +-
.../pinot-spark-3-connector/examples/README.md | 55 +
.../read_pinot_from_proxy_with_auth_token.scala | 36 +-
pinot-connectors/pinot-spark-3-connector/pom.xml | 3 +-
.../spark/v3/datasource/DataExtractor.scala | 8 +
.../spark/v3/datasource/PinotDataSource.scala | 2 +-
.../connector/spark/v3/datasource/PinotScan.scala | 4 +-
.../spark/v3/datasource/PinotScanBuilder.scala | 2 +-
.../v3/datasource/SparkToPinotTypeTranslator.scala | 1 +
.../spark/v3/datasource/DataExtractorTest.scala | 47 +
.../SparkToPinotTypeTranslatorTest.scala | 2 +
pinot-connectors/pinot-spark-common/pom.xml | 7 +-
.../pinot/connector/spark/common/AuthUtils.scala | 51 +
.../pinot/connector/spark/common/GrpcUtils.scala | 125 ++
.../pinot/connector/spark/common/HttpUtils.scala | 131 +-
.../pinot/connector/spark/common/NetUtils.scala | 41 +-
.../spark/common/PinotClusterClient.scala | 119 +-
.../spark/common/PinotDataSourceReadOptions.scala | 116 +-
.../pinot/connector/spark/common/package.scala | 11 +
.../reader/PinotAbstractPartitionReader.scala | 2 +-
.../common/reader/PinotGrpcServerDataFetcher.scala | 68 +-
.../common/reader/PinotServerDataFetcher.scala | 84 +-
.../connector/spark/common/GrpcUtilsTest.scala | 165 ++
.../connector/spark/common/HttpUtilsTest.scala | 115 ++
.../spark/common/PinotClusterClientTest.scala | 91 +
.../common/PinotDataSourceReadOptionsTest.scala | 231 ++-
.../connector/spark/common/PinotSplitterTest.scala | 44 +-
pinot-connectors/pom.xml | 2 +-
pinot-controller/pom.xml | 2 +-
.../pinot/controller/BaseControllerStarter.java | 91 +-
.../apache/pinot/controller/ControllerConf.java | 82 +
.../pinot/controller/api/resources/Constants.java | 5 +
.../api/resources/PinotLogicalTableResource.java | 2 +-
.../api/resources/PinotQueryResource.java | 135 +-
.../PinotQueryWorkloadRestletResource.java | 178 +-
.../api/resources/PinotRealtimeTableResource.java | 68 +
.../api/resources/PinotSegmentRestletResource.java | 18 +-
.../api/resources/PinotTableRestletResource.java | 26 +-
.../api/resources/TableConfigsRestletResource.java | 10 +-
.../controller/helix/ControllerRequestClient.java | 33 +
.../controller/helix/SegmentStatusChecker.java | 2 +-
.../helix/core/PinotHelixResourceManager.java | 107 +-
.../helix/core/PinotTableIdealStateBuilder.java | 6 +-
.../helix/core/SegmentDeletionManager.java | 198 +-
.../segment/RealtimeSegmentAssignment.java | 12 +-
.../helix/core/minion/PinotTaskManager.java | 30 +-
.../core/minion/generator/BaseTaskGenerator.java | 49 +-
.../core/minion/generator/PinotTaskGenerator.java | 16 +-
.../core/minion/generator/TaskGeneratorUtils.java | 70 +
.../core/periodictask/ControllerPeriodicTask.java | 9 +-
.../RealtimeOffsetAutoResetHandler.java | 58 +
.../RealtimeOffsetAutoResetKafkaHandler.java | 173 ++
.../realtime/BlockingSegmentCompletionFSM.java | 4 +-
.../realtime/MissingConsumingSegmentFinder.java | 8 +-
.../realtime/PinotLLCRealtimeSegmentManager.java | 208 +-
.../segment/CommittingSegmentDescriptor.java | 12 +
.../SizeBasedSegmentFlushThresholdComputer.java | 71 +-
.../rebalance/tenant/DefaultTenantRebalancer.java | 288 ---
.../rebalance/tenant/TenantRebalanceChecker.java | 330 ++++
.../rebalance/tenant/TenantRebalanceContext.java | 130 ++
.../tenant/TenantRebalanceProgressStats.java | 14 +-
.../core/rebalance/tenant/TenantRebalancer.java | 332 +++-
.../tenant/ZkBasedTenantRebalanceObserver.java | 61 +-
.../helix/core/retention/RetentionManager.java | 150 +-
.../util/ServerSegmentMetadataReader.java | 38 +-
.../pinot/controller/util/TableMetadataReader.java | 114 +-
...r.java => OfflineSegmentValidationManager.java} | 102 +-
.../validation/RealtimeOffsetAutoResetManager.java | 254 +++
.../RealtimeSegmentValidationManager.java | 1 -
.../controller/workload/QueryWorkloadManager.java | 422 ++++-
.../workload/scheme/DefaultPropagationScheme.java | 7 +-
.../workload/scheme/PropagationScheme.java | 24 +-
.../workload/scheme/PropagationUtils.java | 188 +-
.../workload/scheme/TablePropagationScheme.java | 213 ++-
.../workload/scheme/TenantPropagationScheme.java | 33 +-
.../controller/workload/splitter/CostSplitter.java | 17 +-
.../workload/splitter/DefaultCostSplitter.java | 28 +-
.../src/main/resources/app/requests/index.ts | 4 +-
.../main/resources/app/utils/PinotMethodUtils.ts | 7 +-
.../src/main/resources/package-lock.json | 34 +-
pinot-controller/src/main/resources/package.json | 2 +-
.../api/PinotTableRestletResourceTest.java | 45 +-
.../PinotQueryResourceStaticValidationTest.java | 72 +
...ControllerPeriodicTaskStarterStatelessTest.java | 2 +-
.../pinot/controller/helix/TableCacheTest.java | 4 +-
.../core/minion/PinotTaskManagerStatelessTest.java | 6 +-
.../minion/generator/TaskGeneratorUtilsTest.java | 114 +-
.../PinotLLCRealtimeSegmentManagerTest.java | 45 +-
.../realtime/SegmentCompletionFSMFactoryTest.java | 11 +
...SizeBasedSegmentFlushThresholdComputerTest.java | 394 ++++
.../tenant/TenantRebalanceCheckerTest.java | 556 ++++++
.../rebalance/tenant/TenantRebalancerTest.java | 20 +-
.../helix/core/retention/RetentionManagerTest.java | 96 +-
.../core/util/SegmentDeletionManagerTest.java | 122 +-
.../RealtimeOffsetAutoResetManagerTest.java | 342 ++++
.../validation/ValidationManagerTest.java | 10 +-
.../controller/workload/PropagationSchemeTest.java | 281 +++
.../controller/workload/PropagationUtilsTest.java | 22 +-
pinot-core/pom.xml | 2 +-
...AggregateByQueryIdAccountantFactoryForTest.java | 98 -
.../CPUMemThreadLevelAccountingObjects.java | 190 --
.../HeapUsagePublishingAccountantFactory.java | 58 +-
.../PerQueryCPUMemAccountantFactory.java | 793 +++-----
.../PerQueryCPUMemAccountantFactoryForTest.java | 51 -
.../pinot/core/accounting/QueryAggregator.java | 643 -------
.../pinot/core/accounting/QueryMonitorConfig.java | 86 +-
.../core/accounting/QueryResourceAggregator.java | 413 ++++
.../core/accounting/QueryResourceTrackerImpl.java | 56 +
.../pinot/core/accounting/ResourceAggregator.java | 29 +-
.../accounting/ResourceUsageAccountantFactory.java | 387 ++--
.../core/accounting/ThreadResourceTrackerImpl.java | 73 +
.../pinot/core/accounting/WorkloadAggregator.java | 182 --
.../accounting/WorkloadResourceAggregator.java | 151 ++
.../apache/pinot/core/common/BlockDocIdSet.java | 16 +-
.../apache/pinot/core/common/MinionConstants.java | 8 +
.../org/apache/pinot/core/common/Operator.java | 2 +
.../core/data/manager/BaseTableDataManager.java | 28 +-
.../realtime/RealtimeConsumptionRateManager.java | 158 +-
.../realtime/RealtimeSegmentDataManager.java | 4 +-
.../manager/realtime/RealtimeTableDataManager.java | 13 +
.../ServerRateLimitConfigChangeListener.java | 40 +-
.../pinot/core/data/table/SortedRecordsMerger.java | 8 +-
.../core/metadata/DefaultMetadataExtractor.java | 27 +-
.../pinot/core/operator/BaseDocIdSetOperator.java | 64 +
.../apache/pinot/core/operator/BaseOperator.java | 18 +-
.../pinot/core/operator/BaseProjectOperator.java | 12 +-
.../core/operator/BitmapDocIdSetOperator.java | 73 +-
.../DocIdOrderedOperator.java} | 30 +-
.../pinot/core/operator/DocIdSetOperator.java | 22 +-
.../core/operator/InstanceResponseOperator.java | 22 +-
.../pinot/core/operator/ProjectionOperator.java | 32 +-
.../core/operator/ProjectionOperatorUtils.java | 7 +-
...tOperator.java => ReverseDocIdSetOperator.java} | 71 +-
.../pinot/core/operator/blocks/DocIdSetBlock.java | 11 +-
.../blocks/results/ExceptionResultsBlock.java | 6 +
.../blocks/results/GroupByResultsBlock.java | 6 +-
.../core/operator/combine/BaseCombineOperator.java | 133 +-
.../combine/BaseSingleBlockCombineOperator.java | 29 +-
.../operator/combine/CombineOperatorUtils.java | 77 -
.../operator/combine/GroupByCombineOperator.java | 10 +-
.../combine/SelectionOnlyCombineOperator.java | 15 +-
.../SequentialSortedGroupByCombineOperator.java | 78 +-
.../combine/SortedGroupByCombineOperator.java | 12 +-
.../ExpressionScanDocIdIterator.java | 25 +-
.../pinot/core/operator/docidsets/AndDocIdSet.java | 5 +
.../core/operator/docidsets/BitmapDocIdSet.java | 11 +
.../pinot/core/operator/docidsets/OrDocIdSet.java | 5 +
.../docidsets/RangelessBitmapDocIdSet.java | 10 +-
...yDocIdSet.java => ShortCircuitingDocIdSet.java} | 25 +-
.../core/operator/docidsets/SortedDocIdSet.java | 8 +
.../core/operator/filter/AndFilterOperator.java | 16 +-
.../operator/filter/CombinedFilterOperator.java | 7 +-
.../operator/filter/ExpressionFilterOperator.java | 2 +-
.../core/operator/filter/MapFilterOperator.java | 11 +
.../core/operator/filter/OrFilterOperator.java | 16 +-
.../predicate/PredicateEvaluatorProvider.java | 4 +-
.../RegexpLikePredicateEvaluatorFactory.java | 1 -
.../query/DictionaryBasedDistinctOperator.java | 58 +-
.../operator/query/SelectionOrderByOperator.java | 4 +-
.../SelectionPartiallyOrderedByDescOperation.java | 6 +-
...SelectionPartiallyOrderedByLinearOperator.java} | 31 +-
.../streaming/BaseStreamingCombineOperator.java | 45 +-
.../StreamingInstanceResponseOperator.java | 26 +-
.../core/operator/transform/TransformOperator.java | 17 +
.../transform/function/CastTransformFunction.java | 25 +-
.../function/LogicalOperatorTransformFunction.java | 2 +-
.../apache/pinot/core/plan/DocIdSetPlanNode.java | 7 +-
.../org/apache/pinot/core/plan/FilterPlanNode.java | 21 +-
.../apache/pinot/core/plan/ProjectPlanNode.java | 12 +-
.../apache/pinot/core/plan/SelectionPlanNode.java | 61 +-
.../query/aggregation/AggregationResultHolder.java | 13 +
.../aggregation/DoubleAggregationResultHolder.java | 23 +
.../aggregation/IntAggregateResultHolder.java | 10 +
...tHolder.java => LongAggregateResultHolder.java} | 24 +-
.../aggregation/ObjectAggregationResultHolder.java | 15 +
.../function/AggregationFunctionFactory.java | 2 +
.../BaseDistinctAggregateAggregationFunction.java | 4 +-
...irstDoubleValueWithTimeAggregationFunction.java | 5 +-
...FirstFloatValueWithTimeAggregationFunction.java | 5 +-
.../FirstIntValueWithTimeAggregationFunction.java | 5 +-
.../FirstLongValueWithTimeAggregationFunction.java | 5 +-
...irstStringValueWithTimeAggregationFunction.java | 5 +-
...LastDoubleValueWithTimeAggregationFunction.java | 5 +-
.../LastFloatValueWithTimeAggregationFunction.java | 5 +-
.../LastIntValueWithTimeAggregationFunction.java | 5 +-
.../LastLongValueWithTimeAggregationFunction.java | 5 +-
...LastStringValueWithTimeAggregationFunction.java | 5 +-
.../NullableSingleInputAggregationFunction.java | 37 +-
.../function/SumIntAggregationFunction.java | 219 +++
.../window/FunnelBaseAggregationFunction.java | 4 +-
.../groupby/DoubleGroupByResultHolder.java | 10 +
.../groupby/DummyAggregationResultHolder.java | 9 +
.../groupby/DummyGroupByResultHolder.java | 9 +
.../aggregation/groupby/GroupByResultHolder.java | 19 +
.../groupby/IntGroupByResultHolder.java | 10 +
...ultHolder.java => LongGroupByResultHolder.java} | 24 +-
.../groupby/ObjectGroupByResultHolder.java | 12 +
.../distinct/table/BigDecimalDistinctTable.java | 6 +-
.../query/distinct/table/BytesDistinctTable.java | 5 +-
.../query/distinct/table/DoubleDistinctTable.java | 5 +-
.../query/distinct/table/FloatDistinctTable.java | 5 +-
.../query/distinct/table/IntDistinctTable.java | 5 +-
.../query/distinct/table/LongDistinctTable.java | 5 +-
.../query/distinct/table/StringDistinctTable.java | 5 +-
.../pinot/core/query/executor/QueryExecutor.java | 26 +-
.../query/executor/ServerQueryExecutorV1Impl.java | 7 +-
.../postaggregation/PostAggregationFunction.java | 10 +-
.../query/reduce/AggregationDataTableReducer.java | 6 +-
.../pinot/core/query/reduce/BaseReduceService.java | 4 +-
.../core/query/reduce/BrokerReduceService.java | 38 +-
.../query/reduce/DistinctDataTableReducer.java | 4 +-
.../core/query/reduce/GroupByDataTableReducer.java | 53 +-
.../core/query/reduce/ResultReducerFactory.java | 5 +-
.../core/query/reduce/StreamingReduceService.java | 1 +
.../core/query/request/ServerQueryRequest.java | 48 +-
.../query/scheduler/BinaryWorkloadScheduler.java | 33 +-
.../core/query/scheduler/PriorityScheduler.java | 28 +-
.../pinot/core/query/scheduler/QueryScheduler.java | 100 +-
.../query/scheduler/QuerySchedulerFactory.java | 54 +-
.../core/query/scheduler/WorkloadScheduler.java | 31 +-
.../query/scheduler/fcfs/BoundedFCFSScheduler.java | 22 +-
.../query/scheduler/fcfs/FCFSQueryScheduler.java | 17 +-
.../resources/BinaryWorkloadResourceManager.java | 6 +-
.../resources/PolicyBasedResourceManager.java | 5 +-
.../query/scheduler/resources/ResourceManager.java | 41 +-
.../resources/UnboundedResourceManager.java | 5 +-
.../tokenbucket/TokenPriorityScheduler.java | 21 +-
.../query/selection/SelectionOperatorService.java | 16 +-
.../query/selection/SelectionOperatorUtils.java | 15 +-
.../pinot/core/query/utils/QueryIdUtils.java | 6 +-
.../apache/pinot/core/routing/TableRouteInfo.java | 6 +-
.../processing/framework/SegmentConfig.java | 17 +-
.../framework/SegmentProcessorFramework.java | 9 +-
.../genericrow/AdaptiveSizeBasedWriter.java | 84 +-
.../genericrow/GenericRowDeserializer.java | 29 +
.../genericrow/GenericRowSerializer.java | 12 +
.../segment/processing/mapper/SegmentMapper.java | 3 +-
.../core/transport/ChannelHandlerFactory.java | 13 +-
.../pinot/core/transport/DataTableHandler.java | 34 +-
.../core/transport/InstanceRequestHandler.java | 164 +-
.../apache/pinot/core/transport/QueryRouter.java | 25 +-
.../pinot/core/transport/ServerChannels.java | 39 +-
.../pinot/core/transport/grpc/GrpcQueryServer.java | 72 +-
.../main/java/org/apache/pinot/core/udf/Udf.java | 2 +-
.../apache/pinot/core/udf/UdfExampleBuilder.java | 2 +-
.../org/apache/pinot/core/util/GroupByUtils.java | 4 +-
.../PerQueryCPUMemAccountCancelTest.java | 189 --
.../accounting/PerQueryCPUMemAccountantTest.java | 268 ---
.../core/accounting/QueryMonitorConfigTest.java | 227 +--
.../accounting/ResourceManagerAccountingTest.java | 652 -------
.../core/accounting/TestResourceAccountant.java | 95 -
.../data/manager/BaseTableDataManagerTest.java | 2 +
.../RealtimeConsumptionRateManagerTest.java | 20 +-
.../ServerRateLimitConfigChangeListenerTest.java | 32 +-
.../metadata/DefaultMetadataExtractorTest.java | 210 +++
.../combine/CombineErrorOperatorsTest.java | 63 +-
.../operator/combine/CombineSlowOperatorsTest.java | 16 +-
.../combine/SelectionCombineOperatorTest.java | 95 +-
.../combine/SortedGroupByCombineOperatorsTest.java | 2 +-
.../operator/filter/AndFilterOperatorTest.java | 31 +
.../core/operator/filter/OrFilterOperatorTest.java | 30 +
.../DefaultAggregationExecutorTest.java | 3 +-
.../function/AggregationFunctionFactoryTest.java | 6 +
...est.java => SumIntAggregationFunctionTest.java} | 60 +-
.../executor/QueryExecutorExceptionsTest.java | 27 +-
.../core/query/executor/QueryExecutorTest.java | 48 +-
.../PostAggregationFunctionTest.java | 5 +
.../core/query/reduce/BrokerReduceServiceTest.java | 45 +-
.../query/reduce/StreamingReduceServiceTest.java | 6 +-
.../scheduler/MultiLevelPriorityQueueTest.java | 15 +-
.../query/scheduler/PrioritySchedulerTest.java | 22 +-
.../query/scheduler/QuerySchedulerFactoryTest.java | 31 +-
.../scheduler/resources/ResourceManagerTest.java | 6 +-
.../resources/UnboundedResourceManagerTest.java | 7 +-
.../core/query/selection/SelectionOrderByTest.java | 68 +-
.../impl/fakestream/FakeStreamMessageBatch.java | 7 +
...HybridTableRouteProviderCalculateRouteTest.java | 17 -
...ogicalTableRouteProviderCalculateRouteTest.java | 23 +-
.../processing/genericrow/GenericRowSerDeTest.java | 93 +
.../core/transport/InstanceRequestHandlerTest.java | 58 +-
.../pinot/core/transport/QueryRoutingTest.java | 31 +-
.../pinot/core/transport/ServerChannelsTest.java | 6 +-
.../org/apache/pinot/queries/BaseQueriesTest.java | 97 +-
.../pinot/queries/ExplainPlanQueriesTest.java | 29 +-
.../apache/pinot/queries/H3IndexQueriesTest.java | 32 +-
...nnerSegmentSelectionSingleValueQueriesTest.java | 205 +-
.../pinot/queries/NullQueriesFluentTest.java | 112 ++
.../queries/TextMatchTransformFunctionTest.java | 3 +-
.../pinot/queries/TextSearchQueriesTest.java | 65 +
.../TextSearchQueriesWithCombinedFilesTest.java | 58 +
pinot-dependency-verifier/pom.xml | 2 +-
pinot-distribution/pom.xml | 2 +-
pinot-integration-test-base/pom.xml | 2 +-
.../tests/BaseClusterIntegrationTest.java | 8 +
.../tests/ClusterIntegrationTestUtils.java | 24 +-
.../pinot/integration/tests/QueryGenerator.java | 52 +-
pinot-integration-tests/pom.xml | 2 +-
.../tests/BaseClusterIntegrationTestSet.java | 12 +-
.../tests/CancelQueryIntegrationTests.java | 21 -
.../tests/CommitTimeCompactionIntegrationTest.java | 1365 ++++++++++++++
.../ControllerPeriodicTasksIntegrationTest.java | 7 +-
.../tests/GroupByOptionsIntegrationTest.java | 2 +-
.../tests/HybridClusterIntegrationTest.java | 76 +-
.../MultiNodesOfflineClusterIntegrationTest.java | 26 +-
.../tests/MultiStageEngineIntegrationTest.java | 366 +++-
.../tests/NullHandlingIntegrationTest.java | 6 +-
.../tests/OOMProtectionEnabledIntegrationTest.java | 80 +-
...lineClusterCpuBasedServerQueryKillingTest.java} | 96 +-
.../tests/OfflineClusterIntegrationTest.java | 28 +-
...flineClusterMemBasedBrokerQueryKillingTest.java | 81 +-
...flineClusterMemBasedServerQueryKillingTest.java | 456 ++---
...rMemBasedServerResourceUsageAccountantTest.java | 130 +-
...OfflineClusterResourceUsageAccountantTest.java} | 136 +-
.../tests/OfflineGRPCServerIntegrationTest.java | 24 +-
...lineGRPCServerOOMAccountingIntegrationTest.java | 44 +-
.../tests/QueryThreadContextIntegrationTest.java | 26 +-
.../tests/QueryWorkloadIntegrationTest.java | 212 +++
.../tests/RetentionManagerIntegrationTest.java | 153 ++
...PartitionLLCRealtimeClusterIntegrationTest.java | 37 +
.../tests/SimpleMinionClusterIntegrationTest.java | 7 +-
.../UpsertCompactMergeTaskIntegrationTest.java | 732 +++++++
.../tests/UpsertTableIntegrationTest.java | 2 +-
.../tests/WindowResourceAccountingTest.java | 107 +-
.../pinot/integration/tests/custom/ArrayTest.java | 6 +-
pinot-minion/pom.xml | 2 +-
pinot-perf/pom.xml | 6 +-
.../org/apache/pinot/perf/BenchmarkEquiJoin.java | 315 ++++
.../apache/pinot/perf/BenchmarkOrderByQueries.java | 183 +-
.../BenchmarkPairwiseCombineOrderByGroupBy.java | 4 -
.../perf/BenchmarkThreadInterruptionCheck.java | 182 --
.../java/org/apache/pinot/perf/Distribution.java | 3 +-
...ava => SumIntAggregationFunctionBenchmark.java} | 17 +-
.../src/main/resources/log4j2-test.properties | 53 +
pinot-plugins/assembly-descriptor/pom.xml | 2 +-
.../pinot-batch-ingestion-common/pom.xml | 2 +-
.../pinot-batch-ingestion-hadoop/pom.xml | 2 +-
.../pinot-batch-ingestion-spark-2.4/pom.xml | 2 +-
.../pinot-batch-ingestion-spark-3/pom.xml | 2 +-
.../pinot-batch-ingestion-spark-base/pom.xml | 2 +-
.../pinot-batch-ingestion-standalone/pom.xml | 2 +-
pinot-plugins/pinot-batch-ingestion/pom.xml | 2 +-
.../pinot-environment/pinot-azure/pom.xml | 2 +-
pinot-plugins/pinot-environment/pom.xml | 2 +-
pinot-plugins/pinot-file-system/pinot-adls/pom.xml | 2 +-
pinot-plugins/pinot-file-system/pinot-gcs/pom.xml | 2 +-
.../apache/pinot/plugin/filesystem/GcsPinotFS.java | 10 +-
pinot-plugins/pinot-file-system/pinot-hdfs/pom.xml | 2 +-
pinot-plugins/pinot-file-system/pinot-s3/pom.xml | 2 +-
pinot-plugins/pinot-file-system/pom.xml | 2 +-
.../pinot-input-format/pinot-avro-base/pom.xml | 2 +-
.../pinot-input-format/pinot-avro/pom.xml | 2 +-
.../pinot-input-format/pinot-clp-log/pom.xml | 2 +-
.../pinot-confluent-avro/pom.xml | 2 +-
.../pinot-confluent-json/pom.xml | 2 +-
.../pinot-confluent-protobuf/pom.xml | 2 +-
pinot-plugins/pinot-input-format/pinot-csv/pom.xml | 2 +-
.../pinot-input-format/pinot-json/pom.xml | 2 +-
pinot-plugins/pinot-input-format/pinot-orc/pom.xml | 2 +-
.../pinot-input-format/pinot-parquet/pom.xml | 2 +-
.../pinot-input-format/pinot-protobuf/pom.xml | 2 +-
.../pinot-input-format/pinot-thrift/pom.xml | 2 +-
pinot-plugins/pinot-input-format/pom.xml | 2 +-
.../pinot-metrics/pinot-compound-metrics/pom.xml | 2 +-
.../pinot-metrics/pinot-dropwizard/pom.xml | 2 +-
pinot-plugins/pinot-metrics/pinot-yammer/pom.xml | 2 +-
pinot-plugins/pinot-metrics/pom.xml | 2 +-
.../pinot-minion-builtin-tasks/pom.xml | 2 +-
.../pinot/plugin/minion/tasks/MergeTaskUtils.java | 5 +
.../pinot/plugin/minion/tasks/MinionTaskUtils.java | 48 +
.../mergerollup/MergeRollupTaskGenerator.java | 4 +-
.../UpsertCompactionTaskExecutor.java | 5 +-
.../UpsertCompactionTaskGenerator.java | 35 +-
.../plugin/minion/tasks/MergeTaskUtilsTest.java | 5 +-
.../plugin/minion/tasks/MinionTaskUtilsTest.java | 158 +-
.../UpsertCompactionTaskGeneratorTest.java | 83 +
.../UpsertCompactMergeTaskExecutorTest.java | 174 +-
.../UpsertCompactMergeTaskGeneratorTest.java | 289 ++-
pinot-plugins/pinot-minion-tasks/pom.xml | 2 +-
.../pinot-segment-uploader-default/pom.xml | 2 +-
pinot-plugins/pinot-segment-uploader/pom.xml | 2 +-
.../pinot-segment-writer-file-based/pom.xml | 2 +-
pinot-plugins/pinot-segment-writer/pom.xml | 2 +-
.../pinot-stream-ingestion/pinot-kafka-2.0/pom.xml | 7 +-
.../KafkaPartitionLevelConnectionHandler.java | 3 +-
.../kafka20/KafkaPartitionLevelConsumer.java | 6 +-
.../kafka20/KafkaStreamMetadataProvider.java | 33 +
.../kafka20/KafkaPartitionLevelConsumerTest.java | 69 +
.../pinot-stream-ingestion/pinot-kafka-3.0/pom.xml | 7 +-
.../KafkaPartitionLevelConnectionHandler.java | 3 +-
.../kafka30/KafkaPartitionLevelConsumer.java | 6 +-
.../kafka30/KafkaStreamMetadataProvider.java | 33 +
.../kafka30/KafkaPartitionLevelConsumerTest.java | 69 +
.../pinot-kafka-base/pom.xml | 2 +-
.../plugin/stream/kafka/KafkaMessageBatch.java | 9 +-
.../pinot-stream-ingestion/pinot-kinesis/pom.xml | 2 +-
.../plugin/stream/kinesis/KinesisConsumer.java | 16 +-
.../plugin/stream/kinesis/KinesisMessageBatch.java | 9 +-
.../plugin/stream/kinesis/KinesisConsumerTest.java | 2 +
.../stream/kinesis/KinesisMessageBatchTest.java | 2 +-
.../kinesis/KinesisStreamMetadataProviderTest.java | 24 +-
.../pinot-stream-ingestion/pinot-pulsar/pom.xml | 2 +-
.../plugin/stream/pulsar/PulsarMessageBatch.java | 9 +-
.../pulsar/PulsarPartitionLevelConsumer.java | 7 +-
.../plugin/stream/pulsar/PulsarConsumerTest.java | 1 +
.../stream/pulsar/PulsarMessageBatchTest.java | 6 +-
pinot-plugins/pinot-stream-ingestion/pom.xml | 2 +-
.../pinot-timeseries-m3ql/pom.xml | 2 +-
pinot-plugins/pinot-timeseries-lang/pom.xml | 2 +-
pinot-plugins/pom.xml | 2 +-
pinot-query-planner/pom.xml | 2 +-
.../apache/pinot/query/MseWorkerThreadContext.java | 304 ---
.../org/apache/pinot/query/QueryEnvironment.java | 19 +-
.../query/context/PhysicalPlannerContext.java | 23 +-
.../pinot/query/function/InternalMseFunctions.java | 60 -
.../planner/partitioning/EmptyKeySelector.java | 4 +-
.../planner/partitioning/HashFunctionSelector.java | 89 +-
.../partitioning/MultiColumnKeySelector.java | 6 +-
.../partitioning/SingleColumnKeySelector.java | 6 +-
.../planner/physical/MailboxAssignmentVisitor.java | 9 +-
.../v2/opt/rules/LiteModeSortInsertRule.java | 23 +-
.../partitioning/HashFunctionSelectorTest.java | 54 +-
.../partitioning/KeySelectorHashFunctionTest.java | 7 +-
.../resources/queries/PhysicalOptimizerPlans.json | 29 +
pinot-query-runtime/pom.xml | 2 +-
.../pinot/query/mailbox/GrpcSendingMailbox.java | 4 +-
.../apache/pinot/query/mailbox/MailboxService.java | 24 +-
.../pinot/query/mailbox/ReceivingMailbox.java | 25 +-
.../query/mailbox/channel/GrpcMailboxServer.java | 30 +-
.../mailbox/channel/MailboxContentObserver.java | 17 +-
.../query/runtime/InStageStatsTreeBuilder.java | 74 +-
.../apache/pinot/query/runtime/QueryRunner.java | 68 +-
.../pinot/query/runtime/blocks/ErrorMseBlock.java | 22 +-
.../query/runtime/executor/OpChainScheduler.java | 86 -
.../runtime/executor/OpChainSchedulerService.java | 186 +-
.../query/runtime/operator/AggregateOperator.java | 25 +-
.../query/runtime/operator/AsofJoinOperator.java | 15 +-
.../query/runtime/operator/BaseJoinOperator.java | 20 +-
.../operator/BaseMailboxReceiveOperator.java | 20 +-
.../query/runtime/operator/ErrorOperator.java | 4 +-
.../query/runtime/operator/FilterOperator.java | 16 +-
.../query/runtime/operator/HashJoinOperator.java | 25 +-
.../pinot/query/runtime/operator/LeafOperator.java | 94 +-
.../runtime/operator/LiteralValueOperator.java | 16 +-
.../query/runtime/operator/LookupJoinOperator.java | 22 +-
.../runtime/operator/MailboxReceiveOperator.java | 2 +-
.../runtime/operator/MailboxSendOperator.java | 28 +-
.../query/runtime/operator/MultiStageOperator.java | 77 +-
.../runtime/operator/NonEquiJoinOperator.java | 9 +-
.../pinot/query/runtime/operator/OpChain.java | 9 -
.../pinot/query/runtime/operator/SortOperator.java | 18 +-
.../query/runtime/operator/TransformOperator.java | 16 +-
.../runtime/operator/WindowAggregateOperator.java | 25 +-
.../runtime/operator/exchange/BlockExchange.java | 2 +-
.../runtime/operator/groupby/GroupIdGenerator.java | 3 +
.../groupby/MultiKeysGroupIdGenerator.java | 11 +-
.../groupby/OneDoubleKeyGroupIdGenerator.java | 6 +-
.../groupby/OneFloatKeyGroupIdGenerator.java | 6 +-
.../groupby/OneIntKeyGroupIdGenerator.java | 6 +-
.../groupby/OneLongKeyGroupIdGenerator.java | 6 +-
.../groupby/OneObjectKeyGroupIdGenerator.java | 10 +-
.../operator/groupby/TwoKeysGroupIdGenerator.java | 10 +-
.../operator/set/RightRowSetBasedSetOperator.java | 4 +-
.../query/runtime/operator/set/SetOperator.java | 16 +-
.../query/runtime/operator/set/UnionOperator.java | 2 +-
.../utils/BlockingMultiStreamConsumer.java | 17 +-
.../runtime/operator/utils/BlockingStream.java | 47 -
.../window/value/LagValueWindowFunction.java | 3 +-
.../query/runtime/plan/MultiStageQueryStats.java | 4 -
.../runtime/plan/OpChainExecutionContext.java | 60 +-
.../plan/pipeline/PipelineBreakerExecutor.java | 43 +-
.../plan/pipeline/PipelineBreakerOperator.java | 49 +-
.../plan/server/ServerPlanRequestUtils.java | 24 +-
.../query/service/dispatch/DispatchClient.java | 25 +-
.../query/service/dispatch/QueryDispatcher.java | 66 +-
.../AsyncQueryTimeSeriesDispatchResponse.java | 57 -
.../pinot/query/service/server/QueryServer.java | 290 ++-
.../apache/pinot/query/QueryServerEnclosure.java | 25 +-
.../query/mailbox/GrpcSendingMailboxTest.java | 42 +-
.../pinot/query/mailbox/MailboxServiceTest.java | 23 +-
.../executor/OpChainSchedulerServiceTest.java | 164 +-
.../operator/BlockListMultiStageOperator.java | 2 +-
.../operator/EnrichedHashJoinOperatorTest.java | 1 -
.../query/runtime/operator/LeafOperatorTest.java | 20 +-
.../runtime/operator/MailboxSendOperatorTest.java | 6 +-
.../runtime/operator/MultiStageAccountingTest.java | 242 ---
.../MultistageResourceUsageAccountingTest.java | 241 ---
.../pinot/query/runtime/operator/OpChainTest.java | 2 +-
.../query/runtime/operator/OperatorTestUtil.java | 9 +-
.../plan/pipeline/PipelineBreakerExecutorTest.java | 69 +-
.../runtime/queries/DefaultAccountantTest.java | 50 -
.../queries/PerQueryCPUMemAccountantTest.java | 168 --
.../runtime/queries/QueryRunnerAccountingTest.java | 87 -
.../query/runtime/queries/QueryRunnerTest.java | 20 +-
.../query/runtime/queries/QueryRunnerTestBase.java | 70 +-
.../runtime/queries/ResourceBasedQueriesTest.java | 20 +-
.../service/dispatch/QueryDispatcherTest.java | 57 +-
.../query/service/server/QueryServerTest.java | 6 +-
.../testutils/MockInstanceDataManagerFactory.java | 4 -
.../test/resources/queries/WindowFunctions.json | 12 +
pinot-segment-local/pom.xml | 2 +-
.../local/data/manager/TableDataManager.java | 8 +
.../immutable/ImmutableSegmentImpl.java | 2 +-
.../immutable/ImmutableSegmentLoader.java | 3 +-
.../indexsegment/mutable/MutableSegmentImpl.java | 11 +-
.../converter/RealtimeSegmentConverter.java | 133 +-
.../CompactedDictEncodedColumnStatistics.java | 148 ++
.../CompactedRawIndexDictColumnStatistics.java | 152 ++
.../RealtimeSegmentSegmentCreationDataSource.java | 19 +-
.../stats/RealtimeSegmentStatsContainer.java | 111 +-
.../realtime/impl/json/MutableJsonIndexImpl.java | 5 +-
.../creator/impl/SegmentColumnarIndexCreator.java | 99 +-
.../impl/SegmentIndexCreationDriverImpl.java | 26 +-
.../stats/MapColumnPreIndexStatsCollector.java | 2 +-
.../text/LuceneCombinedTextIndexConstants.java | 105 ++
.../creator/impl/text/LuceneTextIndexCombined.java | 260 +++
.../creator/impl/text/LuceneTextIndexCreator.java | 68 +-
.../converter/SegmentV1V2ToV3FormatConverter.java | 13 +-
.../segment/index/loader/ForwardIndexHandler.java | 146 +-
...IndexAndDictionaryBasedForwardIndexCreator.java | 33 +-
.../loader/invertedindex/JsonIndexHandler.java | 4 +-
.../loader/invertedindex/TextIndexHandler.java | 139 +-
.../forward/FixedBitSVForwardIndexReaderV2.java | 17 +
.../readers/json/ImmutableJsonIndexReader.java | 5 +-
.../readers/text/LuceneTextIndexBufferReader.java | 245 +++
.../index/readers/text/LuceneTextIndexHeader.java | 103 +
.../index/readers/text/LuceneTextIndexReader.java | 177 +-
.../index/readers/text/PinotBufferIndexInput.java | 88 +
.../readers/text/PinotBufferLuceneDirectory.java | 139 ++
.../segment/index/text/TextIndexConfigBuilder.java | 16 +
.../local/segment/index/text/TextIndexType.java | 22 +-
.../text/lucene/parsers/MatchQueryParser.java | 335 ++++
.../segment/index/vector/VectorIndexType.java | 2 +-
.../readers/CompactedPinotSegmentRecordReader.java | 136 +-
.../segment/store/SingleFileIndexDirectory.java | 5 +-
.../local/segment/store/TextIndexUtils.java | 2 +-
.../PartitionIdVirtualColumnProvider.java | 237 +++
.../virtualcolumn/VirtualColumnContext.java | 14 +
.../VirtualColumnProviderFactory.java | 5 +
.../upsert/BasePartitionUpsertMetadataManager.java | 28 +-
.../segment/local/utils/LuceneTextIndexUtils.java | 11 +
.../segment/local/utils/TableConfigUtils.java | 96 +-
.../mutable/MutableSegmentImplRawMVTest.java | 8 +
.../CompactedDictEncodedColumnStatisticsTest.java | 223 +++
.../CompactedRawIndexDictColumnStatisticsTest.java | 152 ++
.../stats/MapColumnPreIndexStatsCollectorTest.java | 158 ++
.../text/LuceneTextIndexBufferIntegrationTest.java | 210 +++
.../local/segment/index/ColumnMetadataTest.java | 194 +-
.../text/LuceneTextIndexConfigReloadTest.java | 289 +++
.../parsers/MinimumShouldMatchQueryParserTest.java | 266 +++
.../segment/local/utils/TableConfigUtilsTest.java | 58 +-
pinot-segment-spi/pom.xml | 2 +-
.../pinot/segment/spi/AggregationFunctionType.java | 1 +
.../org/apache/pinot/segment/spi/V1Constants.java | 1 +
.../pinot/segment/spi/creator/SegmentCreator.java | 17 +
.../spi/creator/SegmentGeneratorConfig.java | 23 +-
.../pinot/segment/spi/index/TextIndexConfig.java | 65 +-
.../segment/spi/memory/CompoundDataBuffer.java | 17 +
pinot-server/pom.xml | 2 +-
.../pinot/server/api/resources/DebugResource.java | 159 +-
.../pinot/server/api/resources/TablesResource.java | 69 +
.../pinot/server/starter/ServerInstance.java | 65 +-
.../server/starter/helix/BaseServerStarter.java | 98 +-
.../helix/HelixInstanceDataManagerConfig.java | 8 +
.../helix/QueryWorkloadMessageHandlerFactory.java | 87 -
.../helix/SegmentMessageHandlerFactory.java | 52 +
.../SegmentOnlineOfflineStateModelFactory.java | 33 +-
.../pinot/server/worker/WorkerQueryServer.java | 49 +-
.../apache/pinot/server/api/BaseResourceTest.java | 7 +-
.../pinot/server/api/TablesResourceTest.java | 178 +-
.../test/resources/data/test_data_with_delete.avro | Bin 14086590 -> 15615184
bytes
pinot-spi/pom.xml | 7 +-
.../pinot/spi/accounting/QueryResourceTracker.java | 22 +-
.../pinot/spi/accounting/ThreadAccountant.java | 69 +
.../spi/accounting/ThreadAccountantFactory.java | 2 +-
.../spi/accounting/ThreadAccountantUtils.java | 93 +
.../spi/accounting/ThreadExecutionContext.java | 52 -
.../spi/accounting/ThreadResourceTracker.java | 34 +-
.../accounting/ThreadResourceUsageAccountant.java | 133 --
.../accounting/ThreadResourceUsageProvider.java | 15 +
.../apache/pinot/spi/accounting/TrackingScope.java | 4 +-
.../spi/accounting/WorkloadBudgetManager.java | 85 +-
.../config/instance/InstanceDataManagerConfig.java | 2 +
.../apache/pinot/spi/config/table/PauseState.java | 19 +-
.../SegmentsValidationAndRetentionConfig.java | 18 +
.../pinot/spi/config/table/UpsertConfig.java | 11 +
.../table/ingestion/StreamIngestionConfig.java | 12 +
.../spi/config/workload/PropagationEntity.java | 100 +
.../workload/PropagationEntityOverrides.java | 76 +
.../spi/config/workload/PropagationScheme.java | 66 +-
.../spi/config/workload/QueryWorkloadConfig.java | 28 +-
.../apache/pinot/spi/data/DateTimeFieldSpec.java | 2 +-
.../spi/exception/QueryCancelledException.java | 1 +
.../apache/pinot/spi/exception/QueryErrorCode.java | 1 +
...ictException.java => TerminationException.java} | 8 +-
.../spi/executor/DecoratorExecutorService.java | 27 +-
.../pinot/spi/executor/ExecutorServiceUtils.java | 2 +-
.../ThrottleOnCriticalHeapUsageExecutor.java | 122 +-
.../pinot/spi/query/QueryExecutionContext.java | 187 ++
.../apache/pinot/spi/query/QueryThreadContext.java | 823 +++-----
.../org/apache/pinot/spi/stream/MessageBatch.java | 5 +
.../spi/stream/PartitionGroupMetadataFetcher.java | 10 +-
.../org/apache/pinot/spi/stream/StreamConfig.java | 15 +-
.../pinot/spi/stream/StreamConfigProperties.java | 5 +
.../pinot/spi/stream/StreamMetadataProvider.java | 10 +
.../apache/pinot/spi/trace/LoggerConstants.java | 21 +-
.../java/org/apache/pinot/spi/trace/Tracing.java | 304 +--
.../apache/pinot/spi/utils/CommonConstants.java | 120 +-
.../utils/builder/ControllerRequestURLBuilder.java | 49 +-
.../spi/accounting/WorkloadBudgetManagerTest.java | 8 +-
.../provider/PinotClusterConfigProviderTest.java | 213 +++
.../data/readers/AbstractRecordExtractorTest.java | 15 +-
.../spi/data/readers/AbstractRecordReaderTest.java | 23 +-
.../ThrottleOnCriticalHeapUsageExecutorTest.java | 459 -----
.../stream/PartitionGroupMetadataFetcherTest.java | 216 +++
pinot-timeseries/pinot-timeseries-planner/pom.xml | 2 +-
pinot-timeseries/pinot-timeseries-spi/pom.xml | 2 +-
pinot-timeseries/pom.xml | 2 +-
pinot-tools/pom.xml | 2 +-
.../pinot/tools/ColocatedJoinQuickStart.java | 3 +
.../pinot/tools/MultistageEngineQuickStart.java | 11 +-
.../apache/pinot/tools/SortedColumnQuickstart.java | 187 ++
.../converter/DictionaryToRawIndexConverter.java | 16 +-
.../{tpch/orders => sorted}/ingestionJobSpec.yaml | 31 +-
pinot-udf-test/pom.xml | 2 +-
.../pinot/udf/test/PinotFunctionEnvGenerator.java | 2 +-
.../org/apache/pinot/udf/test/UdfTestResult.java | 2 +-
pom.xml | 106 +-
731 files changed, 34789 insertions(+), 13431 deletions(-)
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/routing/BrokerRoutingManagerConcurrencyTest.java
create mode 100644
pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/BaseResultSetGroup.java
create mode 100644
pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/CursorAwareBrokerResponse.java
create mode 100644
pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/CursorCapable.java
create mode 100644
pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/CursorResultSetGroup.java
create mode 100644
pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/ResultCursor.java
create mode 100644
pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/ResultCursorImpl.java
create mode 100644
pinot-clients/pinot-java-client/src/test/java/org/apache/pinot/client/CursorAwareBrokerResponseTest.java
create mode 100644
pinot-clients/pinot-java-client/src/test/java/org/apache/pinot/client/CursorResultSetGroupTest.java
create mode 100644
pinot-clients/pinot-java-client/src/test/java/org/apache/pinot/client/ResultCursorTest.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/audit/AuditIdentityResolver.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/audit/AuditMetrics.java
copy
pinot-query-planner/src/main/java/org/apache/pinot/query/routing/MailboxInfo.java
=>
pinot-common/src/main/java/org/apache/pinot/common/audit/AuditResponseContext.java
(54%)
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/audit/AuditServiceBinder.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/audit/AuditUrlPathFilter.java
rename {pinot-server/src/main/java/org/apache/pinot/server/starter/helix =>
pinot-common/src/main/java/org/apache/pinot/common/config}/DefaultClusterConfigChangeHandler.java
(75%)
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/config/provider/StaticTableCache.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/response/mapper/TimeSeriesResponseMapper.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/audit/AuditIdentityResolverTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/audit/AuditLogFilterTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/audit/AuditMetricsTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/audit/AuditUrlPathFilterTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/function/scalar/NgramFunctionTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/response/mapper/TimeSeriesResponseMapperTest.java
create mode 100644 pinot-connectors/pinot-spark-3-connector/examples/README.md
copy
pinot-common/src/main/java/org/apache/pinot/sql/parsers/dml/DataManipulationStatementParser.java
=>
pinot-connectors/pinot-spark-3-connector/examples/read_pinot_from_proxy_with_auth_token.scala
(55%)
create mode 100644
pinot-connectors/pinot-spark-common/src/main/scala/org/apache/pinot/connector/spark/common/AuthUtils.scala
create mode 100644
pinot-connectors/pinot-spark-common/src/main/scala/org/apache/pinot/connector/spark/common/GrpcUtils.scala
copy
pinot-spi/src/main/java/org/apache/pinot/spi/accounting/QueryResourceTracker.java
=>
pinot-connectors/pinot-spark-common/src/main/scala/org/apache/pinot/connector/spark/common/NetUtils.scala
(55%)
create mode 100644
pinot-connectors/pinot-spark-common/src/test/scala/org/apache/pinot/connector/spark/common/GrpcUtilsTest.scala
create mode 100644
pinot-connectors/pinot-spark-common/src/test/scala/org/apache/pinot/connector/spark/common/HttpUtilsTest.scala
create mode 100644
pinot-connectors/pinot-spark-common/src/test/scala/org/apache/pinot/connector/spark/common/PinotClusterClientTest.scala
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/periodictask/RealtimeOffsetAutoResetHandler.java
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/periodictask/RealtimeOffsetAutoResetKafkaHandler.java
delete mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/tenant/DefaultTenantRebalancer.java
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/tenant/TenantRebalanceChecker.java
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/tenant/TenantRebalanceContext.java
rename
pinot-controller/src/main/java/org/apache/pinot/controller/validation/{OfflineSegmentIntervalChecker.java
=> OfflineSegmentValidationManager.java} (66%)
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/validation/RealtimeOffsetAutoResetManager.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/PinotQueryResourceStaticValidationTest.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/rebalance/tenant/TenantRebalanceCheckerTest.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/validation/RealtimeOffsetAutoResetManagerTest.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/workload/PropagationSchemeTest.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/accounting/AggregateByQueryIdAccountantFactoryForTest.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/accounting/CPUMemThreadLevelAccountingObjects.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/accounting/PerQueryCPUMemAccountantFactoryForTest.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/accounting/QueryAggregator.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/accounting/QueryResourceAggregator.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/accounting/QueryResourceTrackerImpl.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/accounting/ThreadResourceTrackerImpl.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/accounting/WorkloadAggregator.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/accounting/WorkloadResourceAggregator.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/operator/BaseDocIdSetOperator.java
copy pinot-core/src/main/java/org/apache/pinot/core/{plan/PlanNode.java =>
operator/DocIdOrderedOperator.java} (57%)
copy
pinot-core/src/main/java/org/apache/pinot/core/operator/{DocIdSetOperator.java
=> ReverseDocIdSetOperator.java} (60%)
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/operator/combine/CombineOperatorUtils.java
copy
pinot-core/src/main/java/org/apache/pinot/core/operator/docidsets/{EmptyDocIdSet.java
=> ShortCircuitingDocIdSet.java} (63%)
rename
pinot-core/src/main/java/org/apache/pinot/core/operator/query/{SelectionPartiallyOrderedByAscOperator.java
=> SelectionPartiallyOrderedByLinearOperator.java} (66%)
copy
pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/{IntAggregateResultHolder.java
=> LongAggregateResultHolder.java} (70%)
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/SumIntAggregationFunction.java
copy
pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/groupby/{IntGroupByResultHolder.java
=> LongGroupByResultHolder.java} (83%)
delete mode 100644
pinot-core/src/test/java/org/apache/pinot/core/accounting/PerQueryCPUMemAccountCancelTest.java
delete mode 100644
pinot-core/src/test/java/org/apache/pinot/core/accounting/PerQueryCPUMemAccountantTest.java
delete mode 100644
pinot-core/src/test/java/org/apache/pinot/core/accounting/ResourceManagerAccountingTest.java
delete mode 100644
pinot-core/src/test/java/org/apache/pinot/core/accounting/TestResourceAccountant.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/metadata/DefaultMetadataExtractorTest.java
copy
pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/{SumAggregationFunctionTest.java
=> SumIntAggregationFunctionTest.java} (73%)
create mode 100644
pinot-core/src/test/java/org/apache/pinot/queries/NullQueriesFluentTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/queries/TextSearchQueriesWithCombinedFilesTest.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/CommitTimeCompactionIntegrationTest.java
rename
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/{OfflineClusterServerCPUTimeQueryKillingTest.java
=> OfflineClusterCpuBasedServerQueryKillingTest.java} (67%)
rename
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/{OfflineClusterCPUTimeResourceUsageAccountantTest.java
=> OfflineClusterResourceUsageAccountantTest.java} (63%)
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/QueryWorkloadIntegrationTest.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/RetentionManagerIntegrationTest.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/UpsertCompactMergeTaskIntegrationTest.java
create mode 100644
pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkEquiJoin.java
delete mode 100644
pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkThreadInterruptionCheck.java
copy
pinot-perf/src/main/java/org/apache/pinot/perf/aggregation/{BenchmarkSumQuery.java
=> SumIntAggregationFunctionBenchmark.java} (85%)
create mode 100644 pinot-perf/src/main/resources/log4j2-test.properties
delete mode 100644
pinot-query-planner/src/main/java/org/apache/pinot/query/MseWorkerThreadContext.java
delete mode 100644
pinot-query-planner/src/main/java/org/apache/pinot/query/function/InternalMseFunctions.java
delete mode 100644
pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/executor/OpChainScheduler.java
delete mode 100644
pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/utils/BlockingStream.java
delete mode 100644
pinot-query-runtime/src/main/java/org/apache/pinot/query/service/dispatch/timeseries/AsyncQueryTimeSeriesDispatchResponse.java
delete mode 100644
pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/MultiStageAccountingTest.java
delete mode 100644
pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/MultistageResourceUsageAccountingTest.java
delete mode 100644
pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/queries/DefaultAccountantTest.java
delete mode 100644
pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/queries/PerQueryCPUMemAccountantTest.java
delete mode 100644
pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/queries/QueryRunnerAccountingTest.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedDictEncodedColumnStatistics.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedRawIndexDictColumnStatistics.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneCombinedTextIndexConstants.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCombined.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexBufferReader.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexHeader.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/PinotBufferIndexInput.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/PinotBufferLuceneDirectory.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/text/lucene/parsers/MatchQueryParser.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/virtualcolumn/PartitionIdVirtualColumnProvider.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedDictEncodedColumnStatisticsTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedRawIndexDictColumnStatisticsTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/stats/MapColumnPreIndexStatsCollectorTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexBufferIntegrationTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/text/LuceneTextIndexConfigReloadTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/text/lucene/parsers/MinimumShouldMatchQueryParserTest.java
delete mode 100644
pinot-server/src/main/java/org/apache/pinot/server/starter/helix/QueryWorkloadMessageHandlerFactory.java
copy
pinot-plugins/pinot-input-format/pinot-parquet/src/test/resources/test-data/airlines_parquet/4345e5eef217aa1b-c8f16177f35fd983_1150363067_data.1.parq
=> pinot-server/src/test/resources/data/test_data_with_delete.avro (58%)
create mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/accounting/ThreadAccountant.java
create mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/accounting/ThreadAccountantUtils.java
delete mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/accounting/ThreadExecutionContext.java
delete mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/accounting/ThreadResourceUsageAccountant.java
create mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/config/workload/PropagationEntity.java
create mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/config/workload/PropagationEntityOverrides.java
copy
pinot-spi/src/main/java/org/apache/pinot/spi/exception/{DatabaseConflictException.java
=> TerminationException.java} (76%)
create mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/query/QueryExecutionContext.java
create mode 100644
pinot-spi/src/test/java/org/apache/pinot/spi/config/provider/PinotClusterConfigProviderTest.java
delete mode 100644
pinot-spi/src/test/java/org/apache/pinot/spi/executor/ThrottleOnCriticalHeapUsageExecutorTest.java
create mode 100644
pinot-spi/src/test/java/org/apache/pinot/spi/stream/PartitionGroupMetadataFetcherTest.java
create mode 100644
pinot-tools/src/main/java/org/apache/pinot/tools/SortedColumnQuickstart.java
copy pinot-tools/src/main/resources/examples/batch/{tpch/orders =>
sorted}/ingestionJobSpec.yaml (80%)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
