This is an automated email from the ASF dual-hosted git repository.
jackie pushed a change to branch
maintain-pool-selection-for-minimizeDataMovement
in repository https://gitbox.apache.org/repos/asf/pinot.git
omit 08efc8c2f1 Add logic to consider the case when instances are moved
across pools
omit 83911abc93 Address PR comments
omit d9ae2e9432 Update
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/assignment/instance/InstancePartitionSelector.java
omit 95ed621750 Enhance the minimizeDataMovement to keep the existing pool
assignment
add 6638d4edcb [bugfix]Add logs to track sequence of events for table
creation (#11946)
add d177866527 [bugfix][multistage] explicit warning flags set on each
stage stats (#11936)
add 8f69299aef upgrade lucene to 9.8.0
add 00cf7a8295 Try lucene-backward-codecs package
add 06b58d4dcd adding lucene 80 compatible test
add 45f186903e Support constant filter in QueryContext, and make server
able to handle it (#11956)
add 8961e5460f added check for illegal character '/' in taskName
add 9092244e0b Added invalid argument related tests for AdhocTaskConfig
and rephrased an error message.
add b5e982333d [test][multistage] enhance query plan test (#11966)
add e3f2777e36 Updated for commons-configuration2 in PinotConfiguartion
(#11916)
add 49faa9424a register the new segment when it's fully initalized by
partitionUpsertMetadataManager (#11964)
add fc2395a29c Added invalid argument related tests for AdhocTaskConfig
and rephrased an error message.
add 3e2aa44535 Allow user specify local temp directory for quickstart
add 972b555cc5 fixing dateTimeConvert
add aac489864b Ability to fetch Segment metadata in batches
add 330938fe4c Make groupBy trim size configurable at Broker (#11958)
add 40ede67d76 fixing the wrong import for Preconditions (#11979)
add b92e4da2a0 Bump io.netty:netty-bom from 4.1.94.Final to 4.1.100.Final
(#11972)
add a40ed04bac Bump org.scalatest:scalatest-maven-plugin from 1.0 to 2.2.0
(#11973)
add a986dd1536 fix for #11974, exclude `module-info.class` (#11975)
add 73d82ec666 [multistage][bugfix] fix operator eos pull (#11970)
add 10afb1a89d [enhance]: Add cache for Bcrypt password
add bb52625f8a instrument building datatable (#11942)
add 2beb9a4938 Bump commons-net:commons-net from 3.1 to 3.10.0 (#11982)
add b38bd9fb34 [hotfix] fix filter merge after rule (#11989)
add 610c36afb6 [test][multistage] adding filter pushdown tests (#11994)
add f30fdee7ba [multistage][feature] leaf planning with multi-semi join
support (#11937)
add 0dd2522afd add an option to skip controller cert validation in
AddTableCommand (#11967)
add b2b802a427 fix for #11996 which corrects property name typo (#11997)
add e26404ca54 Bump com.jayway.jsonpath:json-path from 2.7.0 to 2.8.0
(#11987)
add b685a2e318 Bump simpleclient_common.version from 0.8.1 to 0.16.0
(#11986)
add 53883aedb5 Adding byte functions for UUIDs (#11988)
add 5f22d16086 Fix fromULL scalar function (#11995)
add 7217bb9d16 [multistage][bugfix] leaf limit refactor issue (#12001)
add 62abf1fb22 Fix the test failures caused by instance drop failure
(#12002)
add 6d1b0609c5 Fix the misuse of star-tree when all predicates are always
false under OR (#12003)
add 88fe079ecd Fix the race condition of concurrent modification to
segment data managers (#12004)
add 45f7d44dc9 Bump com.mercateo:test-clock from 1.0.2 to 1.0.4 (#12005)
add 5f601bcc01 Bump circe.version from 0.14.2 to 0.14.6 (#12006)
add 1ab9e62b6d Add query option override for Broker MinGroupTrimSize
(#11984)
add 71e9c2cd25 Replace timer with scheduled executor service in
IngestionDelayTracker (#11849)
add 11151e6df6 Bump net.nicoulaj.maven.plugins:checksum-maven-plugin from
1.8 to 1.11 (#12008)
add 41d1a12160 Bump com.google.code.gson:gson from 2.2.4 to 2.10.1 (#12009)
add 09da0eac74 [multistage][feature] support RelDistribution trait
planning (#11976)
add 1d8f2b6366 canonicalize SqlKind.OTHERS as well as
SqlKind.OTHER_FUNCTIONS (#12025)
add 8c604f8354 [Flaky-test] Fix PinotTenantRestletResourceTest (#12019)
add 8d93a169a1 Bump grpc.version from 1.53.0 to 1.59.0 (#12023)
add 968c0a3d2a Bump dropwizard-metrics.version from 4.2.2 to 4.2.22
(#12022)
add 0cb43eb499 Bump org.jacoco:jacoco-maven-plugin from 0.8.9 to 0.8.11
(#12024)
add c1fdb66ae8 Add upsert config - outOfOrderRecordColumn to track
out-of-order events (#11877)
add 244c947fdd Fix flaky PinotTenantRestletResourceTest (#12026)
add 6aecd41641 Add a check to enable size based threshold for realtime
tables (#12016)
add f7f82608e0 Adds support for leveraging StarTree index in conjunction
with filtered aggregations (#11886)
add 32db500160 Fix derived column from MV column (#12028)
add 8e84fc3fc5 Optimize segment commit to not read partition group
metadata (#11943)
add 54e828ad14 Bump com.google.code.findbugs:jsr305 from 3.0.0 to 3.0.2
(#12031)
add f435f4c7e4 Bump org.codehaus.mojo:appassembler-maven-plugin from 1.10
to 2.1.0 (#12030)
add 61556af200 fix by checking the index first (#12029)
add 35748a013b Explicit null handling (#11960)
add 2c88c4f74d fix flakyness by replacing HashSet and HashMap with
LinkedHashSet and LinkedHashMap (#11941)
add 31c5aaec0a Bump org.reactivestreams:reactive-streams from 1.0.3 to
1.0.4 (#12033)
add e4d3a8dde8 Bump io.grpc:grpc-context from 1.14.0 to 1.59.0 (#12034)
add 22403e5675 [multistage][bugfix] fix multi-semi-join issues (#12038)
add b3af476b83 Prevent inverted index on a non dictionary column in table
config (#12043)
add 95d4950dab Added new SpecialValueTransformer and tests.
add 5c8f43ead1 fix typo in comment.
add 2c05f8dca9 addressed comments and added test for ensuring order of
transformers.
add 06e91c13a1 added code to remove NaN from multivalued columns and
modified order of transformers.
add 05be5cfd9a modifed test to ensure order of transformers.
add f79b618141 fixed note in SpecialValueTransformer.
add 97f8f5f6cb Disable rebase/merge merge buttons (#12051)
add a37ced6ec9 Fix rebalance on upsert table (#12054)
add bfd7dce57e Fix the memory leak issue on `CommonsConfigurationUtils`
(#12056)
add 6fe2d25c21 Fix incorrect handling of consumer creation errors (#12045)
add d654cc9db2 Add a new MV forward index to only store unique MV values
(#11993)
add 3bd74b36e4 Allow optional segments that can be skipped by servers
without failing the query (#11978)
add 37270f749c [multistage][cleanup] remove some unused rules (#12052)
add c57117a750 [multistage] don't prune project after agg during
relBuilder (#12058)
add b9ed378355 [multistage][refactor] clean up planner (#12070)
add 88fcd0e4ee Fix bug with silently ignoring force commit call failures
(#12044)
add cc7e7a8b86 Introduce low disk mode to table rebalance (#12072)
add d7c76b9fbb no need for segment locks during segment preloading (#12077)
add fe072c63bd Remove what pinot doesn't do (#12075)
add b131f89461 make deep store upload retry async with configurable
parallelism (#12017)
add 69fbeeffe6 Enabling SegmentGenerationAndPushTask to push segment to
realtime table (#12084)
add 9497f06cba Fix default brokerUpdateFrequencyInMillis for connector
(#12093)
add 6d19f4a3b4 Bump up the parquet version to 1.13.1 (#12076)
add 12beff75df Bump xml-apis:xml-apis from 1.4.01 to 2.0.2 (#12082)
add d7e2e7834c Fixed log4j location in values.yaml #12010 (#12083)
add 9d779b4757 Correct path of default log4j configures (#12069)
add 96c2f3dd9d fix by repalcing HashSet with TreeSet (#12047)
add b25f7cf6c1 Adds support for CTRL key as a modifier for Query shortcuts
(#12087)
add 67cdb27721 Configurable Lucene analyzer (#12027)
add 559623054b Bump com.azure:azure-identity from 1.8.1 to 1.11.1 (#12095)
add 49d0ff01a4 Add support for murmur3 as a partition function (#12049)
add a9e319964f Theta Sketch Aggregation Enhancements (#12042)
add 9013e724a7 Fix compilation issue for lucene analyzer class change
(#12101)
add 49804a4f56 Add support for retention on deleted keys of upsert tables
(#12037)
add 3474db4abb [test] fix multi-stage explain error test (#12098)
add 48dd67e761 Bump net.java.dev.javacc:javacc from 7.0.10 to 7.0.13
(#12103)
add 3811240629 need to synchronize replacing upsert segment (#12105)
add 47cfcb382c Support Vector index and HNSW as the first implementation
(#11977)
add e62db612c9 Metrics for Table Disabled and Consumption Paused (#12000)
add 45d9a4b8cf UI: add lowDiskMode option in rebalance operation (#12112)
add e207844038 [multistage]partition assignment refactor (#12079)
add 20e7fdc1d8 Bump org.apache.maven.plugins:maven-assembly-plugin from
3.1.1 to 3.6.0 (#12109)
add f5f878e1ba Fix helm chart server probe endpoint backward incompatible
(#12114)
add d7dc4c5eff Bump org.apache.avro:avro from 1.10.2 to 1.11.3 (#12116)
add 56ecafc17e Add singleton registry for all metrics (#12119)
add 27cc8a796f Reduce logging volume to log the conversions per record
instead of per conversion for SpecialValueTransformer (#12121)
add ed8a251f07 Bump com.google.guava:guava from 32.0.1-jre to 32.1.3-jre
(#12124)
add a4e5fa3326 Bump com.github.luben:zstd-jni from 1.5.5-6 to 1.5.5-11
(#12125)
add d68a5a8dac PR to address Issue #12127 - Refactor foreign logger
initialization in multiple classes (#12128)
add f4b5de6b8a [hotfix] remove non-exist vector transform (#12140)
add 92e26a5847 Introduce UpsertContext to simplify the upsert metadata
manager constructor (#12120)
add d38e15d1fa fixes #12136 (#12137)
add 05afb65ec7 fix multiple instances where the number of placeholders did
not match the number of arguments to logging methods (#12134)
add a47b9cff16 Bump log4j.version from 2.20.0 to 2.22.0 (#12143)
add f3b94f60e4 Fix for #12133 - add context and proper logging framework
to OsCheck output (#12135)
add 280d368a71 avoid unnecessary transformer (#12138)
add 5b623855bc Bump org.apache.commons:commons-collections4 from 4.1 to
4.4 (#12149)
add f22de0a4be Improve resource management with try-with-resource blocks.
Fix for #12129 (#12130)
add 46ae9e97ff refine warning msg when preloading segments not exist on
server (#12153)
add 7514fc5978 Bump org.apache.zookeeper:zookeeper from 3.6.3 to 3.7.2
(#12152)
add b44b3aeb8f Changes for migration to commons-configuration2 (#11985)
add 618e049dde Bump org.apache.datasketches:datasketches-java from 4.1.0
to 5.0.0 (#12161)
add 979c9125ed Bump com.google.api.grpc:proto-google-common-protos (#12159)
add 0a124ee3be Bump org.glassfish.tyrus.bundles:tyrus-standalone-client
(#12162)
add aa834f4ccc Remove parameters from ThetaSketchAggregation function
(#12147)
add fd45c1bbd2 [feature] add support for StreamNative OAuth2
authentication for pulsar. (#12068)
add 333e17c765 Update LICENSE-binary for commons-configuration2 upgrade.
(#12165)
add ccc5f34507 Bump org.apache.httpcomponents:httpclient from 4.5.13 to
4.5.14 (#12172)
add 3d3012fd20 Bump com.gradle:common-custom-user-data-maven-extension
(#12171)
add c085f0497d Bump org.apache.yetus:audience-annotations from 0.13.0 to
0.15.0 (#12170)
add f0b78b44e3 Bug fix: reset primary key count to 0 when table is deleted
(#12169)
add f36cc10f4f Support initializing broker tags from config (#12175)
add 488b336ab4 Bump dropwizard-metrics.version from 4.2.22 to 4.2.23
(#12178)
add fade75bf4e Fix rebalancer converge check to ensure EV is converged
before reporting success (#12182)
add c706fd04e5 Fix partition handling by always using string values
(#12115)
add 85b5779cff Support array literal functions (#12118)
add 85d3850227 fix regex gen with escape issue on single quote (#12181)
add 24f8433d5f Bump com.google.auto.service:auto-service from 1.0.1 to
1.1.1 (#12183)
add 9d939a007e Add partition level Force Commit (#12088)
add 9b6d0b7cb8 Bump org.freemarker:freemarker from 2.3.30 to 2.3.32
(#12192)
add 50153796c6 Bump com.azure:azure-core from 1.37.0 to 1.45.1 (#12193)
add 9cfce82385 Proper computation of realtime
"segment.flush.threshold.size" in case of force-commit (#12188)
add 95ec64e50e Bump io.grpc:grpc-context from 1.59.0 to 1.60.1 (#12198)
add 344eba125e Bump org.apache.spark:spark-launcher_2.12 from 3.2.1 to
3.5.0 (#12199)
add a20031528d Data generator reorganisation (#12122)
add 50912eb0c4 Create DateTimeGenerator and add it to data generator
(#12206)
add 5b4e19aa4e fix: Pulsar OAuth2 Authentication Factory (#12195)
add d4d910cec1 FileWriter bug fixes (#12208)
add a0a9b6bca6 Specify Multi-Release property in manifest (#12131)
add f9df57a9e3 Add column name to JSONParsing exception message during
index build (#12151)
add add2236a42 [hotfix] fix table name not escaped during routing (#12212)
add d1cc17c579 [multistage][hotfix] use UTF-8 as default CharSet, this is
also true for v1 engine (#12213)
add 8f5fa804cd Add a 'lastUsed' option for resumeConsumption.consumeFrom
(#12200)
add 298e8d9191 Fix connection pool error by creating new default
credential (#12221)
add 436968e080 make all /size users render async (#12210)
add 07bcab80f1 [test] add back quickstart streaming (#12231)
add 388d394c7f Fix upsert tenant tag override check (#12233)
add 23c1e5f3a5 [bugfix] fix literal query return multiple duplicate
results (#12240)
add d1817efff7 [multistage][bugfix] improve sort copy rule (#12237)
add 7cb5973185 fix a deadlock due to getting segmentlock before snapshot
lock when replacing segment (#12241)
add 9947bc5916 refine how to take validDocIds snapshot (#12232)
add d05e3bd05c [bugfix] Handle NPE in controller SQL Resource (#12211)
add 8c86ad4652 fix a bug that would take validdocids snapshots redundantly
(#12246)
add 747e34dede Allow String / numeric data type for deleteRecordColumn
config (#12222)
add 7132a2203f [multistage][bugfix] sort copy rule to always push limit
when no collation (#12251)
add 2e367a202e UI: fix table link issue in task detail page (#12253)
add b4fbfe9687 Making utility accessible to generate metadata file (#12255)
add a4c3286018 add null handling to sketch group-by (#12259)
add ae55a7abf8 Catch-all Regex for JXM -> Prom Exporter (#12073)
add 19e74e80a8 Added dynamic SSL initialization support for the Kafka
client (#12249)
add 21f3d283d4 Allow server level configuration for Upsert metadata class
(#11851)
add 110c5b4947 Bump com.yscope.clp:clp-ffi from 0.4.3 to 0.4.4 (#12203)
add 12c90b2155 Bump flink.version from 1.12.0 to 1.14.6 (#12202)
add 8713dc045f Bump commons-codec:commons-codec from 1.15 to 1.16.0
(#12204)
add 4ad36c3482 Remove TableDataManagerConfig and simplify TableDataManager
construction (#12189)
add 6bb387a10a Support array gen in literal evaluation (#12278)
add 17e1aa11a4 Fix chained literal functions evaluation (#12248)
add 894e56e0a2 Expose metric for table rebalance (#12270)
add 7b2a82dd27 Modify distribution pom to include services (#12289)
add 640ebe52cf Sticky query routing via query options (#12276)
add 0f6015a5d3 JMX Exporter Preserve Original Regexes (#12295)
add 5cc7231dc0 Backwards compatible theta sketch aggregation (#12288)
add 7b69d094be Refactoring the upsert compaction related code (#12275)
add ced6bc282e Use higher fetch timeout for Kinesis (#12214)
add f1fec060a6 Support runtime reload for TLS resources (#12277)
add eeaf1f0811 [feature] allow dim table config to detect/disallow
duplicate PK (#12290)
add 35faeb6712 Revert allowing tag override with upserts (#12311)
add 63e91ef95f Support table suffix in ZkBasicAuthAccessControlFactory
(#12310)
add 91ffcc759f Reduce Disk Footprint for Segment Processor Framework to
Avoid Out of Disk Issues (#12220)
add f43664dd13 Support server level consumption throttle (#12292)
add 9c1bb02dec Misc fixes for upsert metadata manager (#12319)
add 76d0eb25db Reduce Heap Usage of OnHeapStringDictionary (#12223)
add 7978d29031 Mark distribution as multi-release (#12300)
add 5a382f2e7d Shared aggregations in StarTree (#12164)
add 2acf8ea354 correct errmsg format for Preconditions.check... (#12327)
add 23dbb08e8d Make server resource classes configurable (#12324)
add ed6761a982 Fix "rewind()" for CompactedPinotSegmentRecordReader
(#12329)
add 4823802886 Wire soft upsert delete for compaction task (#12330)
add 6cc1915140 Add HttpHeaders in broker event listener requestContext
(#12258)
add e1d20d75d5 Making taskManager resources protected for derived classes
to override in their setUp() method. (#12335)
add 1a82ba6fc6 Add CPU metrics for minion purge task (#12337)
add d5014786db Add first implementation of clpMatch that doesn't
explicitly use indexes. (#12291)
add 3ab28348fb auto reload TLS resources when loading from local files
(#12325)
add 8988b755d6 Remove segments with empty download url in
UpsertCompactionTask (#12320)
add dd8be2a229 add TextMatchFilterOptimizer to maximally push down
`text_match` filters to Lucene (#12339)
add 17db0fd17b add ScalingThreadPoolExecutor and use for realtime Lucene
thread pool (#12274)
new 9550545be3 Enhance the minimizeDataMovement to keep the existing pool
assignment
new 97ab0b7d45 Update
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/assignment/instance/InstancePartitionSelector.java
new 2a934ef00d Address PR comments
new 25b4c3a212 Add logic to consider the case when instances are moved
across pools
new a286bd868f Enhance algorithm
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (08efc8c2f1)
\
N -- N -- N
refs/heads/maintain-pool-selection-for-minimizeDataMovement (a286bd868f)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.asf.yaml | 7 +
.github/workflows/scripts/.pinot_quickstart.sh | 71 +-
.mvn/extensions.xml | 2 +-
LICENSE-binary | 5 +-
README.md | 2 -
compatibility-verifier/compCheck.sh | 6 +
.../config/queries/feature-test-1-sql.queries | 6 +-
.../queries/feature-test-2-sql-realtime.queries | 4 +-
.../query-results/feature-test-1-rest-sql.results | 6 +-
.../feature-test-2-sql-realtime.results | 4 +-
contrib/pinot-fmpp-maven-plugin/pom.xml | 2 +-
.../jmx_prometheus_javaagent/configs/broker.yml | 26 +-
.../configs/controller.yml | 26 +-
.../etc/jmx_prometheus_javaagent/configs/pinot.yml | 27 +-
.../jmx_prometheus_javaagent/configs/server.yml | 27 +-
helm/pinot/templates/server/statefulset.yaml | 4 +-
helm/pinot/values.yaml | 10 +-
.../broker/api/resources/PinotBrokerDebug.java | 90 ++-
.../broker/ZkBasicAuthAccessControlFactory.java | 26 +-
.../broker/broker/helix/BaseBrokerStarter.java | 25 +-
.../requesthandler/BaseBrokerRequestHandler.java | 75 +-
.../requesthandler/GrpcBrokerRequestHandler.java | 19 +-
.../MultiStageBrokerRequestHandler.java | 14 +-
.../SingleConnectionBrokerRequestHandler.java | 8 +-
.../pinot/broker/routing/BrokerRoutingManager.java | 37 +-
.../AdaptiveServerSelectorFactory.java | 2 +
.../instanceselector/BalancedInstanceSelector.java | 31 +-
.../instanceselector/BaseInstanceSelector.java | 45 +-
.../routing/instanceselector/InstanceSelector.java | 21 +-
.../instanceselector/InstanceSelectorFactory.java | 30 +-
.../MultiStageReplicaGroupSelector.java | 38 +-
.../ReplicaGroupInstanceSelector.java | 55 +-
.../StrictReplicaGroupInstanceSelector.java | 9 +-
.../MultiPartitionColumnsSegmentPruner.java | 5 +-
.../SinglePartitionColumnSegmentPruner.java | 5 +-
.../broker/broker/FakeStreamConsumerFactory.java | 6 +
.../broker/broker/HelixBrokerStarterTest.java | 9 +-
.../BaseBrokerRequestHandlerTest.java | 9 +-
.../instanceselector/InstanceSelectorTest.java | 108 +--
.../pinot/client/BrokerCacheUpdaterPeriodic.java | 8 +-
.../apache/pinot/client/utils/ConnectionUtils.java | 2 +-
.../org/apache/pinot/client/PinotConnection.java | 4 -
.../org/apache/pinot/client/utils/DriverUtils.java | 2 +-
pinot-common/pom.xml | 22 +-
.../pinot/common/auth/AuthProviderUtils.java | 3 +
.../config/provider/AccessControlUserCache.java | 243 +++---
.../pinot/common/datatable/BaseDataTable.java | 4 +
.../apache/pinot/common/datatable/DataTable.java | 5 +-
.../pinot/common/datatable/DataTableImplV4.java | 4 +
.../pinot/common/function/FunctionRegistry.java | 51 +-
.../common/function/TransformFunctionType.java | 10 +-
.../common/function/scalar/ArrayFunctions.java | 59 ++
.../common/function/scalar/StringFunctions.java | 33 +
.../common/function/scalar/VectorFunctions.java | 32 +
.../apache/pinot/common/http/MultiHttpRequest.java | 51 +-
.../pinot/common/metrics/AbstractMetrics.java | 6 +
.../pinot/common/metrics/ControllerGauge.java | 10 +-
.../pinot/common/metrics/ControllerMetrics.java | 10 +
.../apache/pinot/common/metrics/MinionMetrics.java | 10 +
.../apache/pinot/common/metrics/MinionTimer.java | 2 +-
.../apache/pinot/common/metrics/ServerMeter.java | 2 +
.../apache/pinot/common/metrics/ServerTimer.java | 5 +-
.../protocols/SegmentCompletionProtocol.java | 2 +
.../apache/pinot/common/request/BrokerRequest.java | 2 +-
.../apache/pinot/common/request/DataSource.java | 2 +-
.../apache/pinot/common/request/Expression.java | 2 +-
.../pinot/common/request/ExpressionType.java | 2 +-
.../org/apache/pinot/common/request/Function.java | 38 +-
.../apache/pinot/common/request/Identifier.java | 2 +-
.../pinot/common/request/InstanceRequest.java | 186 ++++-
.../java/org/apache/pinot/common/request/Join.java | 2 +-
.../org/apache/pinot/common/request/JoinType.java | 2 +-
.../org/apache/pinot/common/request/Literal.java | 469 ++++++++++-
.../apache/pinot/common/request/PinotQuery.java | 2 +-
.../apache/pinot/common/request/QuerySource.java | 2 +-
.../common/request/context/FilterContext.java | 73 +-
.../common/request/context/LiteralContext.java | 20 +
.../request/context/RequestContextUtils.java | 313 +++++---
.../request/context/predicate/Predicate.java | 3 +-
.../predicate/VectorSimilarityPredicate.java | 82 ++
.../pinot/common/response/BrokerResponse.java | 5 +
.../pinot/common/response/ProcessingException.java | 2 +-
.../response/broker/BrokerResponseNative.java | 21 +-
.../response/broker/BrokerResponseNativeV2.java | 11 +-
.../response/broker/BrokerResponseStats.java | 14 +-
.../restlet/resources/ValidDocIdMetadataInfo.java | 56 ++
.../common/tier/FixedTierSegmentSelector.java | 2 +-
.../org/apache/pinot/common/utils/BcryptUtils.java | 53 +-
.../common/utils/FileUploadDownloadClient.java | 10 +-
.../apache/pinot/common/utils/PinotDataType.java | 8 +-
.../common/utils/ScalingThreadPoolExecutor.java | 128 +++
.../org/apache/pinot/common/utils/TlsUtils.java | 258 +++++-
.../common/utils/config/QueryOptionsUtils.java | 12 +
.../pinot/common/utils/grpc/GrpcQueryClient.java | 24 +-
.../pinot/common/utils/request/RequestUtils.java | 49 ++
.../main/java/org/apache/pinot/sql/FilterKind.java | 3 +-
.../apache/pinot/sql/parsers/CalciteSqlParser.java | 20 +
.../sql/parsers/rewriter/CLPDecodeRewriter.java | 177 -----
.../pinot/sql/parsers/rewriter/ClpRewriter.java | 633 +++++++++++++++
.../rewriter/CompileTimeFunctionsInvoker.java | 9 +-
.../sql/parsers/rewriter/ExprMinMaxRewriter.java | 16 +-
.../rewriter/PredicateComparisonRewriter.java | 33 +-
.../apache/pinot/common/data/FieldSpecTest.java | 45 ++
.../org/apache/pinot/common/data/SchemaTest.java | 26 +
.../function/FunctionDefinitionRegistryTest.java | 6 +-
.../pinot/common/http/MultiHttpRequestTest.java | 106 ++-
.../segment/ColumnPartitionMetadataTest.java | 4 +-
.../pinot/common/utils/FALFInternerTest.java | 168 ++++
.../utils/ScalingThreadPoolExecutorTest.java | 69 ++
.../apache/pinot/common/utils/TlsUtilsTest.java | 286 +++++++
.../common/utils/config/TableConfigSerDeTest.java | 2 +-
.../parsers/rewriter/CLPDecodeRewriterTest.java | 65 --
.../sql/parsers/rewriter/ClpRewriterTest.java | 282 +++++++
.../parsers/rewriter/ExprMinMaxRewriterTest.java | 2 +-
.../src/test/resources/tls/keystore-updated.p12 | Bin 0 -> 2581 bytes
pinot-common/src/test/resources/tls/keystore.p12 | Bin 0 -> 2581 bytes
.../src/test/resources/tls/truststore-updated.p12 | Bin 0 -> 1186 bytes
pinot-common/src/test/resources/tls/truststore.p12 | Bin 0 -> 1186 bytes
pinot-common/src/thrift/query.thrift | 5 +
pinot-common/src/thrift/request.thrift | 1 +
pinot-connectors/pinot-spark-2-connector/pom.xml | 2 +-
pinot-connectors/pinot-spark-3-connector/pom.xml | 2 +-
pinot-connectors/pinot-spark-common/pom.xml | 4 +-
.../common/reader/PinotServerDataFetcher.scala | 11 +-
.../pinot/controller/BaseControllerStarter.java | 1 +
.../apache/pinot/controller/ControllerConf.java | 8 +-
.../access/ZkBasicAuthAccessControlFactory.java | 29 +-
.../resources/LLCSegmentCompletionHandlers.java | 6 +-
.../api/resources/PinotQueryResource.java | 4 +
.../api/resources/PinotRealtimeTableResource.java | 52 +-
.../api/resources/PinotRunningQueryResource.java | 2 +-
.../api/resources/PinotTableRestletResource.java | 71 +-
.../controller/helix/ControllerRequestClient.java | 10 +
.../controller/helix/SegmentStatusChecker.java | 36 +-
.../helix/core/PinotHelixResourceManager.java | 15 +-
.../helix/core/SegmentDeletionManager.java | 2 +-
.../instance/FDAwareInstancePartitionSelector.java | 7 +-
.../instance/InstancePartitionSelector.java | 10 +-
.../InstanceReplicaGroupPartitionSelector.java | 768 +++++++++---------
.../instance/InstanceTagPoolSelector.java | 56 +-
.../assignment/segment/SegmentAssignmentUtils.java | 8 +-
.../segment/StrictRealtimeSegmentAssignment.java | 174 ++--
.../minion/generator/TaskGeneratorRegistry.java | 2 +-
.../realtime/PinotLLCRealtimeSegmentManager.java | 320 +++++---
.../segment/CommittingSegmentDescriptor.java | 10 +
.../segment/DefaultFlushThresholdUpdater.java | 4 +-
.../realtime/segment/FlushThresholdUpdater.java | 4 +-
.../segment/SegmentFlushThresholdComputer.java | 43 +-
.../SegmentSizeBasedFlushThresholdUpdater.java | 14 +-
.../helix/core/rebalance/RebalanceChecker.java | 2 +-
.../helix/core/rebalance/RebalanceConfig.java | 16 +
.../helix/core/rebalance/TableRebalancer.java | 126 +--
.../rebalance/ZkBasedTableRebalanceObserver.java | 8 +
.../helix/core/relocation/SegmentRelocator.java | 2 +-
.../recommender/data/DataGenerationHelpers.java | 138 ++++
.../recommender/data/generator/DataGenerator.java | 104 +--
.../data/generator/DataGeneratorSpec.java | 116 ++-
.../data/generator/DateTimeGenerator.java | 51 ++
.../data/generator/PatternSeasonalGenerator.java | 2 +-
.../data/generator/PatternSequenceGenerator.java | 2 +-
.../data/generator/PatternSpikeGenerator.java | 2 +-
.../data/generator/PatternStringGenerator.java | 2 +-
.../data/{generator => writer}/AvroWriter.java | 72 +-
.../recommender/data/writer/AvroWriterSpec.java | 41 +-
.../recommender/data/writer/CsvWriter.java | 66 ++
.../recommender/data/writer/FileWriter.java | 81 ++
.../recommender/data/writer/FileWriterSpec.java | 35 +-
.../recommender/data/writer/JsonWriter.java | 33 +-
.../controller/recommender/data/writer/Writer.java | 55 +-
.../recommender/data/writer/WriterSpec.java | 18 +-
.../controller/recommender/io/InputManager.java | 8 +-
.../realtime/provisioning/MemoryEstimator.java | 15 +-
.../recommender/rules/impl/BloomFilterRule.java | 7 +-
.../recommender/rules/impl/FlagQueryRule.java | 6 +-
.../recommender/rules/impl/JsonIndexRule.java | 2 +-
.../NoDictionaryOnHeapDictionaryJointRule.java | 5 +-
.../rules/impl/PinotTablePartitionRule.java | 7 +-
.../recommender/rules/impl/RangeIndexRule.java | 5 +-
.../utils/QueryInvertedSortedIndexRecommender.java | 7 +-
.../controller/util/CompletionServiceHelper.java | 52 +-
.../util/ServerSegmentMetadataReader.java | 161 +++-
.../pinot/controller/util/TableMetadataReader.java | 76 +-
.../app/components/AsyncInstanceTable.tsx | 122 +++
.../resources/app/components/AsyncPinotSchemas.tsx | 76 ++
.../resources/app/components/AsyncPinotTables.tsx | 220 +++++
.../app/components/Homepage/InstancesTables.tsx | 34 +-
.../Homepage/Operations/RebalanceServerTableOp.tsx | 15 +-
.../app/components/Homepage/TenantsListing.tsx | 65 +-
.../src/main/resources/app/components/Loading.tsx | 13 +-
.../src/main/resources/app/components/NotFound.tsx | 69 ++
.../src/main/resources/app/components/Table.tsx | 2 +-
.../src/main/resources/app/interfaces/types.d.ts | 37 +-
.../src/main/resources/app/pages/HomePage.tsx | 126 ++-
.../main/resources/app/pages/InstanceDetails.tsx | 472 ++++++-----
.../resources/app/pages/InstanceListingPage.tsx | 11 +-
.../src/main/resources/app/pages/Query.tsx | 10 +-
.../main/resources/app/pages/SchemaPageDetails.tsx | 323 ++++----
.../main/resources/app/pages/SegmentDetails.tsx | 440 +++++++---
.../main/resources/app/pages/TablesListingPage.tsx | 112 +--
.../src/main/resources/app/pages/TaskQueue.tsx | 2 +-
.../src/main/resources/app/pages/TenantDetails.tsx | 567 +++++++------
.../src/main/resources/app/pages/Tenants.tsx | 107 +--
.../resources/app/pages/TenantsListingPage.tsx | 31 +-
.../src/main/resources/app/requests/index.ts | 34 +-
pinot-controller/src/main/resources/app/router.tsx | 1 +
.../main/resources/app/utils/PinotMethodUtils.ts | 40 +-
.../src/main/resources/app/utils/Utils.tsx | 73 +-
.../api/PinotTenantRestletResourceTest.java | 247 +++---
.../pinot/controller/helix/ControllerTest.java | 3 +-
.../PinotHelixResourceManagerStatelessTest.java | 4 +-
.../instance/InstanceAssignmentTest.java | 43 +-
.../InstanceReplicaGroupPartitionSelectorTest.java | 141 ++--
.../PinotLLCRealtimeSegmentManagerTest.java | 15 +
.../segment/FlushThresholdUpdaterTest.java | 108 +--
.../segment/SegmentFlushThresholdComputerTest.java | 69 +-
.../helix/core/rebalance/TableRebalancerTest.java | 885 +++++++++++++++++++--
.../TestZkBasedTableRebalanceObserver.java | 3 +
pinot-core/pom.xml | 26 +-
.../apache/pinot/core/common/MinionConstants.java | 1 +
.../apache/pinot/core/common/ObjectSerDeUtils.java | 43 +-
.../core/data/manager/BaseTableDataManager.java | 125 +--
.../core/data/manager/InstanceDataManager.java | 2 +-
.../manager/offline/DimensionTableDataManager.java | 9 +-
.../manager/offline/TableDataManagerProvider.java | 52 +-
.../manager/realtime/IngestionDelayTracker.java | 63 +-
.../realtime/RealtimeConsumptionRateManager.java | 39 +-
.../realtime/RealtimeSegmentDataManager.java | 73 +-
.../manager/realtime/RealtimeTableDataManager.java | 72 +-
.../realtime/SegmentBuildTimeLeaseExtender.java | 4 +-
.../core/function/scalar/SketchFunctions.java | 11 +
.../blocks/results/GroupByResultsBlock.java | 6 +
.../filter/VectorSimilarityFilterOperator.java | 118 +++
.../core/operator/query/AggregationOperator.java | 10 +-
.../query/FilteredAggregationOperator.java | 27 +-
.../operator/query/FilteredGroupByOperator.java | 42 +-
.../pinot/core/operator/query/GroupByOperator.java | 19 +-
.../function/ArrayLiteralTransformFunction.java | 72 ++
.../transform/function/CastTransformFunction.java | 17 +
.../ClpEncodedVarsMatchTransformFunction.java | 150 ++++
.../function/TransformFunctionFactory.java | 9 +-
.../pinot/core/plan/AggregationPlanNode.java | 63 +-
.../org/apache/pinot/core/plan/FilterPlanNode.java | 18 +-
.../apache/pinot/core/plan/GroupByPlanNode.java | 59 +-
.../function/AggregationFunctionUtils.java | 169 ++--
...inctCountRawThetaSketchAggregationFunction.java | 18 +-
...istinctCountThetaSketchAggregationFunction.java | 193 +++--
.../function/HistogramAggregationFunction.java | 33 +-
.../core/query/config/QueryExecutorConfig.java | 2 +-
.../pinot/core/query/executor/QueryExecutor.java | 2 +-
.../query/executor/ServerQueryExecutorV1Impl.java | 33 +-
.../pinot/core/query/logger/ServerQueryLogger.java | 2 +-
.../pinot/core/query/optimizer/QueryOptimizer.java | 8 +-
.../optimizer/filter/NumericalFilterOptimizer.java | 24 +-
.../optimizer/filter/TextMatchFilterOptimizer.java | 186 +++++
.../query/pruner/ColumnValueSegmentPruner.java | 5 +-
.../core/query/pruner/ValueBasedSegmentPruner.java | 11 +-
.../pinot/core/query/reduce/BaseReduceService.java | 3 +
.../core/query/reduce/BrokerReduceService.java | 14 +-
.../core/query/reduce/DataTableReducerContext.java | 8 +-
.../query/reduce/ExecutionStatsAggregator.java | 6 +
.../core/query/reduce/GroupByDataTableReducer.java | 3 +-
.../core/query/reduce/StreamingReduceService.java | 13 +-
.../core/query/request/ServerQueryRequest.java | 8 +
.../context/utils/QueryContextConverterUtils.java | 8 +
.../pinot/core/query/scheduler/QueryScheduler.java | 12 +
.../query/selection/SelectionOperatorUtils.java | 5 +
.../apache/pinot/core/routing/RoutingTable.java | 13 +-
.../processing/framework/SegmentConfig.java | 30 +-
.../framework/SegmentProcessorFramework.java | 139 +++-
.../genericrow/AdaptiveConstraintsWriter.java | 17 +-
.../genericrow/AdaptiveSizeBasedWriter.java | 33 +-
.../segment/processing/genericrow/FileWriter.java | 14 +-
.../genericrow/GenericRowFileWriter.java | 11 +-
.../segment/processing/mapper/SegmentMapper.java | 70 +-
.../partitioner/TableConfigPartitioner.java | 3 +-
.../apache/pinot/core/startree/StarTreeUtils.java | 55 +-
.../executor/StarTreeAggregationExecutor.java | 2 +-
.../startree/executor/StarTreeGroupByExecutor.java | 18 +-
.../pinot/core/transport/DirectOOMHandler.java | 2 +-
.../apache/pinot/core/transport/QueryRouter.java | 24 +-
.../pinot/core/transport/grpc/GrpcQueryServer.java | 12 +-
.../apache/pinot/core/util/ListenerConfigUtil.java | 2 +-
.../java/org/apache/pinot/core/util/OsCheck.java | 18 +-
.../server/access/ZkBasicAuthAccessFactory.java | 26 +-
.../accounting/ResourceManagerAccountingTest.java | 145 ++++
.../pinot/core/common/ObjectSerDeUtilsTest.java | 51 ++
.../function/InbuiltFunctionEvaluatorTest.java | 2 +-
.../BaseTableDataManagerAcquireSegmentTest.java | 40 +-
.../data/manager/BaseTableDataManagerTest.java | 90 +--
.../offline/DimensionTableDataManagerTest.java | 95 ++-
.../RealtimeConsumptionRateManagerTest.java | 49 +-
.../realtime/RealtimeSegmentDataManagerTest.java | 73 +-
.../realtime/RealtimeTableDataManagerTest.java | 77 +-
.../core/function/scalar/SketchFunctionsTest.java | 22 +
.../combine/SelectionCombineOperatorTest.java | 50 +-
...ionTest.java => ClpTransformFunctionsTest.java} | 111 ++-
...adataAndDictionaryAggregationPlanMakerTest.java | 18 +-
.../executor/QueryExecutorExceptionsTest.java | 34 +-
.../core/query/executor/QueryExecutorTest.java | 54 +-
.../core/query/optimizer/QueryOptimizerTest.java | 55 ++
.../BrokerRequestToQueryContextConverterTest.java | 89 ++-
.../query/scheduler/PrioritySchedulerTest.java | 2 +-
.../fakestream/FakeStreamMetadataProvider.java | 8 +
.../core/realtime/stream/StreamConfigTest.java | 27 +
.../framework/SegmentProcessorFrameworkTest.java | 205 ++++-
.../pinot/core/startree/v2/BaseStarTreeV2Test.java | 35 +-
.../v2/DistinctCountRawHLLStarTreeV2Test.java | 58 ++
.../pinot/core/transport/QueryRoutingTest.java | 5 +-
.../stats/ServerRoutingStatsManagerTest.java | 90 ++-
.../DistinctCountThetaSketchQueriesTest.java | 32 +-
.../pinot/queries/ExplainPlanQueriesTest.java | 49 +-
.../apache/pinot/queries/HistogramQueriesTest.java | 4 +-
.../pinot/queries/JsonMalformedIndexTest.java | 131 +++
.../org/apache/pinot/queries/QueriesTestUtils.java | 25 +
.../queries/SegmentWithNullValueVectorTest.java | 60 +-
.../src/test/resources/data/dimBaseballTeams.avro | Bin 2214 -> 0 bytes
.../src/test/resources/data/dimBaseballTeams.avsc | 8 -
.../src/test/resources/data/dimBaseballTeams.csv | 1 +
.../resources/data/dimBaseballTeams_config.json | 0
pinot-distribution/pom.xml | 14 +-
pinot-integration-test-base/pom.xml | 4 +
.../tests/BaseClusterIntegrationTest.java | 20 +-
.../pinot/integration/tests/QueryGenerator.java | 3 +-
pinot-integration-tests/pom.xml | 4 +
.../tests/BaseRealtimeClusterIntegrationTest.java | 6 +
.../tests/HybridClusterIntegrationTest.java | 78 +-
.../tests/LLCRealtimeClusterIntegrationTest.java | 149 ++++
.../tests/OfflineClusterIntegrationTest.java | 125 ++-
...flineClusterMemBasedServerQueryKillingTest.java | 14 +
.../tests/OfflineGRPCServerIntegrationTest.java | 2 +-
...PartialUpsertTableRebalanceIntegrationTest.java | 9 +-
...nsumptionRateLimiterClusterIntegrationTest.java | 298 +++++++
...gmentGenerationMinionRealtimeIngestionTest.java | 167 ++++
...PartitionLLCRealtimeClusterIntegrationTest.java | 2 +-
.../tests/StarTreeClusterIntegrationTest.java | 52 +-
...sertCompactionMinionClusterIntegrationTest.java | 221 -----
.../tests/UpsertTableIntegrationTest.java | 398 +++++++--
.../UpsertTableSegmentPreloadIntegrationTest.java | 11 +-
.../pinot/integration/tests/custom/ArrayTest.java | 196 ++++-
.../integration/tests/custom/ThetaSketchTest.java | 11 +
.../integration/tests/custom/TimestampTest.java | 54 +-
.../models/DummyTableUpsertMetadataManager.java | 115 +++
.../minion/tasks/TestTaskExecutorFactory.java | 3 +-
.../tests/startree/StarTreeQueryGenerator.java | 17 +-
.../src/test/resources/gameScores_large_csv.tar.gz | Bin 0 -> 12810 bytes
.../src/test/resources/test_null_handling.schema | 1 -
.../test/resources/upsert_compaction_test.tar.gz | Bin 9405 -> 0 bytes
.../org/apache/pinot/minion/BaseMinionStarter.java | 1 +
.../org/apache/pinot/minion/MinionContext.java | 2 +
.../minion/taskfactory/TaskFactoryRegistry.java | 2 +-
.../org/apache/pinot/minion/MinionConfTest.java | 15 +-
.../batch/hadoop/HadoopSegmentCreationMapper.java | 2 +-
pinot-plugins/pinot-file-system/pinot-adls/pom.xml | 4 +-
pinot-plugins/pinot-file-system/pinot-gcs/pom.xml | 4 -
pinot-plugins/pinot-file-system/pinot-s3/pom.xml | 6 +-
.../apache/pinot/plugin/filesystem/S3PinotFS.java | 2 +-
.../inputformat/clplog/CLPLogRecordExtractor.java | 8 +-
.../clplog/CLPLogRecordExtractorTest.java | 14 +-
.../parquet/ParquetRecordReaderConfig.java | 2 +-
.../tasks/BaseSingleSegmentConversionExecutor.java | 4 +-
.../pinot/plugin/minion/tasks/MergeTaskUtils.java | 4 +
.../pinot/plugin/minion/tasks/MinionTaskUtils.java | 41 +
.../mergerollup/MergeRollupTaskGenerator.java | 4 +-
.../minion/tasks/purge/PurgeTaskExecutor.java | 19 +-
.../minion/tasks/purge/PurgeTaskGenerator.java | 2 +-
.../RealtimeToOfflineSegmentsTaskGenerator.java | 6 +-
.../SegmentGenerationAndPushTaskGenerator.java | 41 +-
.../UpsertCompactionTaskExecutor.java | 141 +---
.../UpsertCompactionTaskGenerator.java | 120 +--
.../plugin/minion/tasks/MergeTaskUtilsTest.java | 6 +-
.../minion/tasks/purge/PurgeTaskExecutorTest.java | 3 +-
.../UpsertCompactionTaskExecutorTest.java | 7 +-
.../UpsertCompactionTaskGeneratorTest.java | 86 +-
.../pinot-stream-ingestion/pinot-kafka-2.0/pom.xml | 13 +
.../KafkaPartitionLevelConnectionHandler.java | 1 +
.../pinot/plugin/stream/kafka20/KafkaSSLUtils.java | 339 ++++++++
.../kafka20/KafkaStreamMetadataProvider.java | 19 +
.../plugin/stream/kafka20/KafkaSSLUtilsTest.java | 310 ++++++++
.../stream/kinesis/KinesisConnectionHandler.java | 2 +-
.../stream/kinesis/server/KinesisDataProducer.java | 2 +-
.../pinot-stream-ingestion/pinot-pulsar/pom.xml | 12 +-
.../pinot/plugin/stream/pulsar/PulsarConfig.java | 89 ++-
.../PulsarPartitionLevelConnectionHandler.java | 26 +
.../pulsar/PulsarStreamMetadataProvider.java | 19 +-
.../plugin/stream/pulsar/PulsarConfigTest.java | 58 ++
.../apache/calcite/rel/hint/PinotHintOptions.java | 5 +
.../rel/rules/PinotEvaluateLiteralRule.java | 53 +-
.../rel/rules/PinotJoinToDynamicBroadcastRule.java | 24 +-
.../calcite/rel/rules/PinotQueryRuleSets.java | 22 +-
.../rel/rules/PinotRelDistributionTraitRule.java | 177 +++++
.../apache/calcite/rel/rules/PinotRuleUtils.java | 48 +-
.../rel/rules/PinotSortExchangeCopyRule.java | 6 +
.../org/apache/pinot/query/QueryEnvironment.java | 109 ++-
.../apache/pinot/query/context/PlannerContext.java | 13 +-
.../query/parser/CalciteRexExpressionParser.java | 8 +-
.../apache/pinot/query/planner/PlanFragment.java | 9 +-
.../{ => explain}/PhysicalExplainPlanVisitor.java | 83 +-
.../query/planner/logical/LogicalPlanner.java | 6 +-
.../planner/logical/PinotLogicalQueryPlanner.java | 11 +-
.../query/planner/logical/PlanFragmenter.java | 6 +-
.../planner/logical/RelToPlanNodeConverter.java | 7 +-
.../planner/physical/DispatchablePlanContext.java | 1 -
.../{ => physical}/DispatchablePlanFragment.java | 3 +-
.../planner/physical/DispatchablePlanMetadata.java | 43 +-
.../planner/physical/DispatchablePlanVisitor.java | 3 +-
.../{ => physical}/DispatchableSubPlan.java | 2 +-
.../planner/physical/MailboxAssignmentVisitor.java | 31 +-
.../planner/physical/PinotDispatchPlanner.java | 3 -
.../query/planner/plannode/AggregateNode.java | 2 +-
.../pinot/query/planner/plannode/ExchangeNode.java | 10 +-
.../query/planner/plannode/MailboxReceiveNode.java | 6 +-
.../query/planner/plannode/MailboxSendNode.java | 21 +-
.../query/planner/plannode/PlanNodeVisitor.java | 5 +-
.../pinot/query/planner/plannode/WindowNode.java | 2 +-
.../apache/pinot/query/routing/WorkerManager.java | 426 +++++-----
.../org/apache/pinot/query/type/TypeFactory.java | 71 +-
.../rel/rules/PinotSortExchangeCopyRuleTest.java | 12 +-
.../apache/pinot/query/QueryCompilationTest.java | 69 +-
.../pinot/query/QueryEnvironmentTestBase.java | 23 +-
.../query/planner/plannode/SerDeUtilsTest.java | 4 +-
.../query/queries/ResourceBasedQueryPlansTest.java | 26 +-
.../query/testutils/MockRoutingManagerFactory.java | 12 +-
.../apache/pinot/query/type/TypeFactoryTest.java | 221 ++++-
.../src/test/resources/queries/AggregatePlans.json | 49 +-
.../test/resources/queries/BasicQueryPlans.json | 10 +-
.../resources/queries/ExplainPhysicalPlans.json | 455 +++++++++++
.../src/test/resources/queries/GroupByPlans.json | 61 +-
.../src/test/resources/queries/JoinPlans.json | 200 +++--
.../resources/queries/LiteralEvaluationPlans.json | 114 ++-
.../src/test/resources/queries/OrderByPlans.json | 81 +-
.../test/resources/queries/PinotHintablePlans.json | 427 +++++++++-
.../src/test/resources/queries/SetOpPlans.json | 2 +-
.../resources/queries/ValidationErrorPlan.json | 8 +-
.../resources/queries/WindowFunctionPlans.json | 716 ++++++++---------
.../query/mailbox/InMemorySendingMailbox.java | 2 +-
.../apache/pinot/query/runtime/QueryRunner.java | 31 +-
.../query/runtime/operator/AggregateOperator.java | 35 +-
.../query/runtime/operator/HashJoinOperator.java | 59 +-
.../LeafStageTransferableBlockOperator.java | 37 +-
.../runtime/operator/LiteralValueOperator.java | 3 +-
.../query/runtime/operator/MultiStageOperator.java | 16 +-
.../query/runtime/operator/OperatorStats.java | 2 -
.../pinot/query/runtime/operator/SetOperator.java | 18 +-
.../pinot/query/runtime/operator/SortOperator.java | 107 ++-
.../query/runtime/operator/TransformOperator.java | 24 +-
.../runtime/operator/WindowAggregateOperator.java | 21 +-
.../runtime/operator/operands/FunctionOperand.java | 21 +-
.../runtime/operator/utils/OperatorUtils.java | 2 +-
.../query/runtime/operator/utils/TypeUtils.java | 8 +
.../runtime/plan/OpChainExecutionContext.java | 11 +
.../query/runtime/plan/PhysicalPlanVisitor.java | 31 +-
.../runtime/plan/serde/QueryPlanSerDeUtils.java | 4 +-
.../plan/server/ServerPlanRequestContext.java | 68 +-
.../plan/server/ServerPlanRequestUtils.java | 157 ++--
.../plan/server/ServerPlanRequestVisitor.java | 135 ++--
.../query/service/dispatch/QueryDispatcher.java | 4 +-
.../pinot/query/service/server/QueryServer.java | 3 +-
.../runtime/operator/AggregateOperatorTest.java | 50 ++
.../runtime/operator/HashJoinOperatorTest.java | 16 +-
.../query/runtime/queries/QueryRunnerTest.java | 36 +-
.../query/runtime/queries/QueryRunnerTestBase.java | 62 +-
.../runtime/queries/ResourceBasedQueriesTest.java | 14 +-
.../service/dispatch/QueryDispatcherTest.java | 2 +-
.../query/service/server/QueryServerTest.java | 4 +-
.../testutils/MockInstanceDataManagerFactory.java | 12 +-
.../src/test/resources/queries/BasicQuery.json | 4 +
.../src/test/resources/queries/CharacterTypes.json | 2 -
.../src/test/resources/queries/CountDistinct.json | 8 +-
.../test/resources/queries/FromExpressions.json | 30 +
.../test/resources/queries/MetadataTestQuery.json | 2 +-
.../src/test/resources/queries/NullHandling.json | 338 +++++++-
.../src/test/resources/queries/QueryHints.json | 391 ++++++++-
.../test/resources/queries/StringFunctions.json | 2 +
.../src/test/resources/queries/WithStatements.json | 21 +
pinot-segment-local/pom.xml | 13 +-
.../DistinctCountHLLPlusValueAggregator.java | 2 +-
.../local/customobject/ThetaSketchAccumulator.java | 135 ++++
.../local/data/manager/TableDataManager.java | 34 +-
.../local/data/manager/TableDataManagerConfig.java | 113 ---
.../local/data/manager/TableDataManagerParams.java | 66 --
.../local/function/InbuiltFunctionEvaluator.java | 6 +
.../immutable/ImmutableSegmentImpl.java | 18 +-
.../immutable/ImmutableSegmentLoader.java | 21 +-
.../indexsegment/mutable/MutableSegmentImpl.java | 30 +-
.../local/io/util/FixedBitIntReaderWriter.java | 8 +
.../segment/local/io/util/PinotDataBitSet.java | 6 +
.../FixedBitMVEntryDictForwardIndexWriter.java | 126 +++
.../local/realtime/impl/RealtimeSegmentConfig.java | 28 +-
.../invertedindex/RealtimeLuceneTextIndex.java | 15 +-
.../RealtimeLuceneTextIndexSearcherPool.java | 6 +-
.../realtime/impl/json/MutableJsonIndexImpl.java | 1 +
.../realtime/impl/vector/MutableVectorIndex.java | 154 ++++
.../recordtransformer/CompositeTransformer.java | 16 +-
.../recordtransformer/SpecialValueTransformer.java | 128 +++
.../creator/impl/ColumnJsonParserException.java | 48 ++
.../creator/impl/SegmentColumnarIndexCreator.java | 56 +-
.../MultiValueEntryDictForwardIndexCreator.java | 67 ++
.../creator/impl/inv/RangeIndexCreator.java | 5 +-
.../impl/inv/text/LuceneFSTIndexCreator.java | 7 +-
.../stats/AbstractColumnStatisticsCollector.java | 13 +-
.../BigDecimalColumnPreIndexStatsCollector.java | 4 +-
.../stats/BytesColumnPredIndexStatsCollector.java | 4 +-
.../stats/DoubleColumnPreIndexStatsCollector.java | 4 +-
.../stats/FloatColumnPreIndexStatsCollector.java | 4 +-
.../stats/IntColumnPreIndexStatsCollector.java | 4 +-
.../stats/LongColumnPreIndexStatsCollector.java | 4 +-
.../stats/StringColumnPreIndexStatsCollector.java | 4 +-
.../creator/impl/text/LuceneTextIndexCreator.java | 36 +-
.../impl/vector/HnswVectorIndexCreator.java | 120 +++
.../creator/impl/vector/XKnnFloatVectorField.java | 68 ++
.../creator/impl/vector/lucene95/HnswCodec.java | 139 ++++
.../impl/vector/lucene95/HnswVectorsFormat.java | 91 +++
.../converter/SegmentV1V2ToV3FormatConverter.java | 45 +-
.../segment/index/datasource/BaseDataSource.java | 7 +
.../index/dictionary/DictionaryIndexType.java | 55 +-
.../index/dictionary/DictionaryInternerHolder.java | 58 ++
.../index/forward/ForwardIndexCreatorFactory.java | 10 +-
.../index/forward/ForwardIndexReaderFactory.java | 11 +-
.../segment/index/forward/ForwardIndexType.java | 78 +-
.../local/segment/index/fst/FstIndexType.java | 5 +-
.../segment/index/loader/ForwardIndexHandler.java | 284 ++++---
.../segment/index/loader/IndexLoadingConfig.java | 44 +-
.../local/segment/index/loader/LoaderUtils.java | 2 +-
.../ColumnMinMaxValueGenerator.java | 4 +-
.../defaultcolumn/BaseDefaultColumnHandler.java | 39 +-
.../loader/invertedindex/FSTIndexHandler.java | 4 +-
.../loader/invertedindex/VectorIndexHandler.java | 215 +++++
.../index/nullvalue/NullValueIndexType.java | 41 +-
.../index/readers/LuceneFSTIndexReader.java | 3 +-
.../index/readers/OnHeapStringDictionary.java | 17 +-
.../segment/index/readers/StringDictionary.java | 32 +
.../FixedBitMVEntryDictForwardIndexReader.java | 155 ++++
.../readers/json/ImmutableJsonIndexReader.java | 1 +
.../index/readers/text/LuceneTextIndexReader.java | 25 +-
.../index/readers/vector/HnswDocIdCollector.java | 78 ++
.../HnswVectorIndexReader.java} | 97 +--
.../segment/index/text/TextIndexConfigBuilder.java | 3 +
.../local/segment/index/text/TextIndexType.java | 6 +-
.../segment/index/vector/VectorIndexPlugin.java | 17 +-
.../segment/index/vector/VectorIndexType.java | 174 ++++
.../readers/CompactedPinotSegmentRecordReader.java | 111 +++
.../segment/readers/PinotSegmentColumnReader.java | 38 +-
.../local/segment/store/FilePerIndexDirectory.java | 2 +
.../segment/store/SegmentLocalFSDirectory.java | 10 +-
.../segment/store/SingleFileIndexDirectory.java | 11 +-
.../local/segment/store/StarTreeIndexReader.java | 34 +-
.../local/segment/store/TextIndexUtils.java | 35 +-
.../local/segment/store/VectorIndexUtils.java | 98 +++
.../local/startree/StarTreeBuilderUtils.java | 4 +-
.../startree/v2/builder/BaseSingleTreeBuilder.java | 2 +-
.../startree/v2/builder/MultipleTreesBuilder.java | 17 +-
.../v2/builder/OffHeapSingleTreeBuilder.java | 2 +-
.../v2/builder/OnHeapSingleTreeBuilder.java | 2 +-
.../v2/builder/StarTreeIndexSeparator.java | 7 +-
.../v2/builder/StarTreeV2BuilderConfig.java | 14 +-
.../startree/v2/store/StarTreeIndexMapUtils.java | 34 +-
.../upsert/BasePartitionUpsertMetadataManager.java | 157 ++--
.../upsert/BaseTableUpsertMetadataManager.java | 133 ++--
...oncurrentMapPartitionUpsertMetadataManager.java | 78 +-
.../ConcurrentMapTableUpsertMetadataManager.java | 4 +-
.../local/upsert/TableUpsertMetadataManager.java | 6 +-
.../upsert/TableUpsertMetadataManagerFactory.java | 29 +-
.../pinot/segment/local/upsert/UpsertContext.java | 197 +++++
.../local/utils/ConsistentDataPushUtils.java | 2 +-
.../segment/local/utils/CustomSerDeUtils.java | 9 +-
.../pinot/segment/local/utils/SegmentLocks.java | 31 +-
.../segment/local/utils/SegmentPushUtils.java | 2 +-
.../segment/local/utils/TableConfigUtils.java | 164 ++--
.../pinot/segment/local/utils/fst/FSTBuilder.java | 19 +-
.../segment/local/utils/fst/RegexpMatcher.java | 11 +-
.../utils/nativefst/NativeFSTIndexCreator.java | 2 +-
...istinctCountThetaSketchValueAggregatorTest.java | 13 +
.../customobject/ThetaSketchAccumulatorTest.java | 104 +++
.../mutable/MutableSegmentImplTestUtils.java | 6 +-
.../MutableSegmentImplUpsertComparisonColTest.java | 110 ++-
.../mutable/MutableSegmentImplUpsertTest.java | 29 +-
.../invertedindex/LuceneMutableTextIndexTest.java | 5 +-
.../NativeAndLuceneMutableTextIndexTest.java | 7 +-
.../recordtransformer/RecordTransformerTest.java | 127 ++-
.../impl/SegmentColumnarIndexCreatorTest.java | 13 +-
.../segment/index/SegmentMetadataImplTest.java | 3 +-
.../index/creator/LuceneFSTIndexCreatorTest.java | 4 +-
.../index/creator/NativeFSTIndexCreatorTest.java | 4 +-
.../index/dictionary/DictionaryIndexTypeTest.java | 70 +-
.../FixedBitMVEntryDictForwardIndexTest.java | 123 +++
.../index/forward/ForwardIndexTypeTest.java | 56 +-
.../index/loader/ForwardIndexHandlerTest.java | 160 +++-
.../local/segment/index/loader/LoaderTest.java | 208 ++++-
.../index/loader/SegmentPreProcessorTest.java | 45 +-
.../index/nullvalue/NullValueIndexTypeTest.java | 66 ++
.../index/readers/ImmutableDictionaryTest.java | 2 +-
.../ImmutableDictionaryTypeConversionTest.java | 25 +-
.../text/LuceneTextIndexCompatibleTest.java | 32 +-
.../CompactedPinotSegmentRecordReaderTest.java | 131 +++
.../segment/store/FilePerIndexDirectoryTest.java | 24 +-
.../store/SingleFileIndexDirectoryTest.java | 26 +-
.../segment/store/StarTreeIndexReaderTest.java | 3 +-
.../v2/builder/StarTreeIndexSeparatorTest.java | 5 +-
.../v2/builder/StarTreeV2BuilderConfigTest.java | 57 +-
.../BasePartitionUpsertMetadataManagerTest.java | 174 ++++
...rrentMapPartitionUpsertMetadataManagerTest.java | 252 ++++--
...oncurrentMapTableUpsertMetadataManagerTest.java | 27 +-
.../segment/local/utils/TableConfigUtilsTest.java | 321 +++++++-
.../segment/local/utils/fst/FSTBuilderTest.java | 4 +-
.../data/lucene_80_index/Text.lucene.index/_0.cfe | Bin 0 -> 299 bytes
.../data/lucene_80_index/Text.lucene.index/_0.cfs | Bin 0 -> 136123 bytes
.../data/lucene_80_index/Text.lucene.index/_0.si | Bin 0 -> 370 bytes
.../lucene_80_index/Text.lucene.index/segments_1 | Bin 0 -> 137 bytes
.../lucene_80_index/Text.lucene.index/write.lock | 0
.../data/test_upsert_comparison_col_schema.json | 4 +
.../src/test/resources/data/test_vector_data.avro | Bin 0 -> 6219422 bytes
.../org/apache/pinot/segment/spi/V1Constants.java | 7 +-
.../spi/compression/DictIdCompressionType.java | 35 +-
.../pinot/segment/spi/creator/SegmentCreator.java | 2 +-
.../spi/creator/SegmentGeneratorConfig.java | 2 +-
.../pinot/segment/spi/datasource/DataSource.java | 7 +
.../segment/spi/index/DictionaryIndexConfig.java | 37 +-
.../segment/spi/index/ForwardIndexConfig.java | 81 +-
.../pinot/segment/spi/index/StandardIndexes.java | 9 +
.../pinot/segment/spi/index/TextIndexConfig.java | 31 +-
.../spi/index/creator/VectorIndexConfig.java | 126 +++
.../spi/index/creator/VectorIndexCreator.java | 33 +-
.../spi/index/metadata/ColumnMetadataImpl.java | 4 +-
.../spi/index/metadata/SegmentMetadataImpl.java | 12 +-
.../pinot/segment/spi/index/reader/Dictionary.java | 24 +
.../spi/index/reader/ForwardIndexReader.java | 11 +-
.../spi/index/reader/VectorIndexReader.java | 20 +-
.../startree/AggregationFunctionColumnPair.java | 42 +
.../spi/index/startree/AggregationSpec.java | 8 +
.../spi/index/startree/StarTreeV2Metadata.java | 16 +-
.../pinot/segment/spi/memory/PinotByteBuffer.java | 6 +-
.../BoundedColumnValuePartitionFunction.java | 4 +-
.../spi/partition/ByteArrayPartitionFunction.java | 4 +-
.../spi/partition/HashCodePartitionFunction.java | 4 +-
.../spi/partition/ModuloPartitionFunction.java | 15 +-
.../spi/partition/Murmur3PartitionFunction.java | 289 +++++++
.../spi/partition/MurmurPartitionFunction.java | 6 +-
.../segment/spi/partition/PartitionFunction.java | 3 +-
.../spi/partition/PartitionFunctionFactory.java | 8 +-
.../pinot/segment/spi/store/SegmentDirectory.java | 2 +-
.../segment/spi/store/SegmentDirectoryPaths.java | 25 +-
.../segment/spi/utils/SegmentMetadataUtils.java | 17 +-
.../AggregationFunctionColumnPairTest.java | 59 ++
.../spi/index/startree/StarTreeV2MetadataTest.java | 130 +++
.../spi/partition/PartitionFunctionTest.java | 307 ++++++-
pinot-server/pom.xml | 14 -
.../pinot/server/api/AdminApiApplication.java | 10 +-
.../pinot/server/api/resources/TablesResource.java | 88 +-
.../server/starter/helix/BaseServerStarter.java | 5 +-
.../starter/helix/HelixInstanceDataManager.java | 23 +-
.../helix/HelixInstanceDataManagerConfig.java | 133 ++--
.../apache/pinot/server/api/BaseResourceTest.java | 65 +-
.../pinot/server/api/TablesResourceTest.java | 25 +-
pinot-spi/pom.xml | 18 +-
.../pinot/spi/annotations/ScalarFunction.java | 5 +
.../config/instance/InstanceDataManagerConfig.java | 8 +
.../spi/config/table/DimensionTableConfig.java | 11 +-
.../apache/pinot/spi/config/table/FieldConfig.java | 29 +-
.../pinot/spi/config/table/IndexingConfig.java | 28 +
.../org/apache/pinot/spi/config/table/Intern.java | 79 ++
.../pinot/spi/config/table/RoutingConfig.java | 10 +-
.../apache/pinot/spi/config/table/TableConfig.java | 11 +
.../pinot/spi/config/table/UpsertConfig.java | 27 +-
.../pinot/spi/config/task/AdhocTaskConfig.java | 2 +
.../apache/pinot/spi/data/DateTimeFormatSpec.java | 2 +-
.../java/org/apache/pinot/spi/data/FieldSpec.java | 52 +-
.../java/org/apache/pinot/spi/data/Schema.java | 67 +-
.../spi/data/readers/RecordReaderFileConfig.java | 51 +-
.../pinot/spi/env/CommonsConfigurationUtils.java | 184 +++--
.../pinot/spi/env/ConfigFilePropertyReader.java | 6 +-
.../spi/env/ConfigFilePropertyReaderFactory.java | 8 +-
.../apache/pinot/spi/env/PinotConfiguration.java | 30 +-
.../PinotBrokerQueryEventListenerFactory.java | 34 +
.../org/apache/pinot/spi/plugin/PluginManager.java | 2 +-
.../org/apache/pinot/spi/stream/StreamConfig.java | 7 +-
.../pinot/spi/stream/StreamMetadataProvider.java | 11 +-
.../pinot/spi/trace/DefaultRequestContext.java | 11 +
.../org/apache/pinot/spi/trace/RequestContext.java | 4 +
.../org/apache/pinot/spi/utils/BooleanUtils.java | 11 +
.../apache/pinot/spi/utils/CommonConstants.java | 20 +-
.../org/apache/pinot/spi/utils/FALFInterner.java | 148 ++++
.../java/org/apache/pinot/spi/utils/JsonUtils.java | 2 +-
.../java/org/apache/pinot/spi/utils/Pairs.java | 23 +-
.../utils/builder/ControllerRequestURLBuilder.java | 4 +
.../pinot/spi/config/table/IndexingConfigTest.java | 7 +-
.../pinot/spi/config/task/AdhocTaskConfigTest.java | 18 +
.../spi/env/CommonsConfigurationUtilsTest.java | 15 +-
.../pinot/spi/env/PinotConfigurationTest.java | 11 +-
pinot-tools/pom.xml | 4 +-
.../org/apache/pinot/tools/HybridQuickstart.java | 3 +-
.../apache/pinot/tools/PinotTableRebalancer.java | 4 +-
.../org/apache/pinot/tools/QuickStartBase.java | 462 ++++++++++-
.../java/org/apache/pinot/tools/Quickstart.java | 205 +----
.../org/apache/pinot/tools/RealtimeQuickStart.java | 5 +-
.../java/org/apache/pinot/tools/SpeedTest.java | 49 +-
.../admin/command/AbstractBaseAdminCommand.java | 13 +
.../pinot/tools/admin/command/AddTableCommand.java | 26 +-
.../admin/command/ChangeNumReplicasCommand.java | 2 +-
.../tools/admin/command/GenerateDataCommand.java | 88 +-
.../tools/admin/command/RebalanceTableCommand.java | 10 +-
.../tools/filesystem/PinotFSBenchmarkDriver.java | 9 +-
.../converter/DictionaryToRawIndexConverter.java | 13 +-
.../tools/streams/AvroFileSourceGenerator.java | 2 +-
.../org/apache/pinot/tools/utils/JarUtils.java | 47 +-
.../fineFoodReviews_offline_table_config.json | 25 +-
.../fineFoodReviews_realtime_table_config.json | 73 ++
.../fineFoodReviews/fineFoodReviews_schema.json | 48 ++
.../fine_food_reviews_with_embeddings_1k.json.gz | Bin 0 -> 8101775 bytes
pom.xml | 111 ++-
709 files changed, 27584 insertions(+), 8841 deletions(-)
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/request/context/predicate/VectorSimilarityPredicate.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ValidDocIdMetadataInfo.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/utils/ScalingThreadPoolExecutor.java
delete mode 100644
pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/CLPDecodeRewriter.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/ClpRewriter.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/utils/FALFInternerTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/utils/ScalingThreadPoolExecutorTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/utils/TlsUtilsTest.java
delete mode 100644
pinot-common/src/test/java/org/apache/pinot/sql/parsers/rewriter/CLPDecodeRewriterTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/sql/parsers/rewriter/ClpRewriterTest.java
create mode 100644 pinot-common/src/test/resources/tls/keystore-updated.p12
create mode 100644 pinot-common/src/test/resources/tls/keystore.p12
create mode 100644 pinot-common/src/test/resources/tls/truststore-updated.p12
create mode 100644 pinot-common/src/test/resources/tls/truststore.p12
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/DataGenerationHelpers.java
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/DateTimeGenerator.java
rename
pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/{generator
=> writer}/AvroWriter.java (59%)
copy
pinot-common/src/main/java/org/apache/pinot/common/metrics/MinionMetrics.java
=>
pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/writer/AvroWriterSpec.java
(50%)
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/writer/CsvWriter.java
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/writer/FileWriter.java
copy
pinot-common/src/main/java/org/apache/pinot/common/metrics/MinionTimer.java =>
pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/writer/FileWriterSpec.java
(55%)
copy
pinot-common/src/main/java/org/apache/pinot/common/metrics/MinionTimer.java =>
pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/writer/JsonWriter.java
(54%)
copy pinot-common/src/main/java/org/apache/pinot/sql/FilterKind.java =>
pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/writer/Writer.java
(51%)
copy
pinot-query-planner/src/main/java/org/apache/pinot/query/planner/PlanFragmentMetadata.java
=>
pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/writer/WriterSpec.java
(69%)
create mode 100644
pinot-controller/src/main/resources/app/components/AsyncInstanceTable.tsx
create mode 100644
pinot-controller/src/main/resources/app/components/AsyncPinotSchemas.tsx
create mode 100644
pinot-controller/src/main/resources/app/components/AsyncPinotTables.tsx
copy
pinot-query-planner/src/main/java/org/apache/pinot/query/planner/PlanFragmentMetadata.java
=> pinot-controller/src/main/resources/app/components/Loading.tsx (79%)
create mode 100644
pinot-controller/src/main/resources/app/components/NotFound.tsx
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/operator/filter/VectorSimilarityFilterOperator.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ClpEncodedVarsMatchTransformFunction.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TextMatchFilterOptimizer.java
copy
pinot-query-planner/src/main/java/org/apache/pinot/query/planner/PlanFragmentMetadata.java
=>
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/AdaptiveConstraintsWriter.java
(70%)
copy
pinot-common/src/main/java/org/apache/pinot/common/metrics/MinionTimer.java =>
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/AdaptiveSizeBasedWriter.java
(52%)
copy
pinot-query-planner/src/main/java/org/apache/pinot/query/planner/PlanFragmentMetadata.java
=>
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/FileWriter.java
(75%)
rename
pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/{CLPDecodeTransformFunctionTest.java
=> ClpTransformFunctionsTest.java} (69%)
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/startree/v2/DistinctCountRawHLLStarTreeV2Test.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/queries/JsonMalformedIndexTest.java
delete mode 100644 pinot-core/src/test/resources/data/dimBaseballTeams.avro
delete mode 100644 pinot-core/src/test/resources/data/dimBaseballTeams.avsc
copy
pinot-tools/src/main/resources/examples/batch/dimBaseballTeams/dimBaseballTeams_offline_table_config.json
=> pinot-core/src/test/resources/data/dimBaseballTeams_config.json (100%)
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/RealtimeConsumptionRateLimiterClusterIntegrationTest.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/SegmentGenerationMinionRealtimeIngestionTest.java
delete mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/UpsertCompactionMinionClusterIntegrationTest.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/models/DummyTableUpsertMetadataManager.java
create mode 100644
pinot-integration-tests/src/test/resources/gameScores_large_csv.tar.gz
delete mode 100644
pinot-integration-tests/src/test/resources/upsert_compaction_test.tar.gz
create mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/src/main/java/org/apache/pinot/plugin/stream/kafka20/KafkaSSLUtils.java
create mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/src/test/java/org/apache/pinot/plugin/stream/kafka20/KafkaSSLUtilsTest.java
create mode 100644
pinot-query-planner/src/main/java/org/apache/calcite/rel/rules/PinotRelDistributionTraitRule.java
rename pinot-query-planner/src/main/java/org/apache/pinot/query/planner/{ =>
explain}/PhysicalExplainPlanVisitor.java (73%)
rename pinot-query-planner/src/main/java/org/apache/pinot/query/planner/{ =>
physical}/DispatchablePlanFragment.java (97%)
rename pinot-query-planner/src/main/java/org/apache/pinot/query/planner/{ =>
physical}/DispatchableSubPlan.java (98%)
create mode 100644
pinot-query-planner/src/test/resources/queries/ExplainPhysicalPlans.json
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/customobject/ThetaSketchAccumulator.java
delete mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/data/manager/TableDataManagerConfig.java
delete mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/data/manager/TableDataManagerParams.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/FixedBitMVEntryDictForwardIndexWriter.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/vector/MutableVectorIndex.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/SpecialValueTransformer.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/ColumnJsonParserException.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueEntryDictForwardIndexCreator.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/vector/HnswVectorIndexCreator.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/vector/XKnnFloatVectorField.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/vector/lucene95/HnswCodec.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/vector/lucene95/HnswVectorsFormat.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/dictionary/DictionaryInternerHolder.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/VectorIndexHandler.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitMVEntryDictForwardIndexReader.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/vector/HnswDocIdCollector.java
copy
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/{text/LuceneTextIndexReader.java
=> vector/HnswVectorIndexReader.java} (66%)
copy
pinot-spi/src/main/java/org/apache/pinot/spi/env/ConfigFilePropertyReader.java
=>
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/vector/VectorIndexPlugin.java
(67%)
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/vector/VectorIndexType.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/CompactedPinotSegmentRecordReader.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/VectorIndexUtils.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/UpsertContext.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/customobject/ThetaSketchAccumulatorTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/FixedBitMVEntryDictForwardIndexTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/nullvalue/NullValueIndexTypeTest.java
copy
pinot-common/src/main/java/org/apache/pinot/common/metrics/MinionTimer.java =>
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexCompatibleTest.java
(58%)
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/readers/CompactedPinotSegmentRecordReaderTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManagerTest.java
create mode 100644
pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.cfe
create mode 100644
pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.cfs
create mode 100644
pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.si
create mode 100644
pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/segments_1
copy .trivyignore =>
pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/write.lock
(100%)
create mode 100644
pinot-segment-local/src/test/resources/data/test_vector_data.avro
copy
pinot-common/src/main/java/org/apache/pinot/common/metrics/MinionTimer.java =>
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/compression/DictIdCompressionType.java
(51%)
create mode 100644
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/VectorIndexConfig.java
copy
pinot-common/src/main/java/org/apache/pinot/common/metrics/MinionTimer.java =>
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/VectorIndexCreator.java
(56%)
rename
pinot-query-planner/src/main/java/org/apache/pinot/query/planner/PlanFragmentMetadata.java
=>
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/VectorIndexReader.java
(60%)
create mode 100644
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/Murmur3PartitionFunction.java
create mode 100644
pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/index/startree/AggregationFunctionColumnPairTest.java
create mode 100644
pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/index/startree/StarTreeV2MetadataTest.java
create mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/config/table/Intern.java
create mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/utils/FALFInterner.java
create mode 100644
pinot-tools/src/main/resources/examples/stream/fineFoodReviews/fineFoodReviews_realtime_table_config.json
create mode 100644
pinot-tools/src/main/resources/examples/stream/fineFoodReviews/fineFoodReviews_schema.json
create mode 100644
pinot-tools/src/main/resources/examples/stream/fineFoodReviews/rawdata/fine_food_reviews_with_embeddings_1k.json.gz
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
