This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch
dependabot/maven/kotlin.stdlib.version-2.1.0
in repository https://gitbox.apache.org/repos/asf/pinot.git
discard fe121f614b Bump kotlin.stdlib.version from 2.0.21 to 2.1.0
add b9171ddb25 Cleanup long offset in completion protocol (#14557)
add 3a616d8ce5 Cleanup non split commit code (#14559)
add 585e33338e Add immtuable CLPForwardIndex creator and related classes
(#14288)
add 76e1506336 Specify start index of file name in GenerateDataCommand
(#14573)
add 8f262e074d Bump software.amazon.awssdk:bom from 2.29.23 to 2.29.24
(#14577)
add ca83cf1aa4 Bump nl.jqno.equalsverifier:equalsverifier from 3.17.3 to
3.17.4 (#14576)
add 2910caac80 Bump com.puppycrawl.tools:checkstyle from 10.20.1 to
10.20.2 (#14575)
add 0ccc6471e0 Exclude secondary workload queries from some critical
metrics (#14553)
add 76fa5fd290 Bump org.asynchttpclient:async-http-client from 3.0.0 to
3.0.1 (#14579)
add af6fd6c28a [Flaky-test] Fix
ForwardIndexHandlerTest.testAddOtherIndexForForwardIndexDisabledColumn()
(#14580)
add d0d4419488 [SparkConnector] Add proper time column support for Spark
connector segment writer (#14556)
add e5d5bad371 Fix crc mismatch during deepstore upload retry task (#14506)
add 4133c3ca7d Bump org.checkerframework:checker-qual from 3.48.2 to
3.48.3 (#14587)
add 7977c28278 Bump software.amazon.awssdk:bom from 2.29.24 to 2.29.25
(#14585)
add 71dfa7d575 Make RecordExtractor preserve empty array/map and map
entries with empty value (#14547)
add e1de8d31b0 Bump parquet.version from 1.14.4 to 1.15.0 (#14584)
add ed1de92e31 Extend the merge rollup task capabilities (#14355)
add b65497ddb7 allow to prefetch for V2 engine as done for V1 engine
(#14589)
add 4876fb754d Add a Server API to list segments that need to be refreshed
for a table (#14451)
add 8d11cd9ec7 Bump software.amazon.awssdk:bom from 2.29.25 to 2.29.27
(#14597)
add 31933064bf [timeseries] Part-1: Misc. Refactor to Enable Broker Reduce
(#14582)
add df053536c2 Clean up and enhance functions in
SchemaConformingTransformer (#14546)
add 84ca2fc5c4 bugfix: use PropertiesWriter to escape index_map keys
properly (#12018)
add 8dadfd2f32 Add test to check data and index types compatibility
(#14493)
add a4f6df415e Fix the query option override for leaf stage (#14603)
add 64a952f2d2 Bump software.amazon.awssdk:bom from 2.29.27 to 2.29.29
(#14604)
add 95b075b527 Batch reload api to specify what segments to be reloaded on
what servers to be more flexible (#14544)
add 5104565882 Bump com.azure:azure-sdk-bom from 1.2.29 to 1.2.30 (#14605)
add 0e1c12ff46 Marks end criteria reached for the segment if the Index
cannot consume more rows (#14479)
add 1c98d43b07 [timeseries] Part-2: Enable Streaming Response for Time
Series (#14598)
add 3cb3c54630 RefreshSegmentTask - Support for scheduling adhoc tasks
(#14596)
add 70c9898315 Fix integer overflow in GroupByUtils (#14610)
add dafb4f0180 Include column name for v4 raw fwd index build failure
(#14613)
add a506df38a5 Safeguard AbstractBuilder copy constructor from empty
TextIndexConfig (#14616)
add 694546b3b2 Make RecordEnricher extend RecordTransformer (#14601)
add e8dcba123a Fix query option validation for group-by queries (#14618)
add 1b87488aea Strip Matrix parameter from BasePath check (#14383)
add f9655b9c04 Allow configurable initial capacity for IndexedTable
(#14620)
add ed7c9eeacc Bump nl.jqno.equalsverifier:equalsverifier from 3.17.4 to
3.17.5 (#14622)
add 960a1c3d01 [Doc] User API: Add hint for options available for
'components' field (#14593)
add 9f2a727722 Making mutable Index class pluggable. (#14609)
add 442c0fc713 Configurable sketch accuracy in merge rollup task (#14373)
add 1641a6d4bf Bump software.amazon.awssdk:bom from 2.29.29 to 2.29.30
(#14626)
add 1b5858b5b4 [timeseries] Part-3: Add Time Series Exchange Operator,
Plan Node and Serde (#14611)
add b4066fbd40 Fix flaky quota integration tests (#14602)
add 57d1f770da Bump software.amazon.awssdk:bom from 2.29.30 to 2.29.31
(#14637)
add e33ad86d1f Bump org.jline:jline from 3.27.1 to 3.28.0 (#14635)
add dfcc489311 Bump io.grpc:grpc-bom from 1.68.2 to 1.69.0 (#14636)
add 740c110bd3 include reason why TableSegmentValidationInfo is invalid
(#14612)
add 167fb679db Remove info log in DataBlockUtils as it can flood the log
(#14644)
add 7965055573 Fixes RealtimeQuickStartWithMinion (#14642)
add 626c45d717 Upsert small segment merger task in minions (#14477)
add eec29b89c3 Bump software.amazon.awssdk:bom from 2.29.31 to 2.29.32
(#14647)
add 35848996ab Expose QueryScheduler from ServerInstance & Custom
interface (#14630)
add 90b437f06f Small memory leak fix (#14641)
add 607bed279d make AcquireReleaseColumnsSegmentOperator work for V2
(#14649)
add d7505f6986 Add URL scalar functions (#14646)
add 89b0d01b2c Add query request ID to log lines for slow / large queries
(#14627)
add 2a9cd82b98 Make segment completion FSM pluggable (#14088)
add 306be4214c [FEATURE] Add getTopics API to StreamMetadataProvider for
Topic Enumeration in Streaming Connectors (#14654)
add e645a50210 Improve the unit test dataset used by
CLPMutableForwardIndexV2Test and CLPForwardIndexCreatorV2Test (#14632)
add 264d09cf55 Bump com.puppycrawl.tools:checkstyle from 10.20.2 to
10.21.0 (#14658)
add 5e36800dbc Bump software.amazon.awssdk:bom from 2.29.32 to 2.29.33
(#14657)
add 3677671844 Support is_leaf_return_final_result agg option (#14645)
add f4129876f1 Add config to preserve stream topic name as a column for
`CLPLogRecordExtractor` (#14599)
add fda93ab083 Spark batch ingestion common code abstraction. (#14415)
add e89ac9490e Override return type for NOW to TIMESTAMP in the v2 engine
(#14614)
add cecddf17d7 Bump log4j.version from 2.24.2 to 2.24.3 (#14665)
add caa3ce52da Bump software.amazon.awssdk:bom from 2.29.33 to 2.29.34
(#14666)
add 0e6b9a979a Bump org.apache.commons:commons-text from 1.12.0 to 1.13.0
(#14667)
add 2701f6c954 Cleanup Redundant JMX Exporter Rules (#14348)
add 4c824aa699 fix: Add null check in query and storage component (#14663)
add 4b26bf6249 Increase timeout for kafka client connection for metadata
fetch (#14638)
add 9d12d38e9a Bump software.amazon.awssdk:bom from 2.29.34 to 2.29.35
(#14674)
add b64bd81de1 [timeseries] Part-3.1: Add Support for Partial Aggregate
and Complex Intermediate Type (#14631)
add 65622006a4 Bump kotlin.stdlib.version from 2.0.21 to 2.1.0
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (fe121f614b)
\
N -- N -- N
refs/heads/dependabot/maven/kotlin.stdlib.version-2.1.0 (65622006a4)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.../jmx_prometheus_javaagent/configs/broker.yml | 233 +----
.../configs/controller.yml | 280 +-----
.../jmx_prometheus_javaagent/configs/minion.yml | 52 +-
.../jmx_prometheus_javaagent/configs/server.yml | 246 +-----
.../pinot/broker/broker/AuthenticationFilter.java | 5 +-
.../BaseSingleStageBrokerRequestHandler.java | 28 +-
.../SingleConnectionBrokerRequestHandler.java | 11 +-
.../pinot/common/auth/AuthProviderUtils.java | 13 +
.../pinot/common/datablock/DataBlockUtils.java | 21 +-
.../pinot/common/function/scalar/UrlFunctions.java | 743 ++++++++++++++++
.../apache/pinot/common/metrics/BrokerMeter.java | 4 +
.../apache/pinot/common/metrics/BrokerTimer.java | 4 +-
.../protocols/SegmentCompletionProtocol.java | 33 +-
.../response/PinotBrokerTimeSeriesResponse.java | 2 +-
...nfo.java => TableLLCSegmentUploadResponse.java} | 31 +-
.../resources/TableSegmentValidationInfo.java | 9 +
.../common/utils/FileUploadDownloadClient.java | 27 +
.../org/apache/pinot/common/utils/HashUtil.java | 10 +-
.../apache/pinot/common/utils/SegmentUtils.java | 2 +-
.../common/utils/config/QueryOptionsUtils.java | 160 ++--
pinot-common/src/main/proto/plan.proto | 1 +
pinot-common/src/main/proto/worker.proto | 5 +-
.../common/function/scalar/UrlFunctionsTest.java | 530 ++++++++++++
.../prometheus/BrokerPrometheusMetricsTest.java | 4 +-
.../ControllerPrometheusMetricsTest.java | 3 +-
.../prometheus/MinionPrometheusMetricsTest.java | 14 +-
.../prometheus/PinotPrometheusMetricsTest.java | 7 +-
.../prometheus/ServerPrometheusMetricsTest.java | 13 +-
.../protocols/SegmentCompletionProtocolTest.java | 6 +-
.../resources/TableSegmentValidationInfoTest.java | 50 ++
.../common/utils/config/QueryOptionsUtilsTest.java | 137 +--
.../connector/flink/sink/FlinkSegmentWriter.java | 7 +-
.../spark/v3/datasource/PinotDataWriter.scala | 38 +-
.../connector/spark/v3/datasource/PinotWrite.scala | 3 +-
.../v3/datasource/SparkToPinotTypeTranslator.scala | 22 +-
.../ExampleSparkPinotConnectorWriteTest.scala | 2 +
.../spark/v3/datasource/PinotDataWriterTest.scala | 60 +-
.../spark/v3/datasource/PinotWriteTest.scala | 19 +-
.../SparkToPinotTypeTranslatorTest.scala | 20 +-
.../spark/common/PinotDataSourceWriteOptions.scala | 16 +
.../common/PinotDataSourceWriteOptionsTest.scala | 8 +-
.../pinot/controller/BaseControllerStarter.java | 6 +-
.../api/access/AuthenticationFilter.java | 4 +-
.../resources/LLCSegmentCompletionHandlers.java | 277 ++----
.../PinotAccessControlUserRestletResource.java | 10 +-
.../api/resources/PinotSegmentRestletResource.java | 154 +++-
.../api/resources/PinotTableRestletResource.java | 4 +-
.../api/resources/TableStaleSegmentResponse.java | 67 ++
.../helix/core/PinotHelixResourceManager.java | 121 ++-
.../realtime/BlockingSegmentCompletionFSM.java | 902 ++++++++++++++++++++
.../realtime/PinotLLCRealtimeSegmentManager.java | 48 +-
.../core/realtime/SegmentCompletionConfig.java | 58 ++
.../helix/core/realtime/SegmentCompletionFSM.java | 133 +++
.../core/realtime/SegmentCompletionFSMFactory.java | 123 +++
.../core/realtime/SegmentCompletionManager.java | 937 +--------------------
.../recommender/data/DataGenerationHelpers.java | 38 +-
.../recommender/data/writer/AvroWriter.java | 3 +-
.../recommender/data/writer/AvroWriterSpec.java | 4 +-
.../recommender/data/writer/FileWriter.java | 2 +-
.../recommender/data/writer/FileWriterSpec.java | 5 +-
.../recommender/data/writer/WriterSpec.java | 8 +-
.../util/ServerSegmentMetadataReader.java | 35 +
.../pinot/controller/util/TableMetadataReader.java | 14 +
.../Homepage/Operations/AddQueryComponent.tsx | 4 +-
.../Homepage/Operations/AddStorageComponent.tsx | 2 +-
.../api/access/AuthenticationFilterTest.java | 22 +-
.../resources/PinotSegmentRestletResourceTest.java | 88 ++
.../PinotLLCRealtimeSegmentManagerTest.java | 129 +++
.../helix/core/realtime/SegmentCompletionTest.java | 223 ++---
.../PerQueryCPUMemAccountantFactory.java | 2 +-
.../apache/pinot/core/common/MinionConstants.java | 59 ++
.../core/data/manager/BaseTableDataManager.java | 288 +++++++
.../realtime/RealtimeSegmentDataManager.java | 29 +-
.../core/data/table/ConcurrentIndexedTable.java | 10 +-
.../apache/pinot/core/data/table/IndexedTable.java | 36 +-
.../pinot/core/data/table/SimpleIndexedTable.java | 9 +-
.../table/UnboundedConcurrentIndexedTable.java | 8 +-
.../AcquireReleaseColumnsSegmentOperator.java | 8 +-
.../blocks/results/GroupByResultsBlock.java | 10 +
.../operator/combine/GroupByCombineOperator.java | 36 +-
.../streaming/StreamingGroupByCombineOperator.java | 36 +-
.../core/plan/maker/InstancePlanMakerImplV2.java | 92 +-
.../function/SumPrecisionAggregationFunction.java | 1 +
.../groupby/AggregationGroupByResult.java | 4 +
.../pinot/core/query/reduce/BaseReduceService.java | 4 +
.../core/query/reduce/BrokerReduceService.java | 7 +-
.../core/query/reduce/DataTableReducerContext.java | 8 +-
.../core/query/reduce/GroupByDataTableReducer.java | 41 +-
.../core/query/reduce/StreamingReduceService.java | 7 +-
.../core/query/request/context/QueryContext.java | 10 +
.../DistinctCountCPCSketchAggregator.java | 3 +-
.../aggregator/DistinctCountHLLAggregator.java | 3 +-
.../DistinctCountThetaSketchAggregator.java | 24 +-
.../aggregator/DistinctCountULLAggregator.java | 3 +-
.../aggregator/IntegerTupleSketchAggregator.java | 21 +-
.../processing/aggregator/MaxValueAggregator.java | 3 +-
.../processing/aggregator/MinValueAggregator.java | 3 +-
.../processing/aggregator/SumValueAggregator.java | 3 +-
.../processing/aggregator/ValueAggregator.java | 5 +-
.../framework/SegmentProcessorConfig.java | 56 +-
.../framework/SegmentProcessorFramework.java | 17 +-
.../segment/processing/mapper/SegmentMapper.java | 7 +-
.../segment/processing/reducer/ReducerFactory.java | 3 +-
.../segment/processing/reducer/RollupReducer.java | 20 +-
.../core/transport/InstanceRequestHandler.java | 18 +-
.../org/apache/pinot/core/util/GroupByUtils.java | 164 +++-
.../accounting/ResourceManagerAccountingTest.java | 4 +-
.../BaseTableDataManagerNeedRefreshTest.java | 730 ++++++++++++++++
.../data/manager/BaseTableDataManagerTest.java | 2 +-
.../pinot/core/data/manager/TableIndexingTest.java | 711 ++++++++++++++++
.../realtime/RealtimeSegmentDataManagerTest.java | 23 +
.../pinot/core/data/table/IndexedTableTest.java | 79 +-
.../TimeSeriesAggregationOperatorTest.java | 2 +-
.../aggregation/function/ArrayAggFunctionTest.java | 374 ++++----
.../core/query/executor/QueryExecutorTest.java | 14 +-
.../apache/pinot/core/util/GroupByUtilsTest.java | 74 ++
.../org/apache/pinot/queries/ExprMinMaxTest.java | 5 +-
...terSegmentAggregationMultiValueQueriesTest.java | 8 +-
...SegmentAggregationMultiValueRawQueriesTest.java | 8 +-
...erSegmentAggregationSingleValueQueriesTest.java | 8 +-
.../InterSegmentGroupByMultiValueQueriesTest.java | 10 +-
...nterSegmentGroupByMultiValueRawQueriesTest.java | 10 +-
.../InterSegmentGroupBySingleValueQueriesTest.java | 10 +-
.../pinot/queries/WithOptionQueriesTest.java | 190 -----
.../src/test/resources/TableIndexingTest.csv | 485 +++++++++++
.../tests/MultiStageEngineIntegrationTest.java | 18 +-
.../tests/OfflineClusterIntegrationTest.java | 18 +-
.../tests/OfflineGRPCServerIntegrationTest.java | 2 +-
.../tests/QueryQuotaClusterIntegrationTest.java | 53 +-
.../tests/StaleSegmentCheckIntegrationTest.java | 201 +++++
.../apache/pinot/perf/BenchmarkCombineGroupBy.java | 6 +-
.../apache/pinot/perf/BenchmarkIndexedTable.java | 6 +-
.../pinot-batch-ingestion-spark-2.4/pom.xml | 2 +-
.../spark/SparkSegmentMetadataPushJobRunner.java | 112 +--
.../batch/spark/SparkSegmentTarPushJobRunner.java | 111 +--
.../batch/spark/SparkSegmentUriPushJobRunner.java | 111 +--
.../pinot-batch-ingestion-spark-3/pom.xml | 2 +-
.../spark3/SparkSegmentMetadataPushJobRunner.java | 1 -
.../batch/spark3/SparkSegmentTarPushJobRunner.java | 111 +--
.../batch/spark3/SparkSegmentUriPushJobRunner.java | 111 +--
.../{ => pinot-batch-ingestion-spark-base}/pom.xml | 35 +-
.../BaseSparkSegmentMetadataPushJobRunner.java} | 50 +-
.../common/BaseSparkSegmentTarPushJobRunner.java} | 49 +-
.../common/BaseSparkSegmentUriPushJobRunner.java} | 52 +-
pinot-plugins/pinot-batch-ingestion/pom.xml | 1 +
.../inputformat/avro/AvroRecordExtractor.java | 24 +-
.../avro/AvroRecordExtractorConfig.java | 10 -
.../avro/AvroRecordToPinotRowGeneratorTest.java | 151 ++--
.../inputformat/clplog/CLPLogRecordExtractor.java | 13 +-
.../clplog/CLPLogRecordExtractorConfig.java | 15 +
.../clplog/CLPLogRecordExtractorTest.java | 21 +-
.../plugin/inputformat/csv/CSVRecordExtractor.java | 18 +-
.../inputformat/json/JSONRecordExtractor.java | 6 +-
.../parquet/ParquetNativeRecordExtractor.java | 105 +--
.../parquet/ParquetRecordReaderTest.java | 7 +-
.../protobuf/ProtoBufRecordExtractor.java | 91 +-
.../inputformat/thrift/ThriftRecordExtractor.java | 25 +-
.../pinot/plugin/minion/tasks/MergeTaskUtils.java | 22 +
.../mergerollup/DimensionValueTransformer.java | 74 ++
.../tasks/mergerollup/MergeRollupTaskExecutor.java | 16 +-
.../tasks/mergerollup/MergeRollupTaskUtils.java | 36 +-
.../RealtimeToOfflineSegmentsTaskExecutor.java | 4 +
.../RefreshSegmentTaskGenerator.java | 137 +--
.../UpsertCompactMergeTaskExecutor.java | 203 +++++
.../UpsertCompactMergeTaskExecutorFactory.java | 52 ++
.../UpsertCompactMergeTaskGenerator.java | 425 ++++++++++
...ertCompactMergeTaskProgressObserverFactory.java | 25 +-
.../plugin/minion/tasks/MergeTaskUtilsTest.java | 30 +-
.../mergerollup/DimensionValueTransformerTest.java | 130 +++
.../mergerollup/MergeRollupTaskExecutorTest.java | 26 +
.../mergerollup/MergeRollupTaskUtilsTest.java | 64 +-
.../UpsertCompactMergeTaskExecutorTest.java | 95 +++
.../UpsertCompactMergeTaskGeneratorTest.java | 254 ++++++
.../filebased/FileBasedSegmentWriter.java | 3 +-
.../kafka20/KafkaStreamMetadataProvider.java | 26 +
.../kafka30/KafkaStreamMetadataProvider.java | 25 +
.../kafka30/KafkaPartitionLevelConsumerTest.java | 28 +
.../pinot/tsdb/m3ql/M3TimeSeriesPlanner.java | 5 +-
.../tsdb/m3ql/operator/KeepLastValueOperator.java | 2 +-
.../tsdb/m3ql/operator/TransformNullOperator.java | 2 +-
.../tsdb/m3ql/plan/KeepLastValuePlanNode.java | 11 +-
.../tsdb/m3ql/plan/TransformNullPlanNode.java | 15 +-
.../pinot/tsdb/m3ql/time/TimeBucketComputer.java | 2 +-
.../pinot/calcite/rel/hint/PinotHintOptions.java | 1 +
.../calcite/rel/hint/PinotHintStrategyTable.java | 99 ++-
.../calcite/rel/logical/PinotLogicalAggregate.java | 43 +-
.../PinotAggregateExchangeNodeInsertRule.java | 64 +-
.../pinot/calcite/sql/fun/PinotOperatorTable.java | 4 +-
.../query/planner/explain/PlanNodeMerger.java | 3 +
.../planner/logical/EquivalentStagesFinder.java | 3 +-
.../planner/logical/RelToPlanNodeConverter.java | 3 +-
.../query/planner/plannode/AggregateNode.java | 17 +-
.../query/planner/serde/PlanNodeDeserializer.java | 2 +-
.../query/planner/serde/PlanNodeSerializer.java | 92 +-
.../apache/pinot/query/QueryCompilationTest.java | 4 +-
.../pinot/query/QueryEnvironmentTestBase.java | 3 +-
.../src/test/resources/queries/AggregatePlans.json | 48 +-
.../test/resources/queries/BasicQueryPlans.json | 8 +-
.../src/test/resources/queries/GroupByPlans.json | 64 +-
.../src/test/resources/queries/JoinPlans.json | 68 +-
.../resources/queries/LiteralEvaluationPlans.json | 8 +-
.../src/test/resources/queries/OrderByPlans.json | 12 +-
.../test/resources/queries/PinotHintablePlans.json | 74 +-
.../src/test/resources/queries/SetOpPlans.json | 4 +-
.../resources/queries/WindowFunctionPlans.json | 108 +--
.../pinot/query/mailbox/GrpcSendingMailbox.java | 16 +-
.../apache/pinot/query/runtime/QueryRunner.java | 37 +-
.../query/runtime/operator/AggregateOperator.java | 4 +-
.../operator/MultistageGroupByExecutor.java | 95 ++-
.../plan/server/ServerPlanRequestUtils.java | 9 +-
.../plan/server/ServerPlanRequestVisitor.java | 3 +
...va => PhysicalTimeSeriesServerPlanVisitor.java} | 19 +-
.../TimeSeriesExchangeReceiveOperator.java | 182 ++++
.../TimeSeriesExchangeReceivePlanNode.java | 74 ++
.../timeseries/TimeSeriesExecutionContext.java | 10 +-
.../timeseries/TimeSeriesPhysicalTableScan.java | 8 +
.../timeseries/serde/TimeSeriesBlockSerde.java | 303 +++++++
.../query/service/dispatch/QueryDispatcher.java | 3 +-
.../pinot/query/service/server/QueryServer.java | 4 +-
.../runtime/operator/AggregateOperatorTest.java | 3 +-
.../runtime/operator/MultiStageAccountingTest.java | 2 +-
.../query/runtime/queries/QueryRunnerTest.java | 120 ++-
...> PhysicalTimeSeriesServerPlanVisitorTest.java} | 34 +-
.../TimeSeriesExchangeReceiveOperatorTest.java | 156 ++++
.../timeseries/serde/TimeSeriesBlockSerdeTest.java | 176 ++++
.../src/test/resources/queries/QueryHints.json | 64 +-
.../segment/local/data/manager/StaleSegment.java | 48 +-
.../local/data/manager/TableDataManager.java | 8 +
.../immutable/ImmutableSegmentLoader.java | 26 +-
.../indexsegment/mutable/MutableSegmentImpl.java | 20 +-
.../impl/VarByteChunkForwardIndexWriterV4.java | 3 +-
.../converter/RealtimeSegmentConverter.java | 4 +-
.../stats/MutableNoDictionaryColStatistics.java | 9 +
.../impl/forward/CLPMutableForwardIndexV2.java | 37 +-
.../forward/FixedByteMVMutableForwardIndex.java | 10 +
.../recordtransformer/ComplexTypeTransformer.java | 10 +-
.../recordtransformer/CompositeTransformer.java | 70 +-
.../recordtransformer/DataTypeTransformer.java | 7 +
.../recordtransformer/ExpressionTransformer.java | 16 +
.../local/recordtransformer/FilterTransformer.java | 7 +
.../recordtransformer/NullValueTransformer.java | 1 +
.../recordtransformer/SanitizationTransformer.java | 1 +
.../SchemaConformingTransformer.java | 1 +
.../SchemaConformingTransformerV2.java | 245 +++---
.../recordtransformer/SpecialValueTransformer.java | 1 +
.../TimeValidationTransformer.java | 1 +
.../enricher/clp/CLPEncodingEnricher.java | 8 +-
.../enricher/clp/CLPEncodingEnricherConfig.java} | 6 +-
.../enricher/clp/CLPEncodingEnricherFactory.java | 12 +-
.../enricher/function/CustomFunctionEnricher.java | 10 +-
.../function/CustomFunctionEnricherConfig.java | 7 +-
.../function/CustomFunctionEnricherFactory.java | 10 +-
.../RecordReaderSegmentCreationDataSource.java | 9 -
.../local/segment/creator/TransformPipeline.java | 14 +-
.../impl/SegmentIndexCreationDriverImpl.java | 27 +-
.../creator/impl/fwd/CLPForwardIndexCreatorV2.java | 488 +++++++++++
.../creator/impl/stats/CLPStatsProvider.java | 25 +
.../index/forward/ForwardIndexCreatorFactory.java | 4 +
.../index/forward/ForwardIndexReaderFactory.java | 9 +
.../segment/index/forward/ForwardIndexType.java | 6 +-
.../local/segment/index/map/MapIndexType.java | 20 +-
.../segment/index/map/MutableMapIndexImpl.java | 3 +-
.../readers/forward/CLPForwardIndexReaderV2.java | 260 ++++++
.../segment/store/SingleFileIndexDirectory.java | 19 +-
.../pinot/segment/local/utils/IngestionUtils.java | 133 +--
.../segment/local/utils/TableConfigUtils.java | 10 +-
.../MutableSegmentEntriesAboveThresholdTest.java | 194 +++++
.../recordtransformer/RecordTransformerTest.java | 1 +
.../SchemaConformingTransformerTest.java | 1 +
.../SchemaConformingTransformerV2Test.java | 483 +++++++----
.../creator/CLPForwardIndexCreatorV2Test.java | 167 ++++
.../mutable/CLPMutableForwardIndexV2Test.java | 43 +-
.../FixedByteMVMutableForwardIndexTest.java | 41 +
.../FixedByteSVMutableForwardIndexTest.java | 1 +
.../index/loader/ForwardIndexHandlerTest.java | 7 +-
.../store/SingleFileIndexDirectoryTest.java | 11 +-
.../local/segment/store/TextIndexUtilsTest.java | 7 +-
.../segment/local/utils/IngestionUtilsTest.java | 63 +-
.../src/test/resources/data/log.jsonl.gz | Bin 0 -> 6148486 bytes
.../pinot/segment/spi/AggregationFunctionType.java | 96 ++-
.../segment/spi/creator/name/SegmentNameUtils.java | 2 +
.../segment/spi/index/ForwardIndexConfig.java | 1 +
.../pinot/segment/spi/index/TextIndexConfig.java | 20 +-
.../segment/spi/index/mutable/MutableIndex.java | 7 +
.../api/resources/ControllerJobStatusResource.java | 53 +-
.../pinot/server/api/resources/TablesResource.java | 146 +++-
.../pinot/server/starter/ServerInstance.java | 4 +
.../spi/annotations/RecordEnricherFactory.java | 30 -
.../apache/pinot/spi/config/table/FieldConfig.java | 5 +-
.../SchemaConformingTransformerV2Config.java | 166 +++-
.../spi/data/readers/BaseRecordExtractor.java | 126 ++-
.../pinot/spi/data/readers/RecordExtractor.java | 12 +-
.../pinot/spi/env/CommonsConfigurationUtils.java | 18 +
.../spi/recordenricher/RecordEnricherConfig.java | 23 -
.../spi/recordenricher/RecordEnricherPipeline.java | 75 --
.../spi}/recordtransformer/RecordTransformer.java | 12 +-
.../enricher}/RecordEnricher.java | 21 +-
.../enricher}/RecordEnricherFactory.java | 20 +-
.../enricher}/RecordEnricherRegistry.java | 40 +-
.../enricher}/RecordEnricherValidationConfig.java | 2 +-
.../spi/stream/PartitionGroupMetadataFetcher.java | 2 +-
.../pinot/spi/stream/StreamConfigProperties.java | 5 +
.../pinot/spi/stream/StreamMetadataProvider.java | 16 +
.../java/org/apache/pinot/spi/trace/Tracing.java | 2 +-
.../apache/pinot/spi/utils/CommonConstants.java | 11 +-
.../utils/builder/ControllerRequestURLBuilder.java | 4 +
.../pinot/tsdb/planner/TimeSeriesExchangeNode.java | 75 ++
.../tsdb/planner/TimeSeriesPlanConstants.java | 1 +
.../tsdb/planner/TimeSeriesPlanFragmenter.java | 127 +++
.../tsdb/planner/TimeSeriesQueryEnvironment.java | 2 +-
.../tsdb/planner/physical/TableScanVisitor.java | 2 +-
.../physical/TimeSeriesQueryServerInstance.java | 11 +-
.../tsdb/planner/TimeSeriesPlanFragmenterTest.java | 167 ++++
.../java/org/apache/pinot/tsdb/spi/AggInfo.java | 30 +-
.../tsdb/spi/plan/BaseTimeSeriesPlanNode.java | 16 +-
.../tsdb/spi/plan/LeafTimeSeriesPlanNode.java | 15 +-
.../tsdb/spi/plan/serde/TimeSeriesPlanSerde.java | 18 +-
.../tsdb/spi/series/BaseTimeSeriesBuilder.java | 18 +-
.../apache/pinot/tsdb/spi/series/TimeSeries.java | 19 +-
.../tsdb/spi/plan/LeafTimeSeriesPlanNodeTest.java | 9 +-
.../spi/plan/serde/TimeSeriesPlanSerdeTest.java | 4 +-
.../pinot/tsdb/spi/series/TimeSeriesTest.java | 54 ++
.../org/apache/pinot/tools/BootstrapTableTool.java | 11 +-
.../org/apache/pinot/tools/QuickStartBase.java | 8 +-
.../apache/pinot/tools/QuickstartTableRequest.java | 14 +
.../pinot/tools/RealtimeQuickStartWithMinion.java | 21 +-
.../pinot/tools/admin/command/AddTableCommand.java | 23 +-
.../tools/admin/command/GenerateDataCommand.java | 9 +-
.../tools/admin/command/QuickstartRunner.java | 2 +-
pom.xml | 27 +-
330 files changed, 14136 insertions(+), 5499 deletions(-)
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/function/scalar/UrlFunctions.java
copy
pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/{TableSegmentValidationInfo.java
=> TableLLCSegmentUploadResponse.java} (62%)
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/function/scalar/UrlFunctionsTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/restlet/resources/TableSegmentValidationInfoTest.java
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableStaleSegmentResponse.java
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/BlockingSegmentCompletionFSM.java
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/SegmentCompletionConfig.java
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/SegmentCompletionFSM.java
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/SegmentCompletionFSMFactory.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResourceTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerNeedRefreshTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/data/manager/TableIndexingTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/util/GroupByUtilsTest.java
delete mode 100644
pinot-core/src/test/java/org/apache/pinot/queries/WithOptionQueriesTest.java
create mode 100644 pinot-core/src/test/resources/TableIndexingTest.csv
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/StaleSegmentCheckIntegrationTest.java
copy pinot-plugins/pinot-batch-ingestion/{ =>
pinot-batch-ingestion-spark-base}/pom.xml (68%)
copy
pinot-plugins/pinot-batch-ingestion/{pinot-batch-ingestion-spark-2.4/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark/SparkSegmentTarPushJobRunner.java
=>
pinot-batch-ingestion-spark-base/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark/common/BaseSparkSegmentMetadataPushJobRunner.java}
(67%)
copy
pinot-plugins/pinot-batch-ingestion/{pinot-batch-ingestion-spark-2.4/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark/SparkSegmentTarPushJobRunner.java
=>
pinot-batch-ingestion-spark-base/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark/common/BaseSparkSegmentTarPushJobRunner.java}
(67%)
copy
pinot-plugins/pinot-batch-ingestion/{pinot-batch-ingestion-spark-2.4/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark/SparkSegmentUriPushJobRunner.java
=>
pinot-batch-ingestion-spark-base/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark/common/BaseSparkSegmentUriPushJobRunner.java}
(68%)
create mode 100644
pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/mergerollup/DimensionValueTransformer.java
create mode 100644
pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompactmerge/UpsertCompactMergeTaskExecutor.java
create mode 100644
pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompactmerge/UpsertCompactMergeTaskExecutorFactory.java
create mode 100644
pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompactmerge/UpsertCompactMergeTaskGenerator.java
copy
pinot-segment-local/src/main/java/org/apache/pinot/plugin/record/enricher/clp/ClpEnricherConfig.java
=>
pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompactmerge/UpsertCompactMergeTaskProgressObserverFactory.java
(62%)
create mode 100644
pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/mergerollup/DimensionValueTransformerTest.java
create mode 100644
pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/upsertcompactmerge/UpsertCompactMergeTaskExecutorTest.java
create mode 100644
pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/upsertcompactmerge/UpsertCompactMergeTaskGeneratorTest.java
rename
pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/timeseries/{PhysicalTimeSeriesPlanVisitor.java
=> PhysicalTimeSeriesServerPlanVisitor.java} (90%)
create mode 100644
pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/timeseries/TimeSeriesExchangeReceiveOperator.java
create mode 100644
pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/timeseries/TimeSeriesExchangeReceivePlanNode.java
create mode 100644
pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/timeseries/serde/TimeSeriesBlockSerde.java
rename
pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/timeseries/{PhysicalTimeSeriesPlanVisitorTest.java
=> PhysicalTimeSeriesServerPlanVisitorTest.java} (75%)
create mode 100644
pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/timeseries/TimeSeriesExchangeReceiveOperatorTest.java
create mode 100644
pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/timeseries/serde/TimeSeriesBlockSerdeTest.java
copy
pinot-plugins/pinot-timeseries-lang/pinot-timeseries-m3ql/src/main/java/org/apache/pinot/tsdb/m3ql/plan/KeepLastValuePlanNode.java
=>
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/data/manager/StaleSegment.java
(51%)
rename pinot-segment-local/src/main/java/org/apache/pinot/{plugin/record =>
segment/local/recordtransformer}/enricher/clp/CLPEncodingEnricher.java (92%)
rename
pinot-segment-local/src/main/java/org/apache/pinot/{plugin/record/enricher/clp/ClpEnricherConfig.java
=>
segment/local/recordtransformer/enricher/clp/CLPEncodingEnricherConfig.java}
(85%)
rename pinot-segment-local/src/main/java/org/apache/pinot/{plugin/record =>
segment/local/recordtransformer}/enricher/clp/CLPEncodingEnricherFactory.java
(80%)
rename pinot-segment-local/src/main/java/org/apache/pinot/{plugin/record =>
segment/local/recordtransformer}/enricher/function/CustomFunctionEnricher.java
(86%)
rename pinot-segment-local/src/main/java/org/apache/pinot/{plugin/record =>
segment/local/recordtransformer}/enricher/function/CustomFunctionEnricherConfig.java
(89%)
rename pinot-segment-local/src/main/java/org/apache/pinot/{plugin/record =>
segment/local/recordtransformer}/enricher/function/CustomFunctionEnricherFactory.java
(87%)
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/CLPForwardIndexCreatorV2.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/CLPForwardIndexReaderV2.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/indexsegment/mutable/MutableSegmentEntriesAboveThresholdTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/CLPForwardIndexCreatorV2Test.java
create mode 100644 pinot-segment-local/src/test/resources/data/log.jsonl.gz
delete mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/annotations/RecordEnricherFactory.java
delete mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/recordenricher/RecordEnricherConfig.java
delete mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/recordenricher/RecordEnricherPipeline.java
rename {pinot-segment-local/src/main/java/org/apache/pinot/segment/local =>
pinot-spi/src/main/java/org/apache/pinot/spi}/recordtransformer/RecordTransformer.java
(82%)
rename pinot-spi/src/main/java/org/apache/pinot/spi/{recordenricher =>
recordtransformer/enricher}/RecordEnricher.java (65%)
rename pinot-spi/src/main/java/org/apache/pinot/spi/{recordenricher =>
recordtransformer/enricher}/RecordEnricherFactory.java (75%)
rename pinot-spi/src/main/java/org/apache/pinot/spi/{recordenricher =>
recordtransformer/enricher}/RecordEnricherRegistry.java (69%)
rename pinot-spi/src/main/java/org/apache/pinot/spi/{recordenricher =>
recordtransformer/enricher}/RecordEnricherValidationConfig.java (95%)
create mode 100644
pinot-timeseries/pinot-timeseries-planner/src/main/java/org/apache/pinot/tsdb/planner/TimeSeriesExchangeNode.java
create mode 100644
pinot-timeseries/pinot-timeseries-planner/src/main/java/org/apache/pinot/tsdb/planner/TimeSeriesPlanFragmenter.java
create mode 100644
pinot-timeseries/pinot-timeseries-planner/src/test/java/org/apache/pinot/tsdb/planner/TimeSeriesPlanFragmenterTest.java
create mode 100644
pinot-timeseries/pinot-timeseries-spi/src/test/java/org/apache/pinot/tsdb/spi/series/TimeSeriesTest.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
