This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a change to branch json-indexing
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
discard ee6714c Adding index creator and reader
discard fd32a3e Initial commit
add d2d47b2 Fix quick demo instructions #6155 (#6156)
add 2cd6abb Deep Extraction Support for ORC, Thrift, and ProtoBuf Records
(#6046)
add b22045a Do not apply metadata/dictionary based agg operator to upsert
table (#6154)
add e15fa80 Framework for adding compatibility tests (#6129)
add 0c19741 Do not prune segments for selection queries over upsert table
(#6158)
add cf675a5 Setup classpath correctly for the compat test runner (#6160)
add da451a8 Add more validation for upsert config (#6153)
add 81248d1 Update tyrus version (#6162)
add 08c46a8 Support using ordinals in GROUP BY and ORDER BY clause (#6152)
add 2484f5b add option flags for controller host:port, jar path, and
admin path (#6163)
add df4911d Bump up the helix version to 0.9.8 (#6166)
add ac7b0e7 UI integration of instance and segment operations (#6148)
add 1bf5d02 RealtimeToOfflineSegments task generator (#6124)
add 59c188d Fixing the issue of applying ordinals in order by for
distinct queries (#6171)
add d08fd5c Support reloading upsert table (#6167)
add 5817f15 Update license and notice for 0.6.0 release (#6175)
add 1c17ab8 Add table name to the log on validation failure (#6173)
add 669bd59 add execution timeout and fix exception stats (#6177)
add 73d2839 Merge common APIs for Dictionary (#6176)
add 5577e87 Add table level lock for segment upload (#6165)
add 7a3062c Initial commit
add cd062fb Adding index creator and reader
add f20a359 changing table name from super to personJson
add 9e6f837 adding example data for json
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (ee6714c)
\
N -- N -- N refs/heads/json-indexing (9e6f837)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
LICENSE-binary | 14 +-
NOTICE-binary | 14 +-
README.md | 2 +-
compatibility-verifier/compCheck.sh | 45 +-
docker/images/pinot/bin/generator.sh | 79 +-
.../pinot/common/metadata/ZKMetadataProvider.java | 5 +
.../common/minion/MinionTaskMetadataUtils.java | 81 +
.../RealtimeToOfflineSegmentsTaskMetadata.java | 88 +
.../pinot/common/utils/helix/HelixHelper.java | 3 +-
.../apache/pinot/sql/parsers/CalciteSqlParser.java | 43 +-
.../RealtimeToOfflineSegmentsTaskMetadataTest.java | 46 +
.../pinot/sql/parsers/CalciteSqlCompilerTest.java | 70 +-
.../api/resources/PinotTableIndexingConfigs.java | 3 +-
.../api/resources/PinotTableMetadataConfigs.java | 3 +-
.../api/resources/PinotTableRestletResource.java | 16 +-
.../api/resources/PinotTableSegmentConfigs.java | 3 +-
.../helix/core/PinotHelixResourceManager.java | 45 +-
...rInfoProvider.java => ClusterInfoAccessor.java} | 40 +-
.../helix/core/minion/PinotTaskManager.java | 10 +-
.../generator/ConvertToRawIndexTaskGenerator.java | 14 +-
.../RealtimeToOfflineSegmentsTaskGenerator.java | 312 ++
.../minion/generator/TaskGeneratorRegistry.java | 7 +-
.../core/minion/generator/TaskGeneratorUtils.java | 52 +-
pinot-controller/src/main/resources/.eslintrc | 7 +-
pinot-controller/src/main/resources/app/App.tsx | 2 +-
.../main/resources/app/components/AppLoader.tsx | 2 +-
.../main/resources/app/components/Breadcrumbs.tsx | 52 +-
.../src/main/resources/app/components/Confirm.tsx | 12 +-
.../components/{AppLoader.tsx => CustomButton.tsx} | 48 +-
.../resources/app/components/CustomCodemirror.tsx | 19 +-
.../main/resources/app/components/CustomDialog.tsx | 87 +
.../resources/app/components/CustomMultiSelect.tsx | 123 +
.../app/components/CustomNotification.tsx | 69 +
.../src/main/resources/app/components/Header.tsx | 2 +-
.../app/components/Homepage/InstanceTable.tsx | 6 +-
.../Homepage/Operations/EditConfigOp.tsx | 59 +
.../components/Homepage/Operations/EditTagsOp.tsx | 59 +
.../src/main/resources/app/components/Layout.tsx | 8 +-
.../main/resources/app/components/MaterialTree.tsx | 14 +-
.../src/main/resources/app/components/SideBar.tsx | 10 +-
.../resources/app/components/SimpleAccordion.tsx | 3 +-
.../app/components/SvgIcons/ClusterManagerIcon.tsx | 8 +-
.../resources/app/components/SvgIcons/Logo.tsx | 30 +-
.../app/components/SvgIcons/QueryConsoleIcon.tsx | 3 +-
.../app/components/SvgIcons/SwaggerIcon.tsx | 6 +-
.../app/components/SvgIcons/ZookeeperIcon.tsx | 14 +-
.../src/main/resources/app/components/TabPanel.tsx | 2 +-
.../src/main/resources/app/components/Table.tsx | 13 +-
.../app/components/Zookeeper/TreeDirectory.tsx | 65 +-
.../src/main/resources/app/interfaces/types.d.ts | 11 +-
.../src/main/resources/app/pages/HomePage.tsx | 6 +-
.../main/resources/app/pages/InstanceDetails.tsx | 260 +-
.../resources/app/pages/InstanceListingPage.tsx | 10 +-
.../src/main/resources/app/pages/Query.tsx | 11 +-
.../main/resources/app/pages/SegmentDetails.tsx | 107 +-
.../main/resources/app/pages/TablesListingPage.tsx | 12 +-
.../src/main/resources/app/pages/TenantDetails.tsx | 1 -
.../resources/app/pages/TenantsListingPage.tsx | 8 +-
.../src/main/resources/app/pages/ZookeeperPage.tsx | 58 +-
.../src/main/resources/app/requests/index.ts | 36 +-
pinot-controller/src/main/resources/app/router.tsx | 28 +-
.../src/main/resources/app/styles/styles.css | 4 +
.../main/resources/app/utils/PinotMethodUtils.ts | 110 +-
.../src/main/resources/app/utils/Utils.tsx | 6 +-
pinot-controller/src/main/resources/package.json | 11 +-
.../src/main/resources/webpack.config.js | 3 +
...RealtimeToOfflineSegmentsTaskGeneratorTest.java | 452 +++
.../apache/pinot/core/common/MinionConstants.java | 26 +-
.../manager/realtime/RealtimeTableDataManager.java | 58 +-
.../immutable/ImmutableSegmentImpl.java | 31 +-
.../immutable/ImmutableSegmentLoader.java | 10 +-
.../indexsegment/mutable/MutableSegmentImpl.java | 21 +-
.../core/operator/filter/FilterOperatorUtils.java | 4 +-
.../filter/RangeIndexBasedFilterOperator.java | 8 +-
.../filter/SortedIndexBasedFilterOperator.java | 10 +-
.../predicate/RangePredicateEvaluatorFactory.java | 19 +-
.../core/plan/maker/InstancePlanMakerImplV2.java | 5 +-
.../query/pruner/SelectionQuerySegmentPruner.java | 7 +-
.../converter/stats/RealtimeColumnStatistics.java | 33 +-
.../impl/dictionary/BaseMutableDictionary.java | 72 -
.../dictionary/BaseOffHeapMutableDictionary.java | 3 +-
.../dictionary/BaseOnHeapMutableDictionary.java | 3 +-
.../impl/dictionary/MutableDictionaryFactory.java | 5 +-
.../core/segment/index/readers/BaseDictionary.java | 72 -
.../index/readers/BaseImmutableDictionary.java | 43 +-
.../segment/index/readers/BytesDictionary.java | 15 +-
.../readers/ConstantValueBytesDictionary.java | 19 +-
.../readers/ConstantValueDoubleDictionary.java | 19 +-
.../readers/ConstantValueFloatDictionary.java | 19 +-
.../index/readers/ConstantValueIntDictionary.java | 19 +-
.../index/readers/ConstantValueLongDictionary.java | 19 +-
.../readers/ConstantValueStringDictionary.java | 19 +-
.../core/segment/index/readers/Dictionary.java | 102 +-
.../segment/index/readers/MutableDictionary.java | 32 +-
.../processing/framework/SegmentMapper.java | 12 +-
.../upsert/PartitionUpsertMetadataManager.java | 173 +-
.../apache/pinot/core/upsert/RecordLocation.java | 30 +-
.../org/apache/pinot/core/util/SchemaUtils.java | 8 +
.../apache/pinot/core/util/TableConfigUtils.java | 8 +-
...ngeOfflineDictionaryPredicateEvaluatorTest.java | 53 +-
...adataAndDictionaryAggregationPlanMakerTest.java | 50 +-
.../pruner/SelectionQuerySegmentPrunerTest.java | 30 +
.../impl/dictionary/MutableDictionaryTest.java | 39 +-
.../processing/framework/SegmentMapperTest.java | 2 +-
.../upsert/PartitionUpsertMetadataManagerTest.java | 212 ++
.../apache/pinot/core/util/SchemaUtilsTest.java | 18 +
.../pinot/core/util/TableConfigUtilsTest.java | 13 +
pinot-integration-tests/pom.xml | 8 +
.../java/org/apache/pinot/compat/tests/BaseOp.java | 73 +
.../pinot/compat/tests/CompatTestOperation.java | 29 +-
.../pinot/compat/tests/CompatibilityOpsRunner.java | 71 +
.../org/apache/pinot/compat/tests/QueryOp.java | 61 +
.../org/apache/pinot/compat/tests/SegmentOp.java | 95 +
.../org/apache/pinot/compat/tests/StreamOp.java | 88 +
.../org/apache/pinot/compat/tests/TableOp.java | 95 +
...fflineSegmentsMinionClusterIntegrationTest.java | 216 ++
.../tests/SimpleMinionClusterIntegrationTest.java | 12 +-
.../compat-tests/configs/FeatureTest1-schema.json | 82 +
.../compat-tests/configs/feature-test-1.json | 54 +
.../compat-tests/data/FeatureTest1-data-00.csv | 13 +
.../compat-tests/data/FeatureTest1-data-01.csv | 10 +
.../compat-tests/data/recordReaderConfig.json | 5 +
.../compat-tests/post-broker-rollback.yaml | 49 +
.../compat-tests/post-controller-rollback.yaml | 49 +
.../compat-tests/post-server-rollback.yaml | 49 +
.../compat-tests/post-server-upgrade.yaml | 49 +
.../resources/compat-tests/pre-broker-upgrade.yaml | 49 +
.../compat-tests/pre-controller-upgrade.yaml | 49 +
.../resources/compat-tests/pre-server-upgrade.yaml | 49 +
.../src/test/resources/compat-tests/sample.yaml | 49 +
.../org/apache/pinot/minion/MinionStarter.java | 4 +-
.../BaseMultipleSegmentsConversionExecutor.java | 15 +
.../pinot/minion/executor/BaseTaskExecutor.java | 3 +-
.../executor/MinionTaskZkMetadataManager.java | 57 +
.../RealtimeToOfflineSegmentsTaskExecutor.java | 88 +-
...altimeToOfflineSegmentsTaskExecutorFactory.java | 12 +-
.../executor/TaskExecutorFactoryRegistry.java | 4 +-
.../RealtimeToOfflineSegmentsTaskExecutorTest.java | 95 +-
.../inputformat/avro/AvroRecordExtractor.java | 55 +-
.../pinot/plugin/inputformat/avro/AvroUtils.java | 90 +-
.../inputformat/avro/AvroRecordExtractorTest.java | 7 -
.../avro/AvroRecordToPinotRowGeneratorTest.java | 3 -
...aConfluentSchemaRegistryAvroMessageDecoder.java | 2 -
.../plugin/inputformat/csv/CSVRecordExtractor.java | 63 +-
.../inputformat/csv/CSVRecordExtractorConfig.java | 27 +
.../plugin/inputformat/csv/CSVRecordReader.java | 7 +-
.../csv/CSVRecordExtractorConfigTest.java} | 23 +-
.../inputformat/json/JSONRecordExtractor.java | 20 +-
.../inputformat/json/JSONRecordExtractorUtils.java | 100 -
.../inputformat/json/JSONRecordExtractorTest.java | 136 +-
.../json/JSONRecordExtractorUtilsTest.java | 120 -
.../plugin/inputformat/orc/ORCRecordReader.java | 223 +-
.../inputformat/orc/ORCRecordExtractorTest.java | 274 +-
.../inputformat/parquet/ParquetRecordReader.java | 2 +-
.../inputformat/protobuf/ProtoBufFieldInfo.java} | 30 +-
.../protobuf/ProtoBufRecordExtractor.java | 194 +-
.../inputformat/protobuf/ProtoBufRecordReader.java | 6 +-
.../plugin/inputformat/protobuf/ComplexTypes.java | 3393 ++++++++++++++++++++
.../protobuf/ProtoBufRecordExtractorTest.java | 203 ++
.../src/test/resources/complex_types.desc | Bin 0 -> 1281 bytes
.../src/test/resources/complex_types.proto | 32 +
.../inputformat/thrift/ThriftRecordExtractor.java | 71 +-
.../inputformat/thrift/ThriftRecordReader.java | 14 +-
.../plugin/inputformat/thrift/ComplexTypes.java | 1712 ++++++++++
.../plugin/inputformat/thrift/NestedType.java | 484 +++
.../pinot/plugin/inputformat/thrift/TestEnum.java | 66 +
.../thrift/ThriftRecordExtractorTest.java | 215 ++
.../src/test/resources/complex_types.thrift | 49 +-
.../starter/helix/HelixInstanceDataManager.java | 6 -
.../spi/data/readers/BaseRecordExtractor.java | 192 ++
.../pinot/spi/data/readers/RecordExtractor.java | 18 +-
.../pinot/spi/data/readers/RecordReader.java | 2 +-
.../pinot/spi/data/readers/RecordReaderUtils.java | 68 +-
.../spi/utils/retry/RandomDelayRetryPolicy.java | 26 +-
.../pinot/spi/utils/retry/RetryPolicies.java | 12 +
.../data/readers/AbstractRecordExtractorTest.java | 13 +-
.../spi/data/readers/RecordReaderUtilsTest.java | 46 -
.../pinot/spi/utils/retry/RetryPolicyTest.java | 32 +
pinot-spi/src/test/resources/TestRecordReader.java | 3 -
pinot-tools/pom.xml | 4 +
.../org/apache/pinot/tools/UpsertQuickStart.java | 9 +-
.../pinot/tools/admin/command/JSONQuickstart.java | 40 +-
.../org/apache/pinot/tools/perf/QueryRunner.java | 82 +-
.../batch/personJson/ingestionJobSpec.yaml | 139 +
.../personJson_offline_table_config.json | 25 +
.../batch/personJson/personJson_schema.json | 9 +
.../batch/personJson/rawdata/personJson_data.csv | 101 +
.../batch/personJson/sparkIngestionJobSpec.yaml | 147 +
pom.xml | 4 +-
189 files changed, 12736 insertions(+), 1638 deletions(-)
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/minion/MinionTaskMetadataUtils.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/minion/RealtimeToOfflineSegmentsTaskMetadata.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/metadata/RealtimeToOfflineSegmentsTaskMetadataTest.java
rename
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/{ClusterInfoProvider.java
=> ClusterInfoAccessor.java} (70%)
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/generator/RealtimeToOfflineSegmentsTaskGenerator.java
copy pinot-controller/src/main/resources/app/components/{AppLoader.tsx =>
CustomButton.tsx} (61%)
create mode 100644
pinot-controller/src/main/resources/app/components/CustomDialog.tsx
create mode 100644
pinot-controller/src/main/resources/app/components/CustomMultiSelect.tsx
create mode 100644
pinot-controller/src/main/resources/app/components/CustomNotification.tsx
create mode 100644
pinot-controller/src/main/resources/app/components/Homepage/Operations/EditConfigOp.tsx
create mode 100644
pinot-controller/src/main/resources/app/components/Homepage/Operations/EditTagsOp.tsx
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/minion/generator/RealtimeToOfflineSegmentsTaskGeneratorTest.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/dictionary/BaseMutableDictionary.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/index/readers/BaseDictionary.java
copy
pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordExtractorConfig.java
=>
pinot-core/src/main/java/org/apache/pinot/core/segment/index/readers/MutableDictionary.java
(51%)
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/upsert/PartitionUpsertMetadataManagerTest.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/BaseOp.java
copy
pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordExtractorConfig.java
=>
pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/CompatTestOperation.java
(59%)
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/CompatibilityOpsRunner.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/QueryOp.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/SegmentOp.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/StreamOp.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/TableOp.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/RealtimeToOfflineSegmentsMinionClusterIntegrationTest.java
create mode 100644
pinot-integration-tests/src/test/resources/compat-tests/configs/FeatureTest1-schema.json
create mode 100644
pinot-integration-tests/src/test/resources/compat-tests/configs/feature-test-1.json
create mode 100644
pinot-integration-tests/src/test/resources/compat-tests/data/FeatureTest1-data-00.csv
create mode 100644
pinot-integration-tests/src/test/resources/compat-tests/data/FeatureTest1-data-01.csv
create mode 100644
pinot-integration-tests/src/test/resources/compat-tests/data/recordReaderConfig.json
create mode 100644
pinot-integration-tests/src/test/resources/compat-tests/post-broker-rollback.yaml
create mode 100644
pinot-integration-tests/src/test/resources/compat-tests/post-controller-rollback.yaml
create mode 100644
pinot-integration-tests/src/test/resources/compat-tests/post-server-rollback.yaml
create mode 100644
pinot-integration-tests/src/test/resources/compat-tests/post-server-upgrade.yaml
create mode 100644
pinot-integration-tests/src/test/resources/compat-tests/pre-broker-upgrade.yaml
create mode 100644
pinot-integration-tests/src/test/resources/compat-tests/pre-controller-upgrade.yaml
create mode 100644
pinot-integration-tests/src/test/resources/compat-tests/pre-server-upgrade.yaml
create mode 100644
pinot-integration-tests/src/test/resources/compat-tests/sample.yaml
create mode 100644
pinot-minion/src/main/java/org/apache/pinot/minion/executor/MinionTaskZkMetadataManager.java
copy
pinot-plugins/pinot-input-format/pinot-csv/src/{main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordExtractorConfig.java
=>
test/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordExtractorConfigTest.java}
(67%)
delete mode 100644
pinot-plugins/pinot-input-format/pinot-json/src/main/java/org/apache/pinot/plugin/inputformat/json/JSONRecordExtractorUtils.java
delete mode 100644
pinot-plugins/pinot-input-format/pinot-json/src/test/java/org/apache/pinot/plugin/inputformat/json/JSONRecordExtractorUtilsTest.java
copy
pinot-plugins/pinot-input-format/{pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordExtractorConfig.java
=>
pinot-protobuf/src/main/java/org/apache/pinot/plugin/inputformat/protobuf/ProtoBufFieldInfo.java}
(53%)
create mode 100644
pinot-plugins/pinot-input-format/pinot-protobuf/src/test/java/org/apache/pinot/plugin/inputformat/protobuf/ComplexTypes.java
create mode 100644
pinot-plugins/pinot-input-format/pinot-protobuf/src/test/java/org/apache/pinot/plugin/inputformat/protobuf/ProtoBufRecordExtractorTest.java
create mode 100644
pinot-plugins/pinot-input-format/pinot-protobuf/src/test/resources/complex_types.desc
create mode 100644
pinot-plugins/pinot-input-format/pinot-protobuf/src/test/resources/complex_types.proto
create mode 100644
pinot-plugins/pinot-input-format/pinot-thrift/src/test/java/org/apache/pinot/plugin/inputformat/thrift/ComplexTypes.java
create mode 100644
pinot-plugins/pinot-input-format/pinot-thrift/src/test/java/org/apache/pinot/plugin/inputformat/thrift/NestedType.java
create mode 100644
pinot-plugins/pinot-input-format/pinot-thrift/src/test/java/org/apache/pinot/plugin/inputformat/thrift/TestEnum.java
create mode 100644
pinot-plugins/pinot-input-format/pinot-thrift/src/test/java/org/apache/pinot/plugin/inputformat/thrift/ThriftRecordExtractorTest.java
copy pinot-controller/src/main/resources/app/components/TabPanel.tsx =>
pinot-plugins/pinot-input-format/pinot-thrift/src/test/resources/complex_types.thrift
(51%)
create mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/BaseRecordExtractor.java
copy
pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordExtractorConfig.java
=>
pinot-spi/src/main/java/org/apache/pinot/spi/utils/retry/RandomDelayRetryPolicy.java
(50%)
delete mode 100644
pinot-spi/src/test/java/org/apache/pinot/spi/data/readers/RecordReaderUtilsTest.java
create mode 100644
pinot-tools/src/main/resources/examples/batch/personJson/ingestionJobSpec.yaml
create mode 100644
pinot-tools/src/main/resources/examples/batch/personJson/personJson_offline_table_config.json
create mode 100644
pinot-tools/src/main/resources/examples/batch/personJson/personJson_schema.json
create mode 100644
pinot-tools/src/main/resources/examples/batch/personJson/rawdata/personJson_data.csv
create mode 100644
pinot-tools/src/main/resources/examples/batch/personJson/sparkIngestionJobSpec.yaml
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
