This is an automated email from the ASF dual-hosted git repository. xiangfu pushed a change to branch fixing_spi_comparison in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
discard 5e0ce62 Fixing comparison and hashcode for Schema, FieldSpec and ColumnMetadata add 2b55004 SegmentDirectoryLoader interface and local impl (#6949) add 848c3d4 Fixing comparison and hashcode for Schema, FieldSpec and ColumnMetadata This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (5e0ce62) \ N -- N -- N refs/heads/fixing_spi_comparison (848c3d4) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: .../broker/pruner/PartitionZKMetadataPruner.java | 4 +- .../segmentpruner/PartitionSegmentPruner.java | 4 +- .../broker/pruner/SegmentZKMetadataPrunerTest.java | 2 +- .../routing/segmentpruner/SegmentPrunerTest.java | 2 +- .../metadata/segment/SegmentPartitionMetadata.java | 1 + .../apache/pinot/common/utils/SegmentUtils.java | 2 +- .../common/metadata/SegmentZKMetadataTest.java | 2 +- .../segment/ColumnPartitionMetadataTest.java | 1 + .../resources/LLCSegmentCompletionHandlers.java | 4 +- .../segment/OfflineSegmentAssignment.java | 2 +- .../realtime/PinotLLCRealtimeSegmentManager.java | 6 +- .../segment/CommittingSegmentDescriptor.java | 2 +- .../helix/core/util/ZKMetadataUtils.java | 6 +- .../realtime/provisioning/MemoryEstimator.java | 2 +- .../OfflineReplicaGroupSegmentAssignmentTest.java | 2 +- .../PinotLLCRealtimeSegmentManagerTest.java | 2 +- .../controller/utils/SegmentMetadataMockUtils.java | 6 +- .../realtime/LLRealtimeSegmentDataManager.java | 6 +- .../core/metadata/DefaultMetadataExtractor.java | 2 +- .../pinot/core/minion/RawIndexConverter.java | 6 +- .../apache/pinot/core/minion/SegmentConverter.java | 2 +- .../apache/pinot/core/minion/SegmentPurger.java | 2 +- .../minion/rollup/MergeRollupSegmentConverter.java | 2 +- .../processing/collector/ConcatCollector.java | 2 +- .../partitioner/TableConfigPartitioner.java | 2 +- .../pinot/core/util/GenericRowSerDeUtils.java | 2 +- .../pinot/core/minion/SegmentPurgerTest.java | 17 ++-- .../query/pruner/ColumnValueSegmentPrunerTest.java | 2 +- .../framework/SegmentProcessingFrameworkTest.java | 2 +- .../converter/RealtimeSegmentConverterTest.java | 2 +- ...vertToRawIndexMinionClusterIntegrationTest.java | 2 +- ...PartitionLLCRealtimeClusterIntegrationTest.java | 2 +- .../pinot/perf/BenchmarkFixedBitIntReader.java | 2 +- .../BenchmarkFixedBitSVForwardIndexReader.java | 2 +- .../pinot/perf/BenchmarkOfflineIndexReader.java | 18 ++-- .../pinot/perf/BenchmarkPinotDataBitSet.java | 2 +- .../pinot/perf/BenchmarkPinotDataBuffer.java | 2 +- .../apache/pinot/perf/BenchmarkQueryEngine.java | 2 +- .../org/apache/pinot/perf/RawIndexBenchmark.java | 2 +- .../hadoop/job/HadoopSegmentPreprocessingJob.java | 2 +- .../job/partitioners/PartitionFunctionFactory.java | 8 +- .../RealtimeToOfflineSegmentsTaskExecutorTest.java | 4 +- .../filebased/FileBasedSegmentWriterTest.java | 2 +- .../indexsegment/immutable/EmptyIndexSegment.java | 4 +- .../immutable/ImmutableSegmentImpl.java | 6 +- .../immutable/ImmutableSegmentLoader.java | 49 ++++++---- .../indexsegment/mutable/IntermediateSegment.java | 4 +- .../indexsegment/mutable/MutableSegmentImpl.java | 14 ++- .../local/io/reader/impl/FixedBitIntReader.java | 2 +- .../impl/FixedByteSingleValueMultiColReader.java | 2 +- .../readerwriter/PinotDataBufferMemoryManager.java | 2 +- .../RealtimeIndexOffHeapMemoryManager.java | 2 +- ...ixedByteSingleValueMultiColumnReaderWriter.java | 2 +- .../local/io/util/FixedBitIntReaderWriter.java | 2 +- .../local/io/util/FixedBitIntReaderWriterV2.java | 4 +- .../local/io/util/FixedByteValueReaderWriter.java | 2 +- .../segment/local/io/util/PinotDataBitSet.java | 2 +- .../segment/local/io/util/PinotDataBitSetV2.java | 4 +- .../local/io/util/VarLengthValueReader.java | 2 +- .../local/io/util/VarLengthValueWriter.java | 2 +- .../local/io/writer/impl/DirectMemoryManager.java | 2 +- .../writer/impl/FixedBitMVForwardIndexWriter.java | 2 +- .../writer/impl/FixedBitSVForwardIndexWriter.java | 2 +- .../impl/FixedByteSingleValueMultiColWriter.java | 2 +- .../local/io/writer/impl/MmapMemoryManager.java | 2 +- .../writer/impl/MutableOffHeapByteArrayStore.java | 2 +- .../local/loader/LocalSegmentDirectoryLoader.java | 50 +++++++++++ .../dictionary/BaseOffHeapMutableDictionary.java | 2 +- .../impl/dictionary/OffHeapMutableBytesStore.java | 2 +- .../forward/FixedByteMVMutableForwardIndex.java | 2 +- .../forward/FixedByteSVMutableForwardIndex.java | 2 +- .../creator/impl/SegmentColumnarIndexCreator.java | 7 +- .../creator/impl/SegmentDictionaryCreator.java | 3 +- .../impl/SegmentIndexCreationDriverImpl.java | 3 +- .../impl/bloom/OnHeapGuavaBloomFilterCreator.java | 2 +- .../fwd/MultiValueUnsortedForwardIndexCreator.java | 2 +- .../fwd/SingleValueFixedByteRawIndexCreator.java | 2 +- .../fwd/SingleValueSortedForwardIndexCreator.java | 2 +- .../SingleValueUnsortedForwardIndexCreator.java | 2 +- .../fwd/SingleValueVarByteRawIndexCreator.java | 2 +- .../impl/inv/BitmapInvertedIndexWriter.java | 2 +- .../inv/OffHeapBitmapInvertedIndexCreator.java | 4 +- .../impl/inv/OnHeapBitmapInvertedIndexCreator.java | 2 +- .../creator/impl/inv/RangeIndexCreator.java | 4 +- .../impl/inv/geospatial/BaseH3IndexCreator.java | 2 +- .../impl/inv/geospatial/OffHeapH3IndexCreator.java | 2 +- .../impl/inv/json/BaseJsonIndexCreator.java | 4 +- .../impl/inv/json/OffHeapJsonIndexCreator.java | 2 +- .../impl/inv/text/LuceneFSTIndexCreator.java | 2 +- .../impl/nullvalue/NullValueVectorCreator.java | 2 +- .../stats/AbstractColumnStatisticsCollector.java | 2 +- .../creator/impl/text/LuceneTextIndexCreator.java | 4 +- .../index/column/BaseVirtualColumnProvider.java | 2 +- .../DefaultNullValueVirtualColumnProvider.java | 2 +- .../index/column/PhysicalColumnIndexContainer.java | 8 +- .../converter/SegmentV1V2ToV3FormatConverter.java | 35 +++++--- .../segment/index/datasource/EmptyDataSource.java | 2 +- .../index/datasource/ImmutableDataSource.java | 2 +- .../segment/index/loader/IndexLoadingConfig.java | 27 +++++- .../local/segment/index/loader/LoaderUtils.java | 8 +- .../segment/index/loader/SegmentPreProcessor.java | 18 ++-- .../loader/bloomfilter/BloomFilterHandler.java | 12 +-- .../ColumnMinMaxValueGenerator.java | 10 +-- .../defaultcolumn/BaseDefaultColumnHandler.java | 10 +-- .../defaultcolumn/DefaultColumnHandlerFactory.java | 4 +- .../defaultcolumn/V1DefaultColumnHandler.java | 4 +- .../defaultcolumn/V3DefaultColumnHandler.java | 8 +- .../index/loader/invertedindex/H3IndexHandler.java | 10 +-- .../loader/invertedindex/InvertedIndexHandler.java | 10 +-- .../loader/invertedindex/JsonIndexHandler.java | 10 +-- .../invertedindex/LuceneFSTIndexHandler.java | 10 +-- .../loader/invertedindex/RangeIndexHandler.java | 10 +-- .../loader/invertedindex/TextIndexHandler.java | 16 ++-- .../index/readers/BaseImmutableDictionary.java | 2 +- .../index/readers/BitmapInvertedIndexReader.java | 2 +- .../segment/index/readers/BytesDictionary.java | 2 +- .../segment/index/readers/DoubleDictionary.java | 2 +- .../segment/index/readers/FloatDictionary.java | 2 +- .../local/segment/index/readers/IntDictionary.java | 2 +- .../segment/index/readers/LongDictionary.java | 2 +- .../index/readers/LuceneFSTIndexReader.java | 2 +- .../index/readers/NullValueVectorReaderImpl.java | 2 +- .../segment/index/readers/OnHeapDictionary.java | 2 +- .../index/readers/OnHeapDoubleDictionary.java | 2 +- .../index/readers/OnHeapFloatDictionary.java | 2 +- .../segment/index/readers/OnHeapIntDictionary.java | 2 +- .../index/readers/OnHeapLongDictionary.java | 2 +- .../index/readers/OnHeapStringDictionary.java | 2 +- .../segment/index/readers/RangeIndexReader.java | 2 +- .../segment/index/readers/StringDictionary.java | 2 +- .../readers/bloom/BaseGuavaBloomFilterReader.java | 2 +- .../readers/bloom/BloomFilterReaderFactory.java | 2 +- .../bloom/OffHeapGuavaBloomFilterReader.java | 2 +- .../bloom/OnHeapGuavaBloomFilterReader.java | 2 +- .../forward/BaseChunkSVForwardIndexReader.java | 4 +- .../forward/FixedBitMVForwardIndexReader.java | 2 +- .../forward/FixedBitSVForwardIndexReader.java | 2 +- .../forward/FixedBitSVForwardIndexReaderV2.java | 2 +- .../FixedByteChunkSVForwardIndexReader.java | 2 +- .../forward/VarByteChunkSVForwardIndexReader.java | 2 +- .../readers/geospatial/ImmutableH3IndexReader.java | 2 +- .../readers/json/ImmutableJsonIndexReader.java | 2 +- .../readers/sorted/SortedIndexReaderImpl.java | 2 +- .../index/readers/text/LuceneTextIndexReader.java | 9 +- .../local/segment/store/FilePerIndexDirectory.java | 11 +-- .../segment/local/segment/store/IndexEntry.java | 2 +- .../segment/local/segment/store/IndexKey.java | 1 + .../segment/store/SegmentLocalFSDirectory.java | 92 ++++++++++--------- .../segment/store/SingleFileIndexDirectory.java | 10 ++- .../virtualcolumn/DocIdVirtualColumnProvider.java | 2 +- .../virtualcolumn/VirtualColumnProvider.java | 2 +- .../segment/local/startree/OffHeapStarTree.java | 2 +- .../local/startree/OffHeapStarTreeNode.java | 2 +- .../local/startree/StarTreeBuilderUtils.java | 8 +- .../startree/v2/builder/MultipleTreesBuilder.java | 6 +- .../v2/builder/OffHeapSingleTreeBuilder.java | 2 +- .../startree/v2/builder/StarTreeIndexCombiner.java | 2 +- .../v2/builder/StarTreeV2BuilderConfig.java | 4 +- .../startree/v2/store/StarTreeIndexContainer.java | 4 +- .../startree/v2/store/StarTreeLoaderUtils.java | 6 +- .../apache/pinot/segment/local/utils/CrcUtils.java | 2 +- .../segment/local/utils/SegmentPushUtils.java | 2 +- .../local/utils/fst/PinotBufferIndexInput.java | 2 +- .../io/reader/impl/FixedBitIntReaderTest.java | 2 +- .../segment/local/io/util/PinotDataBitSetTest.java | 2 +- .../local/io/util/PinotDataBitSetV2Test.java | 2 +- .../io/util/VarLengthValueReaderWriterTest.java | 2 +- .../io/writer/impl/MmapMemoryManagerTest.java | 2 +- .../creator/BitmapInvertedIndexCreatorTest.java | 4 +- .../local/segment/creator/IntArraysTest.java | 4 +- .../impl/SegmentColumnarIndexCreatorTest.java | 1 + .../local/segment/index/ColumnMetadataTest.java | 4 +- .../segment/local/segment/index/H3IndexTest.java | 4 +- .../segment/local/segment/index/JsonIndexTest.java | 4 +- .../segment/index/SegmentMetadataImplTest.java | 4 +- .../DefaultNullValueVirtualColumnProviderTest.java | 2 +- .../SegmentV1V2ToV3FormatConverterTest.java | 6 +- .../index/creator/BloomFilterCreatorTest.java | 4 +- .../index/creator/LuceneFSTIndexCreatorTest.java | 4 +- .../index/creator/RangeIndexCreatorTest.java | 4 +- .../segment/index/creator/RawIndexCreatorTest.java | 15 +++- .../SegmentGenerationWithBytesTypeTest.java | 10 ++- .../SegmentGenerationWithFilterRecordsTest.java | 5 +- .../SegmentGenerationWithMultipleRecordsTest.java | 5 +- .../SegmentGenerationWithNoRecordsTest.java | 5 +- .../SegmentGenerationWithTimeColumnTest.java | 11 ++- .../index/creator/SegmentPartitionTest.java | 6 +- .../index/forward/FixedBitMVForwardIndexTest.java | 4 +- .../forward/FixedByteChunkSVForwardIndexTest.java | 2 +- .../forward/SortedForwardIndexReaderTest.java | 2 +- .../forward/VarByteChunkSVForwardIndexTest.java | 6 +- .../local/segment/index/loader/LoaderTest.java | 67 ++++++++------ .../index/loader/SegmentPreProcessorTest.java | 100 ++++++++++++++++----- .../BaseDefaultColumnHandlerTest.java | 7 +- .../FixedByteWidthRowColDataFileReaderTest.java | 2 +- .../index/readers/ImmutableDictionaryTest.java | 4 +- .../ImmutableDictionaryTypeConversionTest.java | 4 +- .../readers/NullValueVectorReaderImplTest.java | 2 +- .../FixedBitSVForwardIndexReaderV2Test.java | 2 +- ...FixedByteWidthRowColForwardIndexWriterTest.java | 4 +- .../store/ColumnIndexDirectoryTestHelper.java | 6 +- .../segment/store/FilePerIndexDirectoryTest.java | 6 +- .../segment/store/SegmentDirectoryPathsTest.java | 3 +- .../segment/store/SegmentLocalFSDirectoryTest.java | 35 ++++---- .../store/SingleFileIndexDirectoryTest.java | 6 +- .../v2/builder/StarTreeV2BuilderConfigTest.java | 4 +- .../segment/local/utils/fst/FSTBuilderTest.java | 2 +- pinot-segment-spi/pom.xml | 4 + .../org/apache/pinot/segment/spi}/V1Constants.java | 4 +- .../segment/spi/index}/creator/TextIndexType.java | 2 +- .../spi}/index/metadata/ColumnMetadata.java | 10 +-- .../spi}/index/metadata/SegmentMetadataImpl.java | 27 +++--- .../segment/spi/loader/SegmentDirectoryLoader.java | 30 +++---- .../spi/loader/SegmentDirectoryLoaderRegistry.java | 89 ++++++++++++++++++ .../pinot/segment/spi/loader/SegmentLoader.java | 8 +- .../segment/spi}/memory/BasePinotLBuffer.java | 2 +- .../pinot/segment/spi/memory}/CleanerUtil.java | 4 +- .../pinot/segment/spi}/memory/PinotByteBuffer.java | 3 +- .../pinot/segment/spi}/memory/PinotDataBuffer.java | 2 +- .../spi}/memory/PinotNativeOrderLBuffer.java | 2 +- .../spi}/memory/PinotNonNativeOrderLBuffer.java | 2 +- .../spi}/partition/ByteArrayPartitionFunction.java | 3 +- .../spi}/partition/HashCodePartitionFunction.java | 3 +- .../spi}/partition/ModuloPartitionFunction.java | 3 +- .../spi}/partition/MurmurPartitionFunction.java | 48 ++++++++-- .../segment/spi/partition/PartitionFunction.java | 1 - .../spi}/partition/PartitionFunctionFactory.java | 3 +- .../metadata}/ColumnPartitionMetadata.java | 2 +- .../segment/spi}/store/ColumnIndexDirectory.java | 12 ++- .../pinot/segment/spi}/store/ColumnIndexType.java | 2 +- .../pinot/segment/spi}/store/SegmentDirectory.java | 34 ++----- .../segment/spi}/store/SegmentDirectoryPaths.java | 10 +-- .../segment/spi}/memory/PinotDataBufferTest.java | 2 +- .../spi}/partition/PartitionFunctionTest.java | 7 +- .../server/api/resources/MmapDebugResource.java | 2 +- .../api/resources/SegmentMetadataFetcher.java | 2 +- .../pinot/server/api/resources/TablesResource.java | 2 +- .../helix/HelixInstanceDataManagerConfig.java | 26 +++++- .../server/starter/helix/HelixServerStarter.java | 2 +- .../starter/helix/SegmentFetcherAndLoader.java | 2 +- .../pinot/server/api/TablesResourceTest.java | 4 +- .../config/instance/InstanceDataManagerConfig.java | 4 + .../org/apache/pinot/tools/SegmentDumpTool.java | 2 +- .../anonymizer/ArrayBasedGlobalDictionaries.java | 2 +- .../pinot/tools/anonymizer/GlobalDictionaries.java | 2 +- .../anonymizer/MapBasedGlobalDictionaries.java | 2 +- .../anonymizer/PinotDataAndQueryAnonymizer.java | 4 +- .../pinot/tools/perf/PerfBenchmarkRunner.java | 2 +- .../apache/pinot/tools/scan/query/Aggregation.java | 2 +- .../apache/pinot/tools/scan/query/Projection.java | 4 +- .../tools/scan/query/SegmentQueryProcessor.java | 4 +- .../apache/pinot/tools/scan/query/Selection.java | 2 +- .../converter/DictionaryToRawIndexConverter.java | 2 +- .../converter/PinotSegmentToAvroConverter.java | 2 +- .../segment/converter/SegmentMergeCommand.java | 2 +- 255 files changed, 970 insertions(+), 637 deletions(-) create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/loader/LocalSegmentDirectoryLoader.java rename {pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl => pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi}/V1Constants.java (96%) rename {pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment => pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index}/creator/TextIndexType.java (93%) rename {pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment => pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi}/index/metadata/ColumnMetadata.java (98%) rename {pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment => pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi}/index/metadata/SegmentMetadataImpl.java (95%) copy pinot-common/src/main/java/org/apache/pinot/common/utils/fetcher/SegmentFetcher.java => pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentDirectoryLoader.java (54%) create mode 100644 pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentDirectoryLoaderRegistry.java copy pinot-spi/src/main/java/org/apache/pinot/spi/config/table/tuner/Tuner.java => pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentLoader.java (85%) rename {pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment => pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi}/memory/BasePinotLBuffer.java (98%) rename {pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils => pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory}/CleanerUtil.java (99%) rename {pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment => pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi}/memory/PinotByteBuffer.java (98%) rename {pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment => pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi}/memory/PinotDataBuffer.java (99%) rename {pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment => pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi}/memory/PinotNativeOrderLBuffer.java (98%) rename {pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment => pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi}/memory/PinotNonNativeOrderLBuffer.java (98%) rename {pinot-segment-local/src/main/java/org/apache/pinot/segment/local => pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi}/partition/ByteArrayPartitionFunction.java (93%) rename {pinot-segment-local/src/main/java/org/apache/pinot/segment/local => pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi}/partition/HashCodePartitionFunction.java (93%) rename {pinot-segment-local/src/main/java/org/apache/pinot/segment/local => pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi}/partition/ModuloPartitionFunction.java (95%) rename {pinot-segment-local/src/main/java/org/apache/pinot/segment/local => pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi}/partition/MurmurPartitionFunction.java (63%) rename {pinot-segment-local/src/main/java/org/apache/pinot/segment/local => pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi}/partition/PartitionFunctionFactory.java (96%) rename {pinot-common/src/main/java/org/apache/pinot/common/metadata/segment => pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/metadata}/ColumnPartitionMetadata.java (99%) rename {pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment => pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi}/store/ColumnIndexDirectory.java (91%) rename {pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment => pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi}/store/ColumnIndexType.java (96%) rename {pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment => pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi}/store/SegmentDirectory.java (83%) rename {pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment => pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi}/store/SegmentDirectoryPaths.java (88%) rename {pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment => pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi}/memory/PinotDataBufferTest.java (99%) rename {pinot-segment-local/src/test/java/org/apache/pinot/segment/local => pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi}/partition/PartitionFunctionTest.java (97%) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org